diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78b6c91..5900c2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,16 +18,67 @@ jobs: with: arch: x64 - - name: Build DLL (Release) + # Git for Windows ships a GNU coreutils `link.exe` in usr/bin; under + # `shell: bash` it shadows the MSVC linker on PATH, so cargo/rustc link + # steps fail with a GNU `link` invoked on MSVC args. Remove it so cargo + # finds MSVC's link.exe. + - name: Remove GNU link.exe (shadows MSVC linker under bash) + shell: pwsh + run: | + $p = "C:\Program Files\Git\usr\bin\link.exe" + if (Test-Path $p) { Remove-Item $p -Force } + + # The full (RAG) build produces the rcore.dll cdylib via `cargo build` + # (the lite C++ build needs no cargo), and the Rust-core tests below need + # cargo. Pin the toolchain for a deterministic, msvc-targeted build. + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-msvc + + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust-core + + - name: Test Rust core + shell: bash + run: cargo test --manifest-path rust-core/Cargo.toml + + - name: Build DLL (Release, lite) + smoke harness shell: bash run: | cd http-1c-dll rm -rf build mkdir build cd build - cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release + cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DHTTP1C_BUILD_TESTS=ON cmake --build . + # Black-box smoke: loads the freshly built bin/libhttp1cWin.dll the way + # 1C does (LoadLibrary + GetClassObject + stubbed 1C interfaces) and + # drives it over real HTTP (T1-T9: init, tools/list union, reserved-name + # rejection, session resurrection, auth races, timeout, UTF-8, native + # list_collections). rcore.dll is not in bin/ here, so the native tools + # exercise the structured rag_not_installed path. + - name: Run smoke harness (black-box, bin/libhttp1cWin.dll) + shell: bash + run: ./http-1c-dll/bin/http1c_smoke.exe + + # Full ("RAG") build check: compile the rcore.dll cdylib with the + # fastembed feature and the dynamic CRT (ort/onnxruntime needs /MD). This + # catches fastembed/ort build breakage without re-running CMake. The model + # is fetched at runtime, so nothing is downloaded here. Rust is cached + # above (Swatinem/rust-cache), so this stays cheap after the first run. + - name: Build rcore.dll (fastembed, dynamic CRT) + shell: bash + env: + RUSTFLAGS: "-C target-feature=-crt-static" + run: | + cargo build --release --features fastembed \ + --target x86_64-pc-windows-msvc \ + --manifest-path rust-core/Cargo.toml + - name: Build tests shell: bash run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a8e5269..b4f14bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ permissions: jobs: build: runs-on: windows-latest - + steps: - uses: actions/checkout@v4 @@ -26,7 +26,30 @@ jobs: with: arch: x64 - - name: Build DLL (Release) + # Git for Windows' GNU `link.exe` (usr/bin) shadows the MSVC linker under + # `shell: bash`, breaking cargo/rustc link steps — remove it. + - name: Remove GNU link.exe (shadows MSVC linker under bash) + shell: pwsh + run: | + $p = "C:\Program Files\Git\usr\bin\link.exe" + if (Test-Path $p) { Remove-Item $p -Force } + + # The full ("RAG") build invokes `cargo build` (via CMake) to produce the + # rcore.dll cdylib (fastembed search core), so the Rust toolchain must be + # present. The lite build is pure C++ and needs no cargo. + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-msvc + + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust-core + + # ---- LITE variant: libhttp1cWin.dll alone (no cargo, search returns + # "install RAG"). The committed Template.bin tracks this bundle. ---- + - name: Build DLL (Release, lite) shell: bash run: | cd http-1c-dll @@ -47,9 +70,29 @@ jobs: cmake --build . ./http1c_tests.exe - - name: Package add-in + - name: Package add-in (lite) + shell: bash + run: VARIANT=lite bash build/package-http1c-addin.sh + + # ---- FULL variant: also build rcore.dll (real fastembed search core). + # -DRCORE_FASTEMBED=ON makes CMake invoke cargo with the dynamic-CRT + # RUSTFLAGS and drop rcore.dll next to the component. The model is fetched + # at RUNTIME, so nothing extra is downloaded at build time. Packaging adds + # rcore.dll + DirectML.dll (from System32) to the bundle; SKIP_TEMPLATE=1 + # keeps the committed lite Template.bin from being clobbered. ---- + - name: Build DLL + rcore.dll (Release, full) + shell: bash + run: | + cd http-1c-dll + rm -rf build + mkdir build + cd build + cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DRCORE_FASTEMBED=ON + cmake --build . + + - name: Package add-in (full) shell: bash - run: bash build/package-http1c-addin.sh + run: VARIANT=full SKIP_TEMPLATE=1 bash build/package-http1c-addin.sh - name: Extract version id: version @@ -70,7 +113,8 @@ jobs: shell: bash run: | VERSION="${{ steps.version.outputs.VERSION }}" - cp "build/artifacts/http1c-addin.zip" "build/artifacts/http1c-addin-v${VERSION}.zip" + cp "build/artifacts/http1c-addin.zip" "build/artifacts/http1c-addin-lite-v${VERSION}.zip" + cp "build/artifacts/http1c-addin-full.zip" "build/artifacts/http1c-addin-full-v${VERSION}.zip" - name: Create Release uses: softprops/action-gh-release@v2 @@ -79,4 +123,7 @@ jobs: name: "http1c v${{ steps.version.outputs.VERSION }}" generate_release_notes: true files: | - build/artifacts/http1c-addin-v${{ steps.version.outputs.VERSION }}.zip + build/artifacts/http1c-addin-lite-v${{ steps.version.outputs.VERSION }}.zip + build/artifacts/http1c-addin-full-v${{ steps.version.outputs.VERSION }}.zip + build/artifacts/http1c-addin.zip + build/artifacts/http1c-addin-full.zip diff --git a/.gitignore b/.gitignore index 36a9efb..c31a2f4 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ build/artifacts/ # CMake and out-of-source build directories http-1c-dll/build/ +http-1c-dll/build-smoke/ http-1c-dll/artifacts/ tests/build/ build_dbg/ diff --git a/README.md b/README.md index 4f25bfc..4700d00 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ This means the project is not a fixed set of built-in utilities. It is an MCP tr - **Tool annotations** — `readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint` - **Output schemas** — typed response contracts for tool results - **Dynamic registration** — register/update tools, resources, prompts at runtime from 1C +- **Built-in semantic search (RAG)** — optional `search` / `grep` / `get_segment` / `list_collections` tools backed by a Rust search core (`rcore`) with dense/keyword/hybrid retrieval (see [Search Subsystem](#search-subsystem-rag)) ## Architecture @@ -260,11 +261,13 @@ Component.AuthToken = ""; The component also enforces: - **Origin validation** — only requests from `localhost` / `127.0.0.1` / VS Code origins are accepted - **Rate limiting** — token-bucket algorithm (60 burst, 20/sec) -- **Session management** — `Mcp-Session-Id` assigned on initialize, validated on subsequent requests +- **Session management** — `Mcp-Session-Id` assigned on initialize, validated on subsequent requests. A request presenting an **unknown** session id is not rejected with 404 — the session is transparently **resurrected** under the presented id, so a server restart does not strand connected MCP clients that never re-initialize (logged as `MCP: unknown session ... resurrected (server restart?)`) ## Native Component API -Methods exposed to 1C (English / Russian names): +Methods exposed to 1C (English / Russian names). All of these are callable +asynchronously from BSL via the platform-generated `BeginCalling` wrappers +(`BeginCallingStartListen`, …): | Method | Description | |--------|-------------| @@ -272,20 +275,45 @@ Methods exposed to 1C (English / Russian names): | `StopListen()` / `ОстановитьПрослушивание` | Stop the server and unblock all pending requests | | `SendResponse(json)` / `ОтправитьОтвет` | Send the final response for a pending request | | `SendProgress(id, progress, total, message)` / `ОтправитьПрогресс` | Send a progress notification for a pending request | - -Properties exposed to 1C: - -| Property | Type | Description | -|----------|------|-------------| -| `Status` / `Статус` | Read-only | Returns JSON with server status | -| `Timeout` / `Таймаут` | Read/Write | Response timeout in seconds (default: 30) | -| `AuthToken` / `ТокенАвторизации` | Write-only | Bearer token for authentication (empty = no auth) | -| `LoggingEnabled` / `ЛогированиеВключено` | Read/Write | Whether runtime logging is active | -| `LogPath` / `ПутьЛога` | Read/Write | Path to the log file | -| `Tools` / `Инструменты` | Write-only | Register/update the tool list (JSON array) | -| `Resources` / `Ресурсы` | Write-only | Register/update the resource list (JSON array) | -| `Prompts` / `Промпты` | Write-only | Register/update the prompt list (JSON array) | -| `Version` / `Версия` | Read-only | Component version string | +| `ApplyConfig(json)` / `ПрименитьНастройки` | **Async-safe config sink** (see below). Applies any subset of `logging_enabled`, `log_path`, `timeout`, `tools_json`, `resources_json`, `prompts_json`, `auth_token` in one call | +| `GetStatus()` / `ПолучитьСтатус` | **Async-safe** read of the status JSON (same payload as the `Status` property) | +| `RagDispatch(method, payload)` / `RagВыполнить` | Drive the Rust search core (`rcore`) — see [Search Subsystem](#search-subsystem-rag) | +| `TakeScreenshot(pid, format, quality, grayscale)` / `СделатьСкриншот` | Capture windows of a process as base64 images | +| `GetProcessId()` / `ПолучитьИдентификаторПроцесса` | Return the current 1C process id | + +### Configuration: async methods vs. synchronous properties + +The component also exposes its configuration as **properties** (`LoggingEnabled`, +`LogPath`, `Timeout`, `Tools`, `Resources`, `Prompts`, `AuthToken`, `Status`, +`Version`). These still exist, but reading or writing an AddIn property is a +**synchronous** platform call. + +> In an **async-only infobase** — i.e. one where *"synchronous extension and +> add-in calls"* are disabled (the modern 1C default) — every `Component.Prop = …` +> assignment and every `… = Component.Prop` read throws +> **`Cannot call synchronous methods on the client!`**. 1C provides **no async +> property accessors** (there is no `BeginSetProperty`), so configuration cannot +> go through properties at all in that mode. + +That is why all configuration is funneled through the **async `ApplyConfig` +method** and all status reads through the **async `GetStatus` function**. The +reference form (`Module.bsl`) uses only these — it never touches a property on +the client. The properties are retained for backward compatibility and for +infobases that still allow synchronous calls. + +| Config field (in `ApplyConfig` JSON) | Equivalent property | Notes | +|--------------------------------------|---------------------|-------| +| `logging_enabled` (bool) | `LoggingEnabled` | runtime logging on/off | +| `log_path` (string) | `LogPath` | log file path (empty → default) | +| `timeout` (int) | `Timeout` | response timeout, seconds | +| `tools_json` (string) | `Tools` | pre-serialized tool-list JSON array | +| `resources_json` (string) | `Resources` | pre-serialized resource-list JSON array | +| `prompts_json` (string) | `Prompts` | pre-serialized prompt-list JSON array | +| `auth_token` (string) | `AuthToken` | Bearer token (empty = no auth) | +| *(read via `GetStatus`)* | `Status` / `Version` | status JSON includes `version` | + +Every field is optional — `ApplyConfig` only touches the keys you send, so it +doubles as a "set just the logging flag" call or a full one-shot configuration. ## ExternalEvent Types @@ -308,7 +336,7 @@ Events sent from the native component to 1C: | `notifications/initialized` | Native — accepted silently | | `ping` | Native — returns empty result | | `tools/list` | Native — paginated, from cache | -| `tools/call` | Delegated to 1C via ExternalEvent | +| `tools/call` | `search` / `grep` / `get_segment` / `list_collections` handled natively by the search core; all other tools delegated to 1C via ExternalEvent | | `resources/list` | Native — paginated, from cache | | `resources/read` | Delegated to 1C via ExternalEvent | | `prompts/list` | Native — paginated, from cache | @@ -358,6 +386,149 @@ Events sent from the native component to 1C: | `analyze1CData` | `topic` (optional) | System prompt for data analysis with metadata context | | `generate1CCode` | `task` (required) | System prompt for BSL code generation with conventions | +## Search Subsystem (RAG) + +The component ships with an optional semantic-search subsystem so a 1C MCP server can index its own content (documentation, code, catalog descriptions, logs, QA scenarios, master data) and let AI clients search it **by meaning**. The retrieval engine is a small Rust core, crate `rcore`, that runs entirely in-process inside the 1C session. + +### Why this exists + +An MCP server already lets an AI client *act* inside a specific 1C installation (run a query, call a method, execute a test). What it lacked was the ability to *find by meaning* the thing to act on. That is retrieval — the **"R" in RAG** — and it is what this subsystem adds. Generation stays with the client model; we only retrieve and ground. + +Concretely, it closes gaps like: an AI writing a Vanessa BDD test needs the **catalog of available steps** and the **existing scenarios** so it reuses real, runnable phrases instead of hallucinating step text; or finding a product/client **by intent** ("that construction contractor from Kyiv", "red winter jacket") rather than by exact code. + +### Design decisions (and why) + +The shape of the subsystem follows directly from its constraints — they are part of the problem, not incidental: + +| Decision | Rationale | +|----------|-----------| +| **Content-agnostic core; domain logic lives in outside adapters** | The core never knows about Gherkin, SKUs, or INNs. Adapters (in 1C/BSL) parse the domain and feed the core generic records. One engine serves QA, products, clients, docs — instead of N bespoke searches. | +| **In-process, in-memory, no disk persistence** | Data is corporate and session-bound (Vanessa runs in the client session); the index lives inside the 1C process and dies with it. A stale cache is worse than none for grounding, so the index is rebuilt from the live source on load. | +| **Flat brute-force vector index (no ANN/Qdrant)** | Target scale is ~5–10k records per session. Search is microseconds and RAM is tens of MB — an ANN index would be pure complexity. | +| **Hybrid retrieval (dense + keyword), not pure semantic** | Half the queries are exact identifiers (SKU, INN, step syntax) where embeddings fail; half are intent where exact match fails. Both channels are needed; keyword is a full scan (cheap at this scale). | +| **Asynchronous ingest (push from 1C)** | Embedding 5–10k records takes 15–60 s and must never freeze the thin-client UI. Ingest calls return immediately; a background worker embeds off-thread. Progress is observed by polling `stats` / `list_collections`. | +| **Async-only config/status methods** | 1C forbids synchronous AddIn property access in async-only infobases, so configuration and status go through the async `ApplyConfig`/`GetStatus` methods (see [Native Component API](#native-component-api)). | +| **`text` vs. `embed_text` split** | The single biggest quality lever — the adapter decides what gets embedded (an enriched composite) separately from what is stored/returned verbatim, without polluting the core with domain fields. | + +### MCP tools + +Four search tools are served **natively by the component** — they are handled inside the DLL and are *not* forwarded to 1C via `ExternalEvent`: + +| Tool | Purpose | +|------|---------| +| `search` | Semantic / keyword / hybrid ranked search over indexed segments. Returns hits with `doc_id`, `name`, `collection`, `score`, segment line ranges, (optionally) text, and the hit's **effective `meta`** — document meta overlaid by segment meta (segment wins on collision), the same view the meta filters match against. | +| `grep` | RE2 regex search (linear time, no backreferences/lookaround) over the stored document text. Works the instant a document is ingested — no vectors needed. | +| `get_segment` | O(1) line-range slice of a raw document by `doc_id` via an offset table. Out-of-range requests are clamped and the actual range is returned. | +| `list_collections` | The collection registry — AI-facing discovery of what is searchable before querying (see [Collections](#collections-and-the-collection-registry)). Takes no arguments. | + +`search` supports three retrieval modes via the `mode` argument: + +- **dense** (default) — cosine/dot-product similarity over normalized embedding vectors. +- **keyword** — lexical scoring, no vectors required. +- **hybrid** — Reciprocal Rank Fusion (RRF) of the dense and keyword result lists. + +Both `search` and `grep` accept metadata filters (`all`/`any` clauses) and a `collection` to scope the query. + +### Ingest + +Documents are pushed into the core through `rcore`'s JSON ABI. Ingest is **asynchronous**: the call is accepted under a short lock and returns immediately, while a background worker embeds the segments off-thread. Text-based tools (`grep`, `get_segment`) work right after acceptance; dense `search` becomes available once embedding finishes (`stats` exposes per-collection progress). + +- **`index_segments`** — ingest a document as a list of pre-chunked segments (each with optional `embed_text`, line range, and per-segment `meta`). `doc_id` is required so segments can be upserted/deleted. +- **`index_raw`** — ingest a raw multi-line document; the core normalizes line endings, builds a line-offset table, stores the full text, and chunks it (line-snapped, by token budget, with overlap). `doc_id` is optional (auto-assigned and returned in the ack). Accepts the same optional `collection_description` as `index_segments` (last non-empty wins). + +Each segment carries two distinct texts: + +- **`text`** — stored and returned verbatim (the canonical record). +- **`embed_text`** — the enriched composite that actually goes into the vector (name + tags + steps for a scenario; name + brand + category for a product). If omitted, `text` is embedded. This is where the adapter spends its quality budget. + +### Collections and the collection registry + +Every document belongs to a **collection** (a named bucket: `qa_steps`, `products`, `clients`, …). Collections are created implicitly on first ingest and can carry a free-text **description**. + +- **`list_collections`** returns the registry: for each collection its `name`, `description`, `n_docs`, `n_segments`, `vector_status` (`empty` / `building` / `ready` / `error`), and `text_ready`. This is how an AI client **discovers what is searchable** before querying — it is exposed both as a `RagDispatch` method and as the fourth **native MCP tool** (no arguments): + + ```json + { "collections": [ + { "name": "qa_steps", "description": "Vanessa BDD step catalog", + "n_docs": 2, "n_segments": 780, "text_ready": true, "vector_status": "ready" } + ] } + ``` +- **`search`** scopes by `collection`: omit it to search **all** collections, pass one name, or pass a **comma-separated list** (or a `collections` array) to search a chosen subset. So an agent can read the registry, pick the relevant collections by description, and search just those. + +### Embedding worker pool + +The embedder runs on a background pool whose size is set by `embed_workers` in `configure` (settable from the 1C form). One worker (default) keeps a single ONNX session and serializes embedding; multiple workers run several sessions concurrently. On CPU this trades memory for throughput (roughly memory-bandwidth bound, not linear in workers); multi-worker forces CPU because concurrent DirectML sessions are not supported. Query embeddings are prioritized so a bulk reindex does not starve live search latency. + +### Core method contract (`RagDispatch`) + +All of the following are invoked from 1C via the async `RagDispatch(method, payload)` method (the C++ component forwards the JSON envelope to `rcore` verbatim and does not interpret the method name). Each returns `{"ok":true,"result":…}` or `{"ok":false,"error":{"code","message"}}`: + +| Method | Purpose | +|--------|---------| +| `configure` | Load the model once — `model` (a built-in name from the whitelist: `multilingual-e5-small` (default) / `multilingual-e5-base` / `multilingual-e5-large`; an unknown name fails with a structural `bad_model` error listing the supported names), `cache_dir` (directory the built-in model is downloaded into / cached in), `model_path` (offline local files, bypasses the whitelist), `device`, `normalize`, `max_seq_len`, `intra_threads`, `embed_workers`. `dim` is fixed by the model. Idempotent; re-configuring with a different model/dim after indexing implies `reset`. | +| `index_segments` | Async ingest of pre-chunked segments (with `embed_text`, line ranges, per-segment `meta`, optional collection `description`). | +| `index_raw` | Async ingest of a raw document; the core chunks it (line-snapped, token-budgeted, overlapping) and builds the offset table. Same optional `collection_description` as `index_segments` (last non-empty wins). | +| `search` | Ranked retrieval — `mode` dense/keyword/hybrid, `collection`(s), meta filters, `k`, `min_score`, `max_per_doc`, `include_text`. Hits echo the **effective** meta (doc overlaid by segment, segment wins) — the same view the filters match. | +| `get_segment` | O(1) line-range slice of a raw document by `doc_id`. | +| `grep` | RE2 regex / substring scan over stored text. | +| `stats` | Global + per-collection counters, model, dim, memory estimate, collection statuses. | +| `list_collections` | The collection registry (names + descriptions + counts + status). | +| `list_models` | The built-in model whitelist — `{default, models:[…]}`. Works before `configure` and in the lite/mock build, so a client can always render a model picker. | +| `delete_document` / `delete_collection` | Remove a document (by `doc_id`) or a whole collection. | +| `reset` | Clear the entire index. | + +### Working with the subsystem — for the 1C developer + +You write **adapters**: thin BSL that turns a domain object into generic records and pushes them in. The core stays domain-free. + +1. **Attach + configure once.** Attach the full component, then `configure` via `RagDispatch` with a built-in `model` name (or a local `model_path`, plus optionally `cache_dir`) and `embed_workers`. (Component-level settings — logging/timeout — go through `ApplyConfig`, never through synchronous properties.) +2. **Index asynchronously.** For each domain object build segments — set `text` (verbatim) and `embed_text` (the enriched composite), attach `meta` (`{sku, inn, type, tags, …}`) and a `doc_id` for upsert — and call `index_segments`. The call returns immediately; embedding happens in the background. +3. **Watch progress** by polling `list_collections` / `stats` until `vector_status = ready`. Text tools (`grep`, `get_segment`) work the instant a document is accepted; dense `search` lights up when embedding finishes. +4. **Keep it fresh within the session.** When one object changes, re-`index_segments` just that `doc_id` (atomic upsert) or `delete_document` — never rebuild the whole corpus. +5. **All file/content reads stay client-side** (`&AtClient`) in the reference form, so source files are read from the operator's machine, not the server. + +The reference end-to-end flow is exercised headlessly by the RAG self-test (stub QA/products/clients adapters → index → embed → search, asserting pass/fail). See the `onec-rag-selftest` skill. + +### Working with the subsystem — for the AI agent / MCP client + +From the client side it is just four MCP tools, but used in a deliberate order: + +1. **Discover.** Call `list_collections` first to see what this specific 1C base has indexed and read each collection's description — don't assume; the corpus is per-installation and per-session. +2. **Search by meaning, scoped.** Call `search` with `mode: hybrid` for anything involving exact identifiers (SKU/INN/article/step syntax) and `dense` for pure intent. Scope to the relevant `collection`(s) from step 1, or omit to search everything. Use meta filters (`all`/`any`) to narrow. +3. **Ground, don't hallucinate.** Treat hits as the source of truth for *this* base. For QA, prefer canonical step phrases and existing scenarios returned by search over inventing them; a reverse step→scenario lookup (a QA-adapter convenience over the same `search`) surfaces real usages with real parameters. +4. **Zoom in.** Use `get_segment` to pull the exact line range of a hit verbatim, and `grep` for exact-string / regex confirmation (it works even before vectors are ready). A `building` collection returns partial results flagged as incomplete — retry once it is `ready`. + +The contract is: **the client retrieves and grounds on what actually exists in this base right now; it never swallows the whole corpus into context and never relies on exact match alone.** + +### Lite vs full + +The same `libhttp1cWin.dll` ships in **two distributions**. The component is pure C++ built `/MT` and does **not** link the Rust core; instead it loads an optional **`rcore.dll`** at runtime (`LoadLibrary` + `GetProcAddress`, resolved next to the component DLL — see `http-1c-dll/src/RustCore.h`): + +| Distribution | Contents | Search behavior | +|--------------|----------|-----------------| +| **lite** | `libhttp1cWin.dll` only | `search` / `grep` / `get_segment` / `list_collections` are advertised in `tools/list` but return a structured error (`code: rag_not_installed`) telling the caller to install the RAG package. | +| **full** | `libhttp1cWin.dll` + `rcore.dll` + `DirectML.dll` | Real fastembed/onnxruntime search core; the four tools run for real. | + +Detection is automatic at runtime: if `rcore.dll` is present next to the component **and** all four ABI entry points (`rcore_version` / `rcore_dispatch` / `rcore_free_string` / `rcore_shutdown`) resolve, the component runs the full search path. A missing, partial, or version-mismatched `rcore.dll` degrades cleanly to the lite path (an "install RAG" error) — never a crash. The tool schemas are identical in both distributions, so a client sees the same `tools/list` either way. + +### GPU / CPU (DirectML) + +The full search core uses onnxruntime's **DirectML** execution provider for GPU acceleration, with **automatic CPU fallback** when no compatible GPU is available. The device is selected via the `configure` method's `device` field: + +| `device` | Behavior | +|----------|----------| +| `auto` (default) | DirectML with onnxruntime's automatic CPU fallback. | +| `dml` | Force the DirectML (GPU) execution provider. | +| `cpu` | Force CPU execution. | + +> **`DirectML.dll` is a hard dependency of the full package.** ort's prebuilt onnxruntime bundles the DirectML provider, so `rcore.dll` hard-imports `DirectML.dll`. The **full** bundle must ship `DirectML.dll` alongside `rcore.dll`, otherwise `rcore.dll` fails to load and the component silently falls back to the lite "install RAG" behavior. On Windows it lives at `C:\Windows\System32\DirectML.dll`. (Version-coupling caveat: this `DirectML.dll` is paired with the bundled onnxruntime; a future CPU-only onnxruntime build would remove the import and this DLL could be dropped.) + +The embedding model itself is **fetched at runtime**, not at build time — nothing model-related is downloaded during the build or in CI. + +## Vanessa Automation Integration + +The repository ships a ready-made integration for [Vanessa Automation](https://github.com/Pr-Mex/vanessa-automation) (1C BDD testing): the **MCPRagSearch** plugin in [`vanessa-plugin/`](vanessa-plugin/README.md). It runs the http1c component inside the VA session and serves Vanessa's **entire MCP contract** through it — all VA tools are collected from MCPVA via a small facade (a micro-hook already present in the VA fork), while `search` / `grep` / `get_segment` / `list_collections` run natively over the indexed VA step library (`vanessa_steps`), the VA knowledge base (`vanessa_kb`), and arbitrary file collections. See [`vanessa-plugin/README.md`](vanessa-plugin/README.md) (Russian) for setup, the collections, the `search-guide` prompt / `get_search_guide` tool, and the auto-reindex watcher. + ## Repository Layout ```text @@ -378,7 +549,13 @@ Events sent from the native component to 1C: │ └── src/ │ ├── AddInNative.cpp/h # Generic 1C add-in framework │ ├── HttpServerComponent.cpp/h # MCP server implementation +│ ├── RustCore.h # Runtime loader for rcore.dll (search) │ └── AddInNative.def # DLL export definitions +├── rust-core/ # `rcore` — Rust search core (rcore.dll) +│ ├── Cargo.toml # cdylib crate, `fastembed` feature +│ └── src/ # dispatch, embed, grep, filter, core +├── vanessa-plugin/ +│ └── MCPRagSearch/ # Vanessa Automation plugin (MCP server + RAG) ├── http-1c-dp/ │ ├── http1c.xml # 1C data processor XML source │ └── http1c/ @@ -395,6 +572,7 @@ Events sent from the native component to 1C: - **[nlohmann/json](https://github.com/nlohmann/json)** — JSON parser - **1C Native API** — integration with 1C:Enterprise - **OneScript** — EPF compilation from XML +- **Rust** (`rcore` crate) + **[fastembed](https://github.com/Anush008/fastembed-rs)** / **onnxruntime (ort)** — optional search core in `rcore.dll` (full distribution only) Based on [lintest/AddinTemplate](https://github.com/lintest/AddinTemplate) for the native add-in layer. @@ -457,6 +635,46 @@ After a successful DLL build, the top-level build scripts also package the nativ http-1c-dp/http1c/Templates/http1c/Ext/Template.bin ``` +### Building the search core (full distribution) + +By default the build produces the **lite** component only — pure C++, no Rust/cargo involved. To also build the **full** search core (`rcore.dll`), configure CMake with `-DRCORE_FASTEMBED=ON`: + +```bash +cd http-1c-dll && mkdir -p build && cd build +# lite — libhttp1cWin.dll alone (no cargo): +cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build . + +# full — also builds rcore.dll via cargo: +cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DRCORE_FASTEMBED=ON +cmake --build . +``` + +The full build invokes `cargo build --release --features fastembed` (handled inside CMake) and drops `rcore.dll` next to the component in `http-1c-dll/bin/`. The embedding model is fetched at runtime, so no model is downloaded at build time. + +**Why two binaries (`/MT` vs `/MD`).** The C++ component is built with the **static** CRT (`/MT`) — it cannot compile under `/MD` because MSVC's `char16_t` stream code hits error C2491. onnxruntime (via `ort`), however, requires the **dynamic** CRT (`/MD`). Linking the two together is impossible, so the search core is a *separate* `rcore.dll` (a cdylib) that the `/MT` component loads at runtime instead of linking. CMake builds the cdylib with the dynamic CRT by removing the `+crt-static` default (`RUSTFLAGS=-C target-feature=-crt-static`); this is independent of the component's `/MT` and they never link together. + +### Packaging the add-in (lite / full) + +`build/package-http1c-addin.sh` assembles a 1C add-in ZIP (and updates `Template.bin`). The variant is selected via the `VARIANT` env var: + +```bash +# lite (default) — libhttp1cWin.dll only: +VARIANT=lite bash build/package-http1c-addin.sh + +# full — also bundles rcore.dll + DirectML.dll: +VARIANT=full bash build/package-http1c-addin.sh +``` + +The **full** packaging step copies `DirectML.dll` from `C:\Windows\System32\DirectML.dll` (override with `DIRECTML_SRC`) into the bundle — it is a hard dependency of `rcore.dll` (see [GPU / CPU](#gpu--cpu-directml)) and the script fails clearly if it is missing. Set `SKIP_TEMPLATE=1` to produce only the ZIP without overwriting the committed `Template.bin` (which tracks the lite bundle). + +The release workflow (`.github/workflows/release.yml`) builds both variants and attaches both ZIPs to the GitHub release: + +```text +http1c-addin-lite-v.zip # libhttp1cWin.dll +http1c-addin-full-v.zip # libhttp1cWin.dll + rcore.dll + DirectML.dll +``` + ## Running the 1C Side The repository includes an external data processor: @@ -554,9 +772,13 @@ Without these steps the platform may silently load an outdated DLL or refuse to The native component version defined in the source is: ```text -1.3.0 +1.5.0 ``` +Recent changes: + +- **1.5.0** — Async-only config/status. Added the async `ApplyConfig` and `GetStatus` methods so the component is fully usable from async-only infobases (where synchronous AddIn property access throws *"Cannot call synchronous methods on the client"*). The reference form no longer touches a property on the client. Search subsystem gained the **collection registry** (`list_collections`, per-collection descriptions, multi-collection `search`) and a configurable **embedding worker pool** (`embed_workers`). + ## License This project is licensed under the MIT License. See `LICENSE`. \ No newline at end of file diff --git a/TESTING.md b/TESTING.md index 42a771b..42c0c8d 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2,6 +2,46 @@ This guide is designed for an AI agent (such as GitHub Copilot in VS Code) that connects to the http1c MCP server. All tests are performed by invoking MCP tools, reading resources, and using prompts through the standard MCP client interface — not via raw HTTP/curl. +## Automated C++ smoke harness + +`http-1c-dll/tests/smoke/main.cpp` builds into a console exe (`http1c_smoke.exe`) +that needs **no 1C at all**: it loads the built `bin/libhttp1cWin.dll` exactly +like 1C:Enterprise does (`LoadLibrary` → `GetClassObject("HttpServer")` → +`setMemManager` → `Init` with stubbed `IMemoryManager` / +`IAddInDefBase(+Ex)` interfaces), then drives the component over real HTTP and +asserts behavior. CI runs it on every push (see `.github/workflows/ci.yml`); +`build/run-tests.sh` also builds and runs it after the unit tests. + +**Build & run locally** (from a VS Developer/vcvars x64 prompt): + +```bash +cd http-1c-dll +mkdir build && cd build +cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DHTTP1C_BUILD_TESTS=ON +cmake --build . # or: cmake --build . --target http1c_smoke +../bin/http1c_smoke.exe # optional arg: path to libhttp1cWin.dll +``` + +The exe exits 0 when everything passes (nonzero = number of failed checks) and +prints PASS/FAIL per test: + +| Test | Covers | +|------|--------| +| T1 | DLL load, `GetClassObject`, `Init`, call plumbing (`GetProcessId` = current pid) | +| T2 | `ApplyConfig` (`tools_json`, `timeout`, logging off) → `StartListen` → MCP `initialize` returns `serverInfo` + `Mcp-Session-Id` | +| T3 | `tools/list` is the union of 1C tools + the 4 native search tools, names unique, count = 5 | +| T4 | Reserved-name rejection: registering a tool named `search` raises `AddError` ("reserved…") and leaves the tool cache unchanged | +| T5 | Unknown `Mcp-Session-Id` is transparently resurrected (HTTP 200, not 404) | +| T5b | Session-map cap: a paced flood of 300 unknown session ids is all served, yet `GetStatus().active_sessions` stays ≤ 256 (LRU eviction, no memory DoS) | +| T6 | Bearer auth (401 without / 200 with token) + concurrency smoke: 4×50 requests racing `ApplyConfig` token flips — only 200/401/429 allowed (429 = the component's own rate limiter), no 5xx/hang | +| T7 | `tools/call` of a 1C tool with `timeout: 1`: `ExternalEvent(ToolCall)` reaches the stub and the "did not respond within 1 second" error returns within ~3 s | +| T8 | UTF-8 round-trip: Russian tool name/description come back byte-identical through `MB2WCHAR`/`WCHAR2MB` | +| T9 | Native `list_collections` end-to-end: ok-collections JSON (with `rcore.dll` beside the DLL) or structured `rag_not_installed` (lite) — both valid | +| T10 | Late `rcore.dll` install: a second component copy in a temp dir without `rcore.dll` answers `rag_not_installed` (message names all 4 native tools); after copying `rcore.dll` beside it the SAME loaded module flips to full — no process restart (flip stage skips on the lite CI build) | + +The harness never enables component logging and always finishes with +`StopListen`, so it leaves no log files and no dangling threads. + ## Prerequisites 1. The `http1c.epf` data processor is open in 1C:Enterprise. @@ -349,7 +389,7 @@ curl -s -w "\nHTTP:%{http_code}" -X POST http://localhost:8888/mcp \ --- -## Test 20: Security — Invalid Session +## Test 20: Unknown Session — Transparent Resurrection **Action:** Send a request with a non-existent session ID: ```bash @@ -359,7 +399,7 @@ curl -s -w "\nHTTP:%{http_code}" -X POST http://localhost:8888/mcp \ -d '{"jsonrpc":"2.0","id":1,"method":"ping"}' ``` -**Expected:** HTTP 404, body: `{"error":"Session not found"}` +**Expected:** HTTP 200 with a normal `ping` result — an unknown session ID is **not** rejected with 404; the session is transparently **resurrected** (recreated under the presented ID). This keeps IDE clients alive across server restarts (they typically never re-initialize after a 404). With logging enabled, the log contains `MCP: unknown session ... resurrected (server restart?)`. See also Test 38. --- @@ -396,7 +436,7 @@ curl -s -w "\nHTTP:%{http_code}" -X DELETE http://localhost:8888/mcp \ -H "Mcp-Session-Id: $SESSION_ID" ``` -**Expected:** HTTP 200. Subsequent requests with this session ID return 404. +**Expected:** HTTP 200 — the server-side session state is freed. Note: a *subsequent* request that still presents this session ID is served again (the ID is transparently resurrected, see Test 20); `DELETE` releases resources, it does not blacklist the ID. --- @@ -498,6 +538,248 @@ curl -s -w "\nHTTP:%{http_code}" -X DELETE http://localhost:8888/mcp \ --- +## Search Subsystem Tests (RAG) + +These tests cover the `search` / `grep` / `get_segment` / `list_collections` +tools served natively by the component (see the +[Search Subsystem](README.md#search-subsystem-rag) docs). +They are **always advertised in `tools/list`** — so the tool counts in Test 1 +and Test 28 are actually **+4** (the demo tools plus `search`, `grep`, +`get_segment`, `list_collections`). + +**Distribution matters:** +- **lite** (`libhttp1cWin.dll` only) — the four tools are listed but every call + returns a structured `rag_not_installed` result. Run Tests 30–31 only. +- **full** (`+ rcore.dll + DirectML.dll`) — the tools run for real. Run all of + Tests 30–40. + +**Prerequisite for the "full" tests:** the operator must have **indexed some +content first**. Indexing, `configure`, and `stats` are **1C-side admin +operations** (driven from the form via the component's `RagDispatch` method) — +they are *not* MCP tools and are not callable by the client. Use the form's +**Sync** button (or the headless self-test) to populate at least one collection +before searching. `list_collections` exists on both sides: as a `RagDispatch` +method for the operator and as the fourth native MCP tool (Test 37) — either +way it is how you confirm a collection reached `vector_status: ready`. + +--- + +## Test 30: List Tools — Includes search / grep / get_segment / list_collections + +**Action:** Ask Copilot to list the tools available from `1c-mcp-server`. + +**Expected:** In addition to the demo + screenshot tools, four search tools are present: +- `search` — semantic / keyword / hybrid ranked search +- `grep` — RE2 regex / substring scan over stored text +- `get_segment` — O(1) line-range slice of a document by `doc_id` +- `list_collections` — the collection registry (no arguments) + +**Verify:** +- All four tools appear with descriptions and an `inputSchema`. +- `search` exposes `query`, `collection`, `mode` (dense/keyword/hybrid), `k`, and meta-filter parameters. +- The schemas are identical whether the lite or full distribution is loaded. + +--- + +## Test 31: search — Lite distribution / backend not installed + +**Action:** On a **lite** install (or before `rcore.dll` is present), call `search` with `query`: `"anything"`. + +**Expected result:** A structured, non-crashing result with `code: rag_not_installed` and a message telling the caller to install the RAG (full) package. + +**Verify:** +- The server stays up; the error is a normal tool result, not a transport failure. +- The same applies to `grep`, `get_segment`, and `list_collections`. + +> On a **full** install with a ready collection this call returns real hits instead — skip to Test 32. + +--- + +## Test 32: grep — Works without vectors + +**Action:** On a **full** install with at least one indexed document, call `grep` with `pattern`: a literal string you know is in the corpus (e.g. a step phrase or article code), `ignore_case`: `true`. + +**Expected result:** Matching lines with `doc_id`, `name`, `line_number`, `line_text`, plus `total_found` and a `truncated` flag. + +**Verify:** +- Matches are returned the instant a document is ingested — `grep` does not wait for embedding. +- An invalid regex returns a structured tool error (not a crashed session). + +--- + +## Test 33: search — dense (semantic) + +**Action:** Call `search` with `mode`: `"dense"`, a natural-language `query` describing intent (not an exact code), and optionally a `collection` from `list_collections`. + +**Expected result:** Ranked hits with `doc_id`, `name`, `collection`, `score`, segment `line_start`/`line_end` (when available), and text/preview. + +**Verify:** +- The semantically closest segment ranks at or near the top, even when no exact words match. +- Omitting `collection` searches all collections; passing a comma-separated list scopes to that subset. +- If the collection is still `building`, results come back flagged as partial/incomplete. + +--- + +## Test 34: search — hybrid (exact identifier + intent) + +**Action:** Call `search` with `mode`: `"hybrid"` and a `query` containing an exact identifier (SKU / article / INN / a precise step token). + +**Expected result:** The record carrying that exact identifier ranks at the top — the keyword channel rescues what pure dense similarity would miss. + +**Verify:** +- The exact-identifier hit outranks merely semantically-similar records. +- `hybrid` is the recommended mode for products/clients/QA-step lookups. + +--- + +## Test 35: get_segment — Verbatim line range + +**Action:** Take a `doc_id` + `line_start`/`line_end` from a Test 33 hit and call `get_segment` with them. + +**Expected result:** The exact stored text for that line range, returned verbatim (1-based, inclusive line numbering). + +**Verify:** +- The returned text matches the source lines exactly. +- An out-of-range request is **clamped** and the response reports the actual range returned (no error). + +--- + +## Test 36: search — Metadata filters + +**Action:** Call `search` with a meta filter — e.g. `all`: `{ "type": "scenario" }` (AND) or `any`: `{ "category": ["laptops", "phones"] }` (OR). + +**Expected result:** Only hits whose `meta` satisfies the filter are returned. + +**Verify:** +- `all` requires every clause to match; `any` requires at least one. +- Filters compose with `collection` scoping and with all three `mode`s. + +--- + +## Test 37: list_collections — Collection registry via MCP + +**Action:** On a **full** install with at least one indexed collection, call the `list_collections` tool (no arguments). + +**Expected result:** A `collections` array, one entry per collection, each with `name`, `description`, `n_docs`, `n_segments`, `vector_status` (`empty`/`building`/`ready`/`error`), and `text_ready`. + +**Verify:** +- Every collection indexed from the 1C side is present, sorted by name. +- `description` carries the human/AI-facing text supplied at ingest (empty if none was given). +- On a **lite** install the call returns `rag_not_installed`, like the other three tools. + +--- + +## Test 38: Session Resurrection — Server restart does not strand clients + +**Action:** +1. Initialize a session and save `$SESSION_ID` (as in Test 10). +2. In 1C, press **Disconnect**, then **Connect** again (server restart — all in-memory sessions are gone). +3. Re-send a request with the *old* session ID: +```bash +curl -s -w "\nHTTP:%{http_code}" -X POST http://localhost:8888/mcp \ + -H "Content-Type: application/json" \ + -H "Mcp-Session-Id: $SESSION_ID" \ + -d '{"jsonrpc":"2.0","id":1,"method":"ping"}' +``` + +**Expected:** HTTP 200 with a normal `ping` result — the old session is transparently recreated under the same ID; no 404, no re-initialize required. + +**Verify:** +- The request succeeds immediately after the restart. +- With logging enabled, the log contains `MCP: unknown session resurrected (server restart?)`. +- An MCP client (VS Code) that stayed configured across the restart keeps working without reconnecting. + +--- + +## Test 39: index_raw — collection_description visible in the registry + +**Action:** From the 1C side, ingest a raw document with a collection description via `RagDispatch`: `index_raw` with `{"collection":"desc_test","collection_description":"Throwaway docs for testing","name":"doc.txt","text":"line one\nline two"}` (the Vanessa plugin's file-collections table does exactly this per file). Then call the `list_collections` MCP tool. + +**Expected result:** The `desc_test` entry carries `"description": "Throwaway docs for testing"`. + +**Verify:** +- The same description is visible in the 1C-side `stats` payload. +- Same last-non-empty-wins semantics as `index_segments`: re-ingesting with an empty description keeps the old text; a new non-empty one replaces it. + +--- + +## Test 40: search — Hits echo the effective segment meta + +**Action:** Index (1C side) a document whose doc-level `meta` is `{"src":"doc","lang":"en"}` and one of whose segments carries `meta: {"lang":"ru"}`. Then `search` for that segment. + +**Expected result:** The hit's `meta` is the **effective** view — doc meta overlaid by segment meta, segment wins on collision: `{"src":"doc","lang":"ru"}`. + +**Verify:** +- Doc-only keys (`src`) are preserved; colliding keys take the segment value (`lang: ru`). +- This is the same view the `filter` clauses match against — `all: {"lang":"ru"}` matches the hit, and the returned `meta` shows why. + +--- + +## Vanessa Plugin Tests (MCPRagSearch) + +These tests target the **MCPRagSearch** plugin (see +[`vanessa-plugin/README.md`](vanessa-plugin/README.md)) rather than the demo +processor. Prerequisites: Vanessa Automation is open, the plugin form is +started in **MCP+RAG** mode (default port **8765**), and the indexing status +shows `Готово`. Adjust the URL/port in the curl examples accordingly. + +--- + +## Test 41: Prompts — search-guide and the get_search_guide tool + +**Action:** +1. Send `prompts/list` (curl, as in Test 13, against port 8765). +2. Send `prompts/get` with `{"name":"search-guide"}`. +3. Call the `get_search_guide` tool via the MCP client. + +**Expected:** +- `prompts/list` returns one prompt: `search-guide` (title `Vanessa: как искать шаги (RAG)`), no arguments. +- `prompts/get` returns one `user` message whose text is the search skill (collections, search modes, workflow) from the `СкилПоиска` template. +- The `get_search_guide` tool returns the **same** skill text — it duplicates the prompt for clients that hide MCP prompts. + +**Verify:** +- The prompt text and the tool text are identical. +- `prompts/get` also accepts the legacy name `search_guide`. + +--- + +## Test 42: File Collections — Indexing from the form table + +**Action:** Add a row to the «Коллекции файлов» table (e.g. collection `project_features`, a description, a folder with `.feature` files), press «Переиндексировать», wait for `Готово`. Then, via MCP: `list_collections`, `search` scoped to the collection, `grep` for a known literal, and `get_segment` with a `doc_id` (file path) + line range taken from a hit. + +**Expected:** +- `list_collections` shows the collection with the table row's description and non-zero counts. +- `search`/`grep` hits carry `doc_id` = full file path and `meta.path`/`meta.ext`. +- `get_segment` returns the exact lines of the source file. + +**Verify:** Files larger than 5 MB are skipped with a message; table rows without a collection name or folder are skipped. + +--- + +## Test 43: vanessa_kb — Knowledge base collection + +**Action:** After indexing, call `list_collections`, then `search` with `{"query":"как провести документ","collection":"vanessa_kb","mode":"dense"}`. + +**Expected:** +- `vanessa_kb` is present in the registry (one document; one segment per Q&A entry from the VA `БазаЗнанийДляMCP` template). +- Hits are question+answer segments; `meta.type` is `qa` and `meta.num` is the question number. + +**Verify:** The top hit's text starts with `Вопрос:` and contains a relevant answer. + +--- + +## Test 44: Watcher — Automatic reindex on language / step library change + +**Action:** With the plugin running in MCP+RAG mode, change the Gherkin generator language in VA settings (or load additional step libraries so the step count changes). Wait up to 30 seconds. + +**Expected:** The plugin notices the change (the watcher polls every 30 s), reports it («Язык генератора Gherkin изменился…» / «Библиотека шагов изменилась…») and reindexes `vanessa_steps` automatically. + +**Verify:** +- After reindexing completes, `vanessa_steps` hits carry the new `meta.lang`. +- The watcher only runs while the transport is started in MCP+RAG mode. + +--- + ## Summary Checklist After completing all tests, verify: @@ -525,10 +807,25 @@ After completing all tests, verify: - [ ] Server remains stable after all tests - [ ] Cross-validation: catalog count from `evaluate` matches resource data - [ ] Origin validation blocks external domains (HTTP 403) -- [ ] Invalid session IDs are rejected (HTTP 404) +- [ ] Unknown session IDs are transparently resurrected (HTTP 200, no 404) - [ ] Invalid JSON returns parse error (-32700) - [ ] Unknown methods return method not found (-32601) - [ ] DELETE terminates sessions +- [ ] `search` / `grep` / `get_segment` / `list_collections` appear in `tools/list` (both distributions) +- [ ] Lite distribution returns `rag_not_installed` (not a crash) for all four +- [ ] `grep` matches work before embedding finishes; bad regex is a structured error +- [ ] `search mode=dense` ranks the semantically closest segment near the top +- [ ] `search mode=hybrid` surfaces the exact-identifier record (SKU/INN/article) +- [ ] `get_segment` returns the exact line range and clamps out-of-range requests +- [ ] `search` meta filters (`all`/`any`) and `collection` scoping work +- [ ] `list_collections` returns the registry with names, descriptions, counts and statuses +- [ ] An old session ID still works after a server restart (resurrection) +- [ ] `index_raw` `collection_description` shows up in `list_collections` / `stats` +- [ ] Search hits echo the effective meta (doc overlaid by segment, segment wins) +- [ ] Plugin: `search-guide` prompt and `get_search_guide` tool return the same skill text +- [ ] Plugin: file collections are indexed and searchable (`search`/`grep`/`get_segment`) +- [ ] Plugin: `vanessa_kb` is present with Q&A segments (`meta.type=qa`, `meta.num`) +- [ ] Plugin: watcher auto-reindexes on Gherkin language / step library change ## Troubleshooting @@ -540,3 +837,6 @@ After completing all tests, verify: | `isError: true` on execute | BSL syntax error | Check the 1C/BSL code syntax | | Empty resource data | Empty infobase | Use an infobase with actual catalogs/documents | | Prompt returns no metadata | Server-side error | Call `getStatus` to check the component state | +| `search`/`grep` return `rag_not_installed` | Lite distribution (no `rcore.dll`) | Install the full RAG package (`rcore.dll` + `DirectML.dll` beside the component) | +| `search` returns empty on full install | No content indexed yet, or collection still `building` | Index via the form's Sync button; poll `list_collections` until `vector_status: ready` | +| `search mode=dense` misses exact codes | Embeddings are weak on identifiers | Use `mode: hybrid` for SKU/INN/article/step lookups | diff --git a/build/_build-dll-vcvars.bat b/build/_build-dll-vcvars.bat new file mode 100644 index 0000000..4686363 --- /dev/null +++ b/build/_build-dll-vcvars.bat @@ -0,0 +1,25 @@ +@echo off +REM One-shot DLL build inside a clean vcvars x64 context (the .sh wrapper mangles +REM MSVC PATH under MSYS bash). Configures Ninja + Release and builds. +REM Paths derive from this script's location; override via the VCVARSALL / CMAKE / +REM NINJA environment variables when the defaults don't match the machine. +setlocal + +set "REPO_ROOT=%~dp0.." + +if not defined VCVARSALL set "VCVARSALL=C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" +call "%VCVARSALL%" x64 || exit /b 1 + +if not defined CMAKE set "CMAKE=%REPO_ROOT%\build\tools\cmake\bin\cmake.exe" +if not defined NINJA set "NINJA=%REPO_ROOT%\build\tools\ninja.exe" + +cd /d "%REPO_ROOT%\http-1c-dll" || exit /b 1 +if exist build rmdir /s /q build +mkdir build +cd build || exit /b 1 + +"%CMAKE%" .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_MAKE_PROGRAM="%NINJA%" || exit /b 2 + +"%CMAKE%" --build . || exit /b 3 + +echo BUILD_OK diff --git a/build/package-http1c-addin.sh b/build/package-http1c-addin.sh index 57431fc..691b89c 100644 --- a/build/package-http1c-addin.sh +++ b/build/package-http1c-addin.sh @@ -1,15 +1,64 @@ #!/usr/bin/env bash # package-http1c-addin.sh -# Packages the built DLL into a 1C add-in ZIP bundle and copies it as template. +# Packages the built DLL(s) into a 1C add-in ZIP bundle and copies it as template. +# +# Two variants (select via the VARIANT env var, default "lite"): +# * lite — libhttp1cWin.dll alone. The search tools (search/grep/get_segment/ +# list_collections) return a structured "install RAG package" error +# at runtime. +# * full — libhttp1cWin.dll + rcore.dll (the real fastembed search core) + +# DirectML.dll. rcore.dll hard-imports DirectML.dll (ort bundles the +# DirectML execution provider), so the full bundle MUST ship it or +# rcore.dll fails to load and the component silently degrades to lite. +# +# Usage: +# VARIANT=lite bash build/package-http1c-addin.sh # default +# VARIANT=full bash build/package-http1c-addin.sh +# +# Optional positional overrides (kept for back-compat): +# $1 — primary DLL path (default: http-1c-dll/bin/libhttp1cWin.dll) +# $2 — output ZIP path (default: build/artifacts/http1c-addin[-VARIANT].zip) +# $3 — Template.bin path (default: the embedded 1C template) +# +# Set SKIP_TEMPLATE=1 to produce only the ZIP and NOT overwrite the embedded 1C +# Template.bin. The committed template tracks the LITE bundle (the Rust-free +# default), so the release workflow uses SKIP_TEMPLATE=1 when packaging full to +# avoid clobbering it with the heavier full bundle. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -DLL_PATH="${1:-$REPO_ROOT/http-1c-dll/bin/libhttp1cWin.dll}" -PACKAGE_PATH="${2:-$REPO_ROOT/build/artifacts/http1c-addin.zip}" +# ---- Variant selection ----------------------------------------------------- +VARIANT="${VARIANT:-lite}" +case "$VARIANT" in + lite|full) ;; + *) + echo "Unknown VARIANT '$VARIANT' (expected 'lite' or 'full')" + exit 1 + ;; +esac + +BIN_DIR="$REPO_ROOT/http-1c-dll/bin" +DLL_PATH="${1:-$BIN_DIR/libhttp1cWin.dll}" + +# Default output ZIP name carries the variant so lite/full don't clobber each +# other. The historic plain "http1c-addin.zip" name is still used for "lite" so +# existing callers/scripts keep working unchanged. +if [[ "$VARIANT" == "lite" ]]; then + DEFAULT_PACKAGE="$REPO_ROOT/build/artifacts/http1c-addin.zip" +else + DEFAULT_PACKAGE="$REPO_ROOT/build/artifacts/http1c-addin-$VARIANT.zip" +fi +PACKAGE_PATH="${2:-$DEFAULT_PACKAGE}" TEMPLATE_PATH="${3:-$REPO_ROOT/http-1c-dp/http1c/Templates/http1c/Ext/Template.bin}" +# Source for DirectML.dll on the GitHub windows-latest runner / dev machines. +# Version-coupling caveat: this DirectML.dll is paired with the onnxruntime that +# ort bundled into rcore.dll. A future CPU-only onnxruntime build would drop the +# DirectML import entirely and this copy could be removed. +DIRECTML_SRC="${DIRECTML_SRC:-/c/Windows/System32/DirectML.dll}" + if [[ ! -f "$DLL_PATH" ]]; then echo "DLL not found: $DLL_PATH" exit 1 @@ -32,16 +81,59 @@ mkdir -p "$TEMPLATE_DIR" STAGE_DIR=$(mktemp -d) trap 'rm -rf "$STAGE_DIR"' EXIT -cat > "$STAGE_DIR/MANIFEST.XML" < - - - -EOF +# ---- Assemble the list of native DLLs that go into the bundle -------------- +# Every entry is "|". The component DLL is +# always first; the full variant appends rcore.dll + DirectML.dll. +PAYLOAD=("$DLL_PATH|$DLL_NAME") + +if [[ "$VARIANT" == "full" ]]; then + RCORE_SRC="$BIN_DIR/rcore.dll" + if [[ ! -f "$RCORE_SRC" ]]; then + echo "FULL variant requested but rcore.dll not found: $RCORE_SRC" + echo "Build it first: cmake .. -DRCORE_FASTEMBED=ON (see README / CMakeLists.txt)" + exit 1 + fi + PAYLOAD+=("$RCORE_SRC|rcore.dll") -cp "$DLL_PATH" "$STAGE_DIR/$DLL_NAME" + if [[ ! -f "$DIRECTML_SRC" ]]; then + echo "FULL variant requested but DirectML.dll not found: $DIRECTML_SRC" + echo "rcore.dll hard-imports DirectML.dll (ort's DirectML execution provider);" + echo "without it rcore.dll fails to load and the component degrades to lite." + echo "Set DIRECTML_SRC to its location, or install the DirectML runtime." + exit 1 + fi + PAYLOAD+=("$DIRECTML_SRC|DirectML.dll") +fi + +# ---- Build MANIFEST.XML listing every native DLL --------------------------- +# Only the component itself is declared as a ; the extra runtime DLLs +# (rcore.dll, DirectML.dll) ship alongside it in the same bundle and are loaded +# by libhttp1cWin.dll at runtime (LoadLibrary), so they are entries. +{ + echo '' + echo '' + echo " " + for entry in "${PAYLOAD[@]}"; do + name="${entry#*|}" + [[ "$name" == "$DLL_NAME" ]] && continue + echo " " + done + echo '' +} > "$STAGE_DIR/MANIFEST.XML" + +# ---- Stage every DLL ------------------------------------------------------- +for entry in "${PAYLOAD[@]}"; do + src="${entry%|*}" + name="${entry#*|}" + cp "$src" "$STAGE_DIR/$name" +done -rm -f "$PACKAGE_PATH" "$TEMPLATE_PATH" +rm -f "$PACKAGE_PATH" +# Only clear the embedded template when we're actually going to rewrite it; +# SKIP_TEMPLATE=1 must leave the existing Template.bin untouched. +if [[ "${SKIP_TEMPLATE:-0}" != "1" ]]; then + rm -f "$TEMPLATE_PATH" +fi # Create ZIP using available tool if command -v zip &>/dev/null; then @@ -56,8 +148,17 @@ else echo "No zip tool found (zip, 7z, or powershell required)" exit 1 fi -cp "$PACKAGE_PATH" "$TEMPLATE_PATH" +if [[ "${SKIP_TEMPLATE:-0}" == "1" ]]; then + TEMPLATE_NOTE="skipped (SKIP_TEMPLATE=1)" +else + cp "$PACKAGE_PATH" "$TEMPLATE_PATH" + TEMPLATE_NOTE="$TEMPLATE_PATH" +fi -echo "Packaged add-in bundle version $VERSION" +echo "Packaged add-in bundle ($VARIANT) version $VERSION" +echo "Contents:" +for entry in "${PAYLOAD[@]}"; do + echo " - ${entry#*|}" +done echo "Archive: $PACKAGE_PATH" -echo "Template updated: $TEMPLATE_PATH" +echo "Template: $TEMPLATE_NOTE" diff --git a/build/run-tests.sh b/build/run-tests.sh index 53b2acf..d4ede8c 100644 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -88,3 +88,23 @@ echo "" echo "Running tests..." echo "========================================" ./http1c_tests.exe + +# ---- Black-box smoke harness (loads bin/libhttp1cWin.dll like 1C does) ---- +# Builds ONLY the http1c_smoke target so a libhttp1cWin.dll locked by a running +# 1C session does not break the run; the exe is tested against the existing +# bin/libhttp1cWin.dll. A separate build dir (build-smoke) keeps this from +# clobbering a developer's Ninja-configured http-1c-dll/build. +DLL_DIR="$REPO_ROOT/http-1c-dll" + +cd "$DLL_DIR" +rm -rf build-smoke +mkdir build-smoke +cd build-smoke + +"$CMAKE" .. -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DHTTP1C_BUILD_TESTS=ON +"$CMAKE" --build . --target http1c_smoke + +echo "" +echo "Running smoke harness..." +echo "========================================" +"$DLL_DIR/bin/http1c_smoke.exe" diff --git a/http-1c-dll/CMakeLists.txt b/http-1c-dll/CMakeLists.txt index e444729..6826dc0 100644 --- a/http-1c-dll/CMakeLists.txt +++ b/http-1c-dll/CMakeLists.txt @@ -73,3 +73,91 @@ else(UNIX) target_compile_options(${TARGET} PRIVATE /utf-8) target_link_libraries(${TARGET} PRIVATE ws2_32 gdiplus ole32 dwmapi) endif(UNIX) + +# =========================================================================== +# Rust search core (rust-core/, crate `rcore`) — runtime-loaded rcore.dll. +# +# The component does NOT link the Rust core. It loads rcore.dll at RUNTIME via +# LoadLibrary + GetProcAddress (see src/RustCore.h), so the lite component is +# PURE C++ (/MT) with ZERO cargo/Rust dependency: building libhttp1cWin.dll +# never invokes cargo. RustCore.h only needs to be on the include path. +# +# The header is in src/, so make it resolvable regardless of package flavour. +target_include_directories(${TARGET} PRIVATE src) + +# --------------------------------------------------------------------------- +# Optional "full" package: also build rcore.dll (the real fastembed search core) +# and drop it next to the component. ort/onnxruntime requires the DYNAMIC CRT +# (/MD), so the cdylib is built with -crt-static REMOVED (overriding the +# +crt-static default in rust-core/.cargo/config.toml) — this is independent of +# the component's /MT and they never link together. DirectML.dll bundling is a +# separate, later step. +# +# OFF by default → the default build is the Rust-free lite component. +# --------------------------------------------------------------------------- +option(RCORE_FASTEMBED "Also build rcore.dll (real fastembed search core) for the full package" OFF) + +# =========================================================================== +# Black-box smoke harness (tests/smoke) — a console exe that LoadLibrary's the +# BUILT bin/libhttp1cWin.dll exactly like 1C does (GetClassObject + stubbed 1C +# interfaces) and drives it over real HTTP. It does NOT link the DLL, so the +# CRT choice is free; we keep the DLL's flags (/MT via add_definitions above) +# for simplicity. OFF by default; CI enables it with -DHTTP1C_BUILD_TESTS=ON. +# =========================================================================== +option(HTTP1C_BUILD_TESTS "Build the http1c_smoke black-box test exe" OFF) + +if (HTTP1C_BUILD_TESTS AND NOT UNIX) + add_executable(http1c_smoke tests/smoke/main.cpp) + target_compile_definitions(http1c_smoke PRIVATE UNICODE _UNICODE + _WIN32_WINNT=0x0601 NOMINMAX WIN32_LEAN_AND_MEAN _WINDOWS + _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING) + target_include_directories(http1c_smoke PRIVATE include) + target_compile_options(http1c_smoke PRIVATE /utf-8) + target_link_libraries(http1c_smoke PRIVATE ws2_32) + set_target_properties(http1c_smoke PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + RUNTIME_OUTPUT_DIRECTORY "${HTTP1C_OUTPUT_DIR}" + PDB_OUTPUT_DIRECTORY "${HTTP1C_OUTPUT_DIR}") +endif() + +if (RCORE_FASTEMBED AND NOT UNIX) + set(RCORE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../rust-core") + set(RCORE_TARGET_TRIPLE "x86_64-pc-windows-msvc") + + # cargo always emits a cdylib under the release profile dir for this target. + set(RCORE_DLL_SRC + "${RCORE_DIR}/target/${RCORE_TARGET_TRIPLE}/release/rcore.dll") + set(RCORE_DLL_OUT "${HTTP1C_OUTPUT_DIR}/rcore.dll") + + # Rust sources that should trigger a rebuild. CONFIGURE_DEPENDS re-globs at + # build time so editing a .rs file rebuilds the cdylib (cargo itself is + # incremental, but CMake needs the dependency to know it must invoke cargo). + # Cargo.lock (dependency bumps) and .cargo/config.toml (CRT/link flags) + # change the produced DLL just as much as source edits do. + file(GLOB_RECURSE RCORE_SOURCES CONFIGURE_DEPENDS + "${RCORE_DIR}/src/*.rs") + list(APPEND RCORE_SOURCES + "${RCORE_DIR}/Cargo.toml" + "${RCORE_DIR}/Cargo.lock" + "${RCORE_DIR}/.cargo/config.toml") + + # Build the cdylib with the dynamic CRT (ort needs /MD), then copy rcore.dll + # into the component output dir (bin/). The copied DLL is the OUTPUT. + add_custom_command( + OUTPUT "${RCORE_DLL_OUT}" + COMMAND ${CMAKE_COMMAND} -E env "RUSTFLAGS=-C target-feature=-crt-static" + cargo build --release --features fastembed + --target ${RCORE_TARGET_TRIPLE} + --manifest-path "${RCORE_DIR}/Cargo.toml" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${RCORE_DLL_SRC}" "${RCORE_DLL_OUT}" + WORKING_DIRECTORY "${RCORE_DIR}" + DEPENDS ${RCORE_SOURCES} + COMMENT "Building Rust core (rcore.dll, fastembed) via cargo and copying to bin/" + VERBATIM + USES_TERMINAL) + + add_custom_target(rcore_dll DEPENDS "${RCORE_DLL_OUT}") + add_dependencies(${TARGET} rcore_dll) +endif() diff --git a/http-1c-dll/THREAD-SAFETY.md b/http-1c-dll/THREAD-SAFETY.md new file mode 100644 index 0000000..9fcc41e --- /dev/null +++ b/http-1c-dll/THREAD-SAFETY.md @@ -0,0 +1,265 @@ +# C++ Transport Thread-Safety Audit — `HttpServerComponent` + +**Card:** `stage0-cpp-thread-safety-review-2026-06-08` +**Scope:** Audit every piece of shared mutable state in `http-1c-dll/src/HttpServerComponent.{cpp,h}` +for correct synchronization now that a **synchronous native search path** (Rust core via +`rcore_dispatch`) runs on httplib worker threads alongside the existing 1C-forwarded +(`ExtEvent` + `pendingRequests`) path and the 1C-thread native AddFunction/AddProperty methods. + +## Where the audited code lives (read this first) + +The native search path (`isNativeTool` / `nativeToolDefinitions` / `dispatchNativeTool`, the +`tools/call` native branch, the `tools/list` merge, `RustCore.h`, the Rust `rust-core/` crate) +exists on branch **`feat/search-core`**, not on the `master` base this worktree was cut from. The +audit was performed against the `feat/search-core` versions of: + +- `http-1c-dll/src/HttpServerComponent.cpp` (1599 lines on that branch) +- `http-1c-dll/src/HttpServerComponent.h` (unchanged from `master`) +- `http-1c-dll/src/RustCore.h` +- `rust-core/src/lib.rs`, `rust-core/src/core.rs` + +All `HttpServerComponent.cpp:NNN` line citations below are **line numbers in the +`feat/search-core` version** of that file. Logic shared with `master` (sessions, caches, +pending map, SSE, logging, auth, rate limiter) is identical on both branches; only the native +additions differ. + +## Threads involved + +1. **httplib worker threads** — `cpp-httplib` serves `/mcp`, `/health`, legacy routes on a + thread pool, so multiple MCP requests (and the native search path) run **concurrently**. +2. **1C main thread** — calls the native AddFunction/AddProcedure/AddProperty entry points: + `SendResponse`, `SendProgress`, `RegisterTools/Resources/Prompts`, `SetAuthToken`, + `LoggingEnabled`, `LogPath`, `StartListen`, `StopListen`, `Status`, `Timeout`. +3. **The httplib listener thread** (`serverThread`) — runs `server->listen(...)`. + +The new native search path executes **entirely on a httplib worker thread** (inside +`handleMcpRequest`). It does not spawn threads and does not touch the 1C thread. + +--- + +## Verdict per shared-state item + +| Shared state | Mutex / guard | Threads | Verdict | +|---|---|---|---| +| `sessions` | `sessionMutex` | workers + 1C (`Status`) | **Safe** | +| `cachedToolsJson` | `toolsMutex` | workers + 1C (`RegisterTools`, `Status`) | **Safe** | +| `cachedResourcesJson` | `resourcesMutex` | workers + 1C | **Safe** | +| `cachedPromptsJson` | `promptsMutex` | workers + 1C | **Safe** | +| `pendingRequests` + `requestCounter` | `pendingMutex` | workers + 1C (`SendResponse`/`SendProgress`) | **Safe** (native path does not touch it) | +| `PendingRequest` internals | `PendingRequest::mtx` + `atomic ready` | producer (1C) + consumer (worker) | **Safe** | +| `sseStreams` | `sseStreamsMutex` (+ per-stream `mtx`) | workers + 1C (broadcast) | **Safe** | +| Logging (`g_logPath`, `g_loggingEnabled`) | `g_loggingMutex` | all | **Safe** | +| Instance logging (`logPath`, `loggingEnabled`) | `loggingMutex` (+ see note) | workers + 1C | **Safe** with one benign data race on `loggingEnabled` (pre-existing) | +| `authToken` | `authTokenMutex` (readers snapshot via `authTokenCopy()`) | workers (read) + 1C (write via `ApplyConfig`/`SetAuthToken`) | **Safe** (was a pre-existing race; fixed during PR review) | +| `rateLimiter` | internal `RateLimiter::mtx` | workers | **Safe** | +| `running` / `timeout` | `std::atomic` / `std::atomic` | workers + 1C | **Safe** (`timeout` was a benign race; made atomic during PR review) | +| `listenPort` | plain int | workers + 1C | **Benign** (written once in `doStartListen` before workers exist) | +| Native helpers (`isNativeTool`, `nativeToolDefinitions`, `dispatchNativeTool`, `metaFiltersSchema`) | — (stateless) | workers | **Safe** (reentrant, no shared state) | +| Rust core singleton (`CORE`) | `RwLock` **inside Rust** | workers (search) + 1C (ingest/configure, once wired) | **Safe** (locking owned by Rust) | + +**Bottom line: the native search path introduces no new C++ data races.** It is a stateless, +synchronous pass-through to a Rust core that does its own locking. The audit originally flagged +four **pre-existing, benign** unsynchronized reads of config strings/ints; two of them +(`authToken`, `timeout`) were subsequently fixed during the PR review (`authTokenMutex` + +`std::atomic`), leaving only `loggingEnabled` and `listenPort` as benign, effectively +write-once values. Details and reasoning follow. + +--- + +## Detailed analysis + +### 1. The native `tools/call` branch is correctly isolated + +`HttpServerComponent.cpp:1071-1094`. When `isNativeTool(toolName)` is true the handler: + +- builds the result via `dispatchNativeTool(toolName, toolArgs)` wrapped in `try/catch (...)`, +- assembles a local `rpcResp`, calls `res.set_content(...)`, and **`return`s at line 1093**. + +The `pendingRequests` / `requestCounter` / `pendingMutex` machinery only begins at +`HttpServerComponent.cpp:1096` (`std::string reqId; auto pending = std::make_shared<...>();` then +`std::lock_guard lock(pendingMutex); reqId = "req_" + std::to_string(++requestCounter);`). +Because the native branch returns before reaching that code, **the native path never inserts into +`pendingRequests`, never increments `requestCounter`, and never acquires `pendingMutex`.** +Verified: there is no path where a native tool call and `SendResponse`/`SendProgress` from 1C can +collide over the same `reqId` (no `reqId` is ever created for a native call). **Verdict: safe.** + +### 2. Native helpers are stateless / reentrant + +All four helpers live in the file-local anonymous namespace (opens +`HttpServerComponent.cpp:24`, closes line 420), so they have internal linkage and are plain free +functions: + +- `isNativeTool` (`:189-193`) — four string compares on its parameter; pure. +- `metaFiltersSchema` (`:197-217`) — constructs and returns a fresh `json` from literals; pure. +- `nativeToolDefinitions` (`:221-351`) — builds four fresh `json` objects from literals each call; + no statics, no globals; pure. +- `dispatchNativeTool` (`:359-402`) — operates only on its parameters and stack locals + (`argsJson`, `raw`, `envelopeStr`, `envelope`, `text`/`errText`); no shared state touched. + +None reads or writes any member of `HttpServerComponent` or any global. Concurrent invocation from +multiple worker threads is therefore data-race-free on the C++ side. **Verdict: safe / reentrant.** + +### 3. `cachedToolsJson` read under `toolsMutex`; merge copies, holds no references + +`HttpServerComponent.cpp:1021-1035`: + +```cpp +json tools; +{ + std::lock_guard lock(toolsMutex); + tools = json::parse(cachedToolsJson, nullptr, false); // deep copy out of shared string + if (tools.is_discarded()) tools = json::array(); +} +if (!tools.is_array()) tools = json::array(); +for (auto& def : nativeToolDefinitions()) { // lock NOT held here + tools.push_back(std::move(def)); +} +``` + +- `cachedToolsJson` is read only while holding `toolsMutex`. `json::parse` produces an + **independent value** (`tools`); the lock is released before the merge. +- The merge appends into the local `tools` copy. It does **not** retain a pointer/reference into + `cachedToolsJson` or any shared object — `nativeToolDefinitions()` returns a temporary built from + literals, and `std::move(def)` moves out of that temporary's elements (the temporary outlives the + range-`for`, so this is well-defined and merely an optimization). +- The writer side, `doRegisterTools` (`HttpServerComponent.cpp` `:cachedToolsJson = utf8;` under + `toolsMutex`), is fully serialized against this read. **Verdict: safe.** + +`cachedResourcesJson` / `cachedPromptsJson` follow the identical, correct pattern under +`resourcesMutex` / `promptsMutex`. The `Status` property also reads all three under their +respective mutexes. **Verdict: safe.** + +### 4. `rcore_dispatch` is called with NO C++ lock held; `RustString` lifetime is correct + +`HttpServerComponent.cpp:367`: + +```cpp +RustString raw = RustString::adopt(rcore_dispatch(toolName.c_str(), argsJson.c_str())); +std::string envelopeStr = raw.str(); // copy out +``` + +- `dispatchNativeTool` acquires **no** `HttpServerComponent` mutex anywhere; `toolsMutex` is not + taken in the `tools/call` branch at all. So `rcore_dispatch` is called outside every C++ lock. + This is the desired property: a slow search cannot stall `tools/list`, `Status`, or any other + worker waiting on `toolsMutex`/`pendingMutex`. +- **Lifetime / ownership** (contract in `RustCore.h` and `rust-core/src/lib.rs`): `rcore_dispatch` + returns a Rust-allocated `char*` that must be freed exactly once via `rcore_free_string`. + `RustString::adopt` takes sole ownership; `RustString` is move-only (copy ctor/assign `= delete`), + so no accidental double-adopt. `.str()` copies the bytes into `envelopeStr`; the original buffer + is freed by `~RustString` (calling `rcore_free_string`) when `raw` leaves scope at the end of + `dispatchNativeTool`. Freed **exactly once**, on the **Rust allocator** (no cross-CRT free). + `rcore_dispatch` is documented to never return null and never panic across the boundary + (`rcore_free_string(nullptr)` is a safe no-op regardless). **Verdict: safe.** + +### 5. The Rust core is the synchronization point for ingest-vs-search + +`rust-core/src/core.rs:268`: `pub static CORE: Lazy> = ...`. Every FFI method +(`search`, `grep`, `get_segment`, `stats`, `configure`, `index_segments`, `reset`) takes +`CORE.read()` or `CORE.write()` inside `rcore_dispatch`. Concurrent searches take read locks; +configure/ingest take write locks; the `RwLock` serializes them. Each `extern "C"` entry is wrapped +in `catch_unwind` (`guard(...)`), so a Rust panic becomes a structured error envelope rather than UB +across the ABI. Therefore the C++ side genuinely is a **thin pass-through**: the +search-vs-ingest concurrency question is answered entirely inside Rust, and the C++ glue neither +needs nor adds a lock for it. **Verdict: safe** (correctness of the Rust locking itself is out of +scope for this C++ audit, but the boundary contract — single-writer/multi-reader, panic-guarded, +single-free — is upheld on the C++ side). + +### 6. `pendingRequests` + `requestCounter` (existing forwarded path) + +`pendingMutex` guards both the map and the counter at every site: legacy handler +(`reqId`/insert/erase), `tools/call` non-native branch (`:1099`), `resources/read`, `prompts/get`, +`doSendResponse`, `doSendProgress`, `doStopListen`, `Status`, `/health`, and the chunked-stream +release callbacks. The shared `PendingRequest` object retrieved under `pendingMutex` is then handed +off; its own fields are guarded by `PendingRequest::mtx` + the `std::atomic ready`, with the +producer (1C thread via `SendResponse`/`SendProgress`) and the consumer (worker in `wait_for`) +synchronizing through the condition variable. This is correct and unchanged by the native work. +**Verdict: safe.** + +### 7. Sessions, SSE streams, logging, rate limiter + +- **`sessions` / `sessionMutex`** — `createSession`, `findSession`, `handleMcpDelete`, + `doStopListen`, `Status` all lock `sessionMutex` around every access. `findSession` also mutates + `lastActivity` under the lock. **Safe.** +- **`sseStreams` / `sseStreamsMutex`** — registration (`handleMcpGet`), removal (stream-release + callback and `doStopListen`), and `broadcastNotification` all hold `sseStreamsMutex`; per-stream + `messages` are additionally guarded by `SseStream::mtx`, with `closed` atomic. **Safe.** +- **Global logging** — `logToFile`, the `LoggingEnabled`/`LogPath` setters, and `Status` lock + `g_loggingMutex` for `g_logPath`/`g_loggingEnabled`. `logToFile` copies the path under the lock, + then writes the file outside it. **Safe.** +- **`rateLimiter`** — `RateLimiter::allow()` locks its own `mtx` for the whole token computation + (`HttpServerComponent.h:42-50`). Concurrent `checkRateLimit` calls are serialized. **Safe.** + +--- + +## Pre-existing, non-material races (audit findings; status updated) + +These predate the native path. They are unsynchronized reads/writes of small scalar/string config +values. The audit originally left all of them unfixed under the card's "minimal C++ change" +mandate; items 1 and 3 (`timeout`) were **subsequently fixed during the PR review** because +`ApplyConfig` made both genuinely runtime-mutable under live traffic (token rotation / timeout +re-config), which promoted them from "benign" to "must-fix" exactly as item 4 of the latent-risks +section predicted. + +1. **`authToken`** — **FIXED.** Originally read on worker threads in `validateAuth`/`Status` and + written on the 1C thread in `doSetAuthToken` with no mutex (a formal data race on a + `std::string`). Now every access goes through `authTokenMutex`: `doSetAuthToken` assigns under + the lock, and readers take a snapshot via `authTokenCopy()` + (`HttpServerComponent.h`), so `validateAuth` compares against a stable copy even while + `ApplyConfig` rotates the token mid-flight. + +2. **`loggingEnabled`** (instance field). The `LoggingEnabled` getter reads it and the setter + writes it without `loggingMutex` (only the mirrored *global* `g_loggingEnabled` is locked); + `Status` reads it under `loggingMutex` but the property accessor does not. Benign `bool` race — + still present, still non-material. + +3. **`timeout`** — **FIXED:** now `std::atomic` (`HttpServerComponent.h`), read with + `.load()` in `wait_for`/`Status` and stored via `ApplyConfig`/the `Timeout` property. + **`listenPort`** — still a plain int, but it is written once in `doStartListen` before the + listener starts serving; benign. + +(`running` is already `std::atomic`, so it is fine.) + +--- + +## Latent risks for future stages + +1. **Native tools must stay off `pendingRequests` / `SendProgress`.** The current safety argument + for the native branch rests on it returning synchronously at `HttpServerComponent.cpp:1093` + *before* any `pendingMutex`/`requestCounter` use. If a future stage makes native search + long-running and wants streaming progress, do **not** reuse the 1C `PendingRequest` + + `SendProgress` plumbing (that channel is driven by the 1C thread via `reqId`, and native calls + have no `reqId`). Instead either (a) stream directly from the worker thread inside the native + branch using `set_chunked_content_provider` with a Rust-side progress callback, or (b) if the + `PendingRequest` machinery is reused, allocate a `reqId` under `pendingMutex` and ensure the + completion/erase paths cannot race a 1C `SendResponse` for a *different* request. Re-audit at + that point. + +2. **1C-thread ingest/configure vs. concurrent worker search.** Once `configure` / + `index_segments` are wired to 1C AddFunction methods, they will call `rcore_dispatch` from the + **1C thread** while worker threads call `search`/`grep` concurrently. Correctness then depends + **entirely on the Rust `RwLock`** (`rust-core/src/core.rs:268`) — the C++ side adds no + lock and must not, or it would serialize unrelated MCP requests. The risk to watch is on the + Rust side: a `configure` that swaps `dim` and resets the index while a `search` read is in + flight must be handled by the write lock (it is, today). No C++ action needed; noted so a future + reviewer does not "helpfully" add a C++ mutex around `rcore_dispatch` and reintroduce + head-of-line blocking. + +3. **`rcore_shutdown` wiring — DONE.** `RustCore.h` documents that `rcore_shutdown()` should be + hooked onto `doStopListen()` / form-close, **not** `~HttpServerComponent` (the Rust singleton + outlives any one component instance). `doStopListen` now calls `RCore::shutdown()` after the + listener thread is joined and all pending requests are released — exactly the wiring this audit + recommended. Calling it from the destructor instead would still be a latent bug; keep it where + it is. + +4. **Config-string races (above) under runtime reconfiguration — TRIGGERED AND RESOLVED.** + `ApplyConfig` made `authToken` and `timeout` runtime-mutable under live traffic, which promoted + them from "benign" to "must-fix"; both got their mutex/atomic (see the updated verdict table). + The remaining write-once fields (`loggingEnabled`, `listenPort`) keep their benign status until + something makes them hot-reconfigurable too. + +--- + +## Build + +No C++ source was modified by this audit (only this Markdown document was added), so no rebuild was +required (per the card: "If the audit finds nothing to change, no build is needed"). diff --git a/http-1c-dll/src/AddInNative.cpp b/http-1c-dll/src/AddInNative.cpp index c34c3e9..c8638d8 100644 --- a/http-1c-dll/src/AddInNative.cpp +++ b/http-1c-dll/src/AddInNative.cpp @@ -449,10 +449,23 @@ void ADDIN_API AddInNative::FreeMemory(void** pMemory) const std::string AddInNative::WCHAR2MB(std::basic_string_view src) { #ifdef _WINDOWS - static std::wstring_convert> cvt_utf8_utf16; - return cvt_utf8_utf16.to_bytes(src.data(), src.data() + src.size()); + // UTF-16 -> UTF-8 via the Win32 API directly. The previous implementation + // used a process-wide `static std::wstring_convert`, whose mutable internal + // state is NOT thread-safe; httplib's worker pool calls this concurrently on + // every request/response, which raced -> corrupted UTF-8 / sporadic crashes + // under load. A local conversion has no shared state and is safe. + if (src.empty()) return std::string(); + const wchar_t* wsrc = reinterpret_cast(src.data()); + const int wlen = static_cast(src.size()); + const int needed = WideCharToMultiByte(CP_UTF8, 0, wsrc, wlen, nullptr, 0, nullptr, nullptr); + if (needed <= 0) return std::string(); + std::string out(static_cast(needed), '\0'); + WideCharToMultiByte(CP_UTF8, 0, wsrc, wlen, &out[0], needed, nullptr, nullptr); + return out; #else - static std::wstring_convert, char16_t> cvt_utf8_utf16; + // Non-Windows: a LOCAL converter (no `static`) — each call owns its own, so + // there is no shared mutable state to race on. + std::wstring_convert, char16_t> cvt_utf8_utf16; return cvt_utf8_utf16.to_bytes(reinterpret_cast(src.data()), reinterpret_cast(src.data() + src.size())); #endif//_WINDOWS @@ -470,11 +483,20 @@ std::wstring AddInNative::WCHAR2WC(std::basic_string_view src) { std::u16string AddInNative::MB2WCHAR(std::string_view src) { #ifdef _WINDOWS - static std::wstring_convert> cvt_utf8_utf16; - std::wstring tmp = cvt_utf8_utf16.from_bytes(src.data(), src.data() + src.size()); - return std::u16string(reinterpret_cast(tmp.data()), tmp.size()); + // UTF-8 -> UTF-16 via the Win32 API directly: thread-safe (no shared static + // state — see WCHAR2MB). The API fills a wchar_t buffer; the result is then + // widened element-wise into char16_t — writing through a casted wchar_t* + // into the u16string's own buffer would violate strict aliasing. + if (src.empty()) return std::u16string(); + const int slen = static_cast(src.size()); + const int needed = MultiByteToWideChar(CP_UTF8, 0, src.data(), slen, nullptr, 0); + if (needed <= 0) return std::u16string(); + std::wstring wide(static_cast(needed), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, src.data(), slen, &wide[0], needed); + return std::u16string(wide.begin(), wide.end()); #else - static std::wstring_convert, char16_t> cvt_utf8_utf16; + // Non-Windows: a LOCAL converter (no `static`) — thread-safe per call. + std::wstring_convert, char16_t> cvt_utf8_utf16; return cvt_utf8_utf16.from_bytes(src.data(), src.data() + src.size()); #endif//_WINDOWS } diff --git a/http-1c-dll/src/HttpServerComponent.cpp b/http-1c-dll/src/HttpServerComponent.cpp index 130c361..9f15253 100644 --- a/http-1c-dll/src/HttpServerComponent.cpp +++ b/http-1c-dll/src/HttpServerComponent.cpp @@ -1,6 +1,7 @@ #include "stdafx.h" #include "HttpServerComponent.h" #include "ScreenCapture.h" +#include "RustCore.h" #include "../version.h" @@ -177,6 +178,277 @@ json paginateJsonArray(const json& arr, const json& params) { return result; } +// ========================================================================= +// Native search tools — served by the Rust core (rcore.dll, loaded at runtime +// via RustCore.h), NOT forwarded to 1C. The component owns these tool names and +// their JSON Schemas, so the search subsystem is self-contained regardless of +// whether 1C declares them. The tool SURFACE is uniform across lite/full: lite +// advertises the same tools but returns "install RAG" when they are called. +// ========================================================================= + +// True if `toolName` is handled natively by the Rust core (search subsystem). +bool isNativeTool(const std::string& toolName) { + return toolName == "search" || + toolName == "get_segment" || + toolName == "grep" || + toolName == "list_collections"; +} + +// Shared `filter` schema fragment (free key-value, OR via `any` / AND via +// `all`; no domain-specific fields — the core stays generic). The property is +// named `filter` to match what the Rust core parses (MetaFilter::parse reads +// the top-level `filter` object); advertising `meta_filters` would be silently +// ignored. +json metaFiltersSchema() { + return json{ + {"type", "object"}, + {"description", "Optional metadata filters over free key-value pairs. " + "`any` matches if ANY pair matches (OR); `all` requires " + "ALL pairs to match (AND). Combinable."}, + {"properties", { + {"any", { + {"type", "object"}, + {"description", "OR filter: keep results matching any of these key-value pairs."}, + {"additionalProperties", true} + }}, + {"all", { + {"type", "object"}, + {"description", "AND filter: keep results matching all of these key-value pairs."}, + {"additionalProperties", true} + }}, + {"tags_all", { + {"type", "array"}, + {"items", {{"type", "string"}}}, + {"description", "Require EVERY listed tag to be present in the hit's " + "effective `tags` meta array."} + }}, + {"tags_any", { + {"type", "array"}, + {"items", {{"type", "string"}}}, + {"description", "Require AT LEAST ONE of the listed tags to be present " + "in the hit's effective `tags` meta array."} + }} + }}, + {"additionalProperties", false} + }; +} + +// The native tool definitions ({name, description, inputSchema}) merged into +// tools/list. Schemas follow the Stage-1/Stage-2 contracts (§4.1, §5.3, §5.4). +json nativeToolDefinitions() { + json searchTool = { + {"name", "search"}, + {"description", "Semantic/keyword/hybrid search over indexed segments. " + "Returns ranked hits with doc_id, name, score and segment " + "line ranges."}, + {"inputSchema", { + {"type", "object"}, + {"properties", { + {"query", { + {"type", "string"}, + {"description", "Search query text."} + }}, + {"collection", { + {"type", "string"}, + {"description", "Restrict the search to a single collection. " + "When omitted, all collections are searched."} + }}, + {"collections", { + {"type", "array"}, + {"items", {{"type", "string"}}}, + {"description", "Search a subset of collections; takes precedence " + "over `collection`. Absent = all collections."} + }}, + {"k", { + {"type", "integer"}, + {"minimum", 1}, + {"description", "Maximum number of hits to return (default 10)."} + }}, + {"mode", { + {"type", "string"}, + {"enum", json::array({"dense", "keyword", "hybrid"})}, + {"description", "Retrieval mode: dense (vectors), keyword " + "(lexical) or hybrid (RRF fusion)."} + }}, + {"min_score", { + {"type", "number"}, + {"description", "Drop hits scoring below this threshold."} + }}, + {"max_per_doc", { + {"type", "integer"}, + {"minimum", 1}, + {"description", "Cap the number of hits returned per document."} + }}, + {"include_text", { + {"type", "boolean"}, + {"description", "Include the full segment text in each hit " + "(default true). When false a short preview is returned."} + }}, + {"filter", metaFiltersSchema()} + }}, + {"required", json::array({"query"})}, + {"additionalProperties", false} + }} + }; + + json getSegmentTool = { + {"name", "get_segment"}, + {"description", "Return a line range of an indexed document by O(1) offset " + "lookup. Out-of-range requests are clamped and the actual " + "range returned."}, + {"inputSchema", { + {"type", "object"}, + {"properties", { + {"doc_id", { + {"type", "string"}, + {"description", "Identifier of the document to slice."} + }}, + {"line_start", { + {"type", "integer"}, + {"minimum", 1}, + {"description", "First line of the range (1-based, inclusive)."} + }}, + {"line_end", { + {"type", "integer"}, + {"minimum", 1}, + {"description", "Last line of the range (inclusive)."} + }}, + {"max_lines", { + {"type", "integer"}, + {"minimum", 1}, + {"description", "Optional hard cap on the number of lines returned."} + }} + }}, + {"required", json::array({"doc_id", "line_start", "line_end"})}, + {"additionalProperties", false} + }} + }; + + json grepTool = { + {"name", "grep"}, + {"description", "Regex search (RE2 / linear time, no backreferences or " + "lookaround) over indexed document text. Returns matching " + "lines with optional context."}, + {"inputSchema", { + {"type", "object"}, + {"properties", { + {"pattern", { + {"type", "string"}, + {"description", "RE2 regular expression to match against each line."} + }}, + {"collection", { + {"type", "string"}, + {"description", "Restrict the search to a single collection. " + "When omitted, all collections are searched."} + }}, + {"ignore_case", { + {"type", "boolean"}, + {"description", "Case-insensitive matching (default false)."} + }}, + {"multiline", { + {"type", "boolean"}, + {"description", "Multiline mode: ^ and $ match at line boundaries."} + }}, + {"context_lines", { + {"type", "integer"}, + {"minimum", 0}, + {"description", "Number of context lines to include before and " + "after each match."} + }}, + {"max_matches", { + {"type", "integer"}, + {"minimum", 1}, + {"description", "Stop after this many total matches; sets `truncated`."} + }}, + {"max_per_doc", { + {"type", "integer"}, + {"minimum", 1}, + {"description", "Cap the number of matches returned per document."} + }}, + {"filter", metaFiltersSchema()} + }}, + {"required", json::array({"pattern"})}, + {"additionalProperties", false} + }} + }; + + json listCollectionsTool = { + {"name", "list_collections"}, + {"description", "List the searchable collections: name, human description " + "of what each one holds, document/segment counts and state " + "(text_ready; vector_status building/ready). Call this first " + "to choose the `collection` argument for search/grep."}, + {"inputSchema", { + {"type", "object"}, + {"properties", json::object()}, + {"additionalProperties", false} + }} + }; + + return json::array({searchTool, getSegmentTool, grepTool, listCollectionsTool}); +} + +// Call the Rust core for a native tool and turn its envelope into an MCP +// tools/call result: {"content":[{"type":"text","text":}], +// "isError":}. The text carries the inner `result` on success, or the +// `error` object on failure — tool errors are STRUCTURAL results, never a +// transport/session failure. Exception/panic-safe: any parse problem degrades +// to a structured isError result rather than throwing. +// +// rcore.dll is loaded at RUNTIME (see RustCore.h). When it's absent — the "lite" +// component — every native tool returns a structured `rag_not_installed` result +// telling the caller to install the RAG (full) package. +json dispatchNativeTool(const std::string& toolName, const json& arguments) { + if (!RCore::available()) { + return makeTextToolResult( + R"({"code":"rag_not_installed","message":"Semantic search backend (rcore.dll) is not installed — this is the lite component. Install the RAG package to enable search/grep/get_segment/list_collections."})", + true); + } + + std::string argsJson; + try { + argsJson = arguments.is_null() ? std::string("{}") : arguments.dump(); + } catch (...) { + argsJson = "{}"; + } + + RustString raw = RCore::dispatch(toolName, argsJson); + std::string envelopeStr = raw.str(); + + if (envelopeStr.empty()) { + return makeTextToolResult( + R"({"code":"internal","message":"rust core returned an empty response"})", + true); + } + + json envelope = json::parse(envelopeStr, nullptr, false); + if (envelope.is_discarded() || !envelope.is_object()) { + // The core promises well-formed JSON; if that ever breaks, surface the + // raw body as a structured tool error instead of crashing. + return makeTextToolResult(envelopeStr, true); + } + + bool ok = envelope.value("ok", false); + if (ok) { + std::string text; + if (envelope.contains("result")) { + text = envelope["result"].dump(); + } else { + text = "null"; + } + return makeTextToolResult(text, false); + } + + // Failure envelope: emit the structured error object as the tool result text. + std::string errText; + if (envelope.contains("error")) { + errText = envelope["error"].dump(); + } else { + errText = R"({"code":"internal","message":"rust core reported failure without an error body"})"; + } + return makeTextToolResult(errText, true); +} + // ========================================================================= // Origin validation for DNS rebinding protection. // Allowed origins: localhost variants only. @@ -222,6 +494,22 @@ std::string HttpServerComponent::generateSessionId() { return std::string(buf); } +// Evict least-recently-active sessions until the map is below MAX_SESSIONS. +// Linear scan is fine at this size; callers hold sessionMutex. +void HttpServerComponent::evictSessionsIfFullLocked() { + while (sessions.size() >= MAX_SESSIONS) { + auto lru = sessions.begin(); + for (auto it = sessions.begin(); it != sessions.end(); ++it) { + if (it->second->lastActivity < lru->second->lastActivity) { + lru = it; + } + } + logToFile("MCP: session cap (" + std::to_string(MAX_SESSIONS) + + ") reached - evicting LRU session " + lru->first); + sessions.erase(lru); + } +} + std::string HttpServerComponent::createSession(const std::string& protocolVersion) { auto session = std::make_shared(); session->sessionId = generateSessionId(); @@ -231,6 +519,7 @@ std::string HttpServerComponent::createSession(const std::string& protocolVersio session->lastActivity = session->createdAt; std::lock_guard lock(sessionMutex); + evictSessionsIfFullLocked(); sessions[session->sessionId] = session; return session->sessionId; } @@ -261,11 +550,12 @@ bool HttpServerComponent::validateOrigin(const httplib::Request& req, httplib::R } bool HttpServerComponent::validateAuth(const httplib::Request& req, httplib::Response& res) { - if (authToken.empty()) return true; // No auth configured + const std::string token = authTokenCopy(); // snapshot: ApplyConfig may rewrite it concurrently + if (token.empty()) return true; // No auth configured std::string auth = req.get_header_value("Authorization"); if (auth.size() > 7 && auth.substr(0, 7) == "Bearer ") { - if (auth.substr(7) == authToken) return true; + if (auth.substr(7) == token) return true; } logToFile("SECURITY: Unauthorized request"); @@ -345,49 +635,18 @@ HttpServerComponent::HttpServerComponent() logToFile("Log path set to: " + this->logPath); }); - // Operational status of the native listener (read-only property). + // Operational status of the native listener (read-only property). Async + // callers use the GetStatus function (property reads are blocked in + // async-only infobases). Both share buildStatusJson(). AddProperty(u"Status", u"Статус", - [&](VH var) { - json status; - status["running"] = running.load(); - status["port"] = listenPort; - { - std::lock_guard lock(pendingMutex); - status["pending_requests"] = (int)pendingRequests.size(); - } - { - std::lock_guard lock(toolsMutex); - json tools = json::parse(cachedToolsJson, nullptr, false); - status["tools_registered"] = tools.is_array() ? (int)tools.size() : 0; - } - { - std::lock_guard lock(resourcesMutex); - json res = json::parse(cachedResourcesJson, nullptr, false); - status["resources_registered"] = res.is_array() ? (int)res.size() : 0; - } - { - std::lock_guard lock(promptsMutex); - json pr = json::parse(cachedPromptsJson, nullptr, false); - status["prompts_registered"] = pr.is_array() ? (int)pr.size() : 0; - } - { - std::lock_guard lock(sessionMutex); - status["active_sessions"] = (int)sessions.size(); - } - { - std::lock_guard lock(loggingMutex); - status["logging_enabled"] = loggingEnabled; - status["log_path"] = logPath; - } - status["auth_enabled"] = !authToken.empty(); - status["version"] = VERSION_SEMVER; - var = MB2WCHAR(status.dump()); - }); + [&](VH var) { var = MB2WCHAR(this->buildStatusJson()); }); // Timeout applied to in-flight forwarded requests (read/write property). AddProperty(u"Timeout", u"Таймаут", - [&](VH var) { var = (int64_t)this->timeout; }, - [&](VH var) { this->timeout = (int)(int64_t)var; }); + [&](VH var) { var = (int64_t)this->timeout.load(); }, + // Same clamp as ApplyConfig: a non-positive timeout would fail every + // forwarded call instantly. + [&](VH var) { this->timeout.store(std::max(1, (int)(int64_t)var)); }); // Tool definitions cache (write-only property; expects JSON array). AddProperty(u"Tools", u"Инструменты", @@ -409,6 +668,59 @@ HttpServerComponent::HttpServerComponent() nullptr, [&](VH var) { this->doSetAuthToken((std::u16string)var); }); + // Drive the Rust search core (rcore.dll) directly from 1C — the ingest/admin + // methods that are NOT MCP tools: configure, index_segments, index_raw, + // stats, reset, delete_document, delete_collection. (search / get_segment / + // grep stay served by the component itself inside tools/call and are NOT + // routed here.) Returns the rcore JSON envelope verbatim: + // {"ok":true,"result":...} / {"ok":false,"error":{"code","message"}} + // On the lite component (rcore.dll absent) returns a rag_not_installed + // envelope so the 1C side can detect it and prompt to install the RAG package. + AddFunction(u"RagDispatch", u"RagВыполнить", + [&](VH method, VH payload) { + std::u16string method16 = (std::u16string)method; + std::u16string payload16 = (std::u16string)payload; + std::string methodUtf8 = WCHAR2MB(std::basic_string_view( + reinterpret_cast(method16.data()), method16.size())); + std::string payloadUtf8 = WCHAR2MB(std::basic_string_view( + reinterpret_cast(payload16.data()), payload16.size())); + if (payloadUtf8.empty()) { + payloadUtf8 = "{}"; + } + + std::string envelope; + if (!RCore::available()) { + envelope = + R"({"ok":false,"error":{"code":"rag_not_installed","message":"Semantic search backend (rcore.dll) is not installed — this is the lite component. Install the RAG (full) package to enable search."}})"; + } else { + RustString raw = RCore::dispatch(methodUtf8, payloadUtf8); + envelope = raw.str(); + if (envelope.empty()) { + envelope = + R"({"ok":false,"error":{"code":"internal","message":"rust core returned an empty response"}})"; + } + } + this->result = MB2WCHAR(envelope); + }, + {{1, std::u16string(u"{}")}}); + + // Async configuration sink. 1C cannot set component properties in + // async-only infobases (synchronous extension calls disabled), so the form + // funnels logging/timeout/tools/resources/prompts/auth through this single + // async procedure. Payload is a JSON object; every field is optional: + // {"logging_enabled":bool,"log_path":str,"timeout":int, + // "tools_json":str,"resources_json":str,"prompts_json":str, + // "auth_token":str} + // The *_json fields are pre-serialized JSON arrays (forwarded verbatim to + // the existing register handlers). + AddProcedure(u"ApplyConfig", u"ПрименитьНастройки", + [&](VH cfg) { this->doApplyConfig((std::u16string)cfg); }); + + // Async status read (the Status property is unreadable in async-only + // infobases). Returns the same JSON as the Status property. + AddFunction(u"GetStatus", u"ПолучитьСтатус", + [&]() { this->result = MB2WCHAR(this->buildStatusJson()); }); + // Capture screenshots of all visible windows belonging to a process. // pid=0 means current process. Returns base64-encoded images. // format: "jpeg" (default, smaller for AI) or "png" (lossless). @@ -497,7 +809,7 @@ void HttpServerComponent::doStartListen(int port) { std::unique_lock lock(pending->mtx); bool gotResponse = pending->cv.wait_for(lock, - std::chrono::seconds(timeout), + std::chrono::seconds(timeout.load()), [&] { return pending->ready.load(); }); if (gotResponse) { @@ -607,6 +919,15 @@ void HttpServerComponent::doStopListen() serverThread.join(); } + // The listener is fully stopped, so no worker thread can still be inside the + // native search path (RCore::dispatch). Now it is safe to cancel + join the + // Rust core's background worker so it isn't running when the DLL unloads. + // Best-effort and idempotent by contract; a no-op if rcore.dll was never + // loaded (lite component). Hooked here (server stop / form close), NOT in + // ~HttpServerComponent — the Rust singleton is process-global and outlives + // any single component instance. + RCore::shutdown(); + delete server; server = nullptr; running = false; @@ -764,10 +1085,23 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: if (!sessionId.empty()) { auto session = findSession(sessionId); if (!session) { - logToFile("MCP: invalid session " + sessionId); - res.status = 404; - res.set_content(R"({"error":"Session not found"})", "application/json"); - return; + // Unknown id usually means the server restarted and the client kept + // its old session. The spec answer is 404 + client re-initialize, + // but common IDE clients don't re-init and every call then looks + // "hung" to the user. Sessions carry no negotiated state we depend + // on, so transparently resurrect the presented id instead. + session = std::make_shared(); + session->sessionId = sessionId; + session->protocolVersion = MCP_PROTOCOL_VERSION; + session->initialized = true; + session->createdAt = std::chrono::steady_clock::now(); + session->lastActivity = session->createdAt; + { + std::lock_guard lock(sessionMutex); + evictSessionsIfFullLocked(); + sessions[sessionId] = session; + } + logToFile("MCP: unknown session " + sessionId + " resurrected (server restart?)"); } } @@ -801,6 +1135,27 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: tools = json::parse(cachedToolsJson, nullptr, false); if (tools.is_discarded()) tools = json::array(); } + if (!tools.is_array()) tools = json::array(); + + // Merge the native search-tool schemas owned by this component. The + // search subsystem is self-contained: these are always present even if + // 1C never registers them. RegisterTools REJECTS reserved native names + // with an explicit error, so the dedupe below is defense-in-depth only + // (e.g. a cache written by an older component build): native wins, to + // stay consistent with tools/call routing. Pagination applies to the + // union. + { + json merged = json::array(); + for (auto& t : tools) { + const std::string name = + t.is_object() ? t.value("name", std::string()) : std::string(); + if (!isNativeTool(name)) merged.push_back(std::move(t)); + } + for (auto& def : nativeToolDefinitions()) { + merged.push_back(std::move(def)); + } + tools = std::move(merged); + } json page = paginateJsonArray(tools, params); @@ -832,6 +1187,35 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: return; } + // ---- Native search tools: served by the Rust core, synchronously ---- + // search / get_segment / grep are handled here and NOT forwarded to 1C. + // Tool-level errors come back as a structural isError result, so a bad + // regex or missing doc_id never tears down the transport/session. + if (isNativeTool(toolName)) { + logToFile("MCP -> tools/call (native): " + toolName); + + json toolResult; + try { + toolResult = dispatchNativeTool(toolName, toolArgs); + } catch (const std::exception& e) { + toolResult = makeTextToolResult( + std::string(R"({"code":"internal","message":"native tool dispatch threw: )") + + e.what() + R"("})", + true); + } catch (...) { + toolResult = makeTextToolResult( + R"({"code":"internal","message":"native tool dispatch threw an unknown exception"})", + true); + } + + json rpcResp; + rpcResp["jsonrpc"] = "2.0"; + rpcResp["id"] = rpcId; + rpcResp["result"] = toolResult; + res.set_content(rpcResp.dump(), "application/json"); + return; + } + std::string reqId; auto pending = std::make_shared(); { @@ -861,7 +1245,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: [this, pending](size_t, httplib::DataSink& sink) { std::unique_lock lock(pending->mtx); bool hasData = pending->cv.wait_for(lock, - std::chrono::seconds(timeout), + std::chrono::seconds(timeout.load()), [&] { return !pending->sseMessages.empty() || pending->streamCompleted; }); @@ -871,7 +1255,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: makeJsonRpcToolResponse(pending->rpcIdJson, makeTextToolResult( "Timeout: 1C did not respond within " + - std::to_string(timeout) + " seconds.", + std::to_string(timeout.load()) + " seconds.", true)))); pending->streamCompleted = true; } @@ -906,7 +1290,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: { std::unique_lock lock(pending->mtx); bool gotResponse = pending->cv.wait_for(lock, - std::chrono::seconds(timeout), + std::chrono::seconds(timeout.load()), [&] { return pending->ready.load(); }); if (gotResponse) { @@ -926,7 +1310,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: rpcResp["result"] = { {"isError", true}, {"content", {{{"type", "text"}, {"text", - "Timeout: 1C did not respond within " + std::to_string(timeout) + " seconds"}}}} + "Timeout: 1C did not respond within " + std::to_string(timeout.load()) + " seconds"}}}} }; } } @@ -997,7 +1381,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: { std::unique_lock lock(pending->mtx); bool gotResponse = pending->cv.wait_for(lock, - std::chrono::seconds(timeout), + std::chrono::seconds(timeout.load()), [&] { return pending->ready.load(); }); json rpcResp; @@ -1019,7 +1403,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: } } else { rpcResp["error"] = {{"code", -32000}, - {"message", "Timeout: 1C did not respond within " + std::to_string(timeout) + " seconds"}}; + {"message", "Timeout: 1C did not respond within " + std::to_string(timeout.load()) + " seconds"}}; } res.set_content(rpcResp.dump(), "application/json"); @@ -1090,7 +1474,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: { std::unique_lock lock(pending->mtx); bool gotResponse = pending->cv.wait_for(lock, - std::chrono::seconds(timeout), + std::chrono::seconds(timeout.load()), [&] { return pending->ready.load(); }); json rpcResp; @@ -1111,7 +1495,7 @@ void HttpServerComponent::handleMcpRequest(const httplib::Request& req, httplib: } } else { rpcResp["error"] = {{"code", -32000}, - {"message", "Timeout: 1C did not respond within " + std::to_string(timeout) + " seconds"}}; + {"message", "Timeout: 1C did not respond within " + std::to_string(timeout.load()) + " seconds"}}; } res.set_content(rpcResp.dump(), "application/json"); @@ -1269,6 +1653,29 @@ void HttpServerComponent::doRegisterTools(const std::u16string& jsonStr) return; } + // Reserved names: the native search subsystem owns search/grep/get_segment/ + // list_collections, and tools/call routes them to the Rust core BEFORE the + // 1C registry — a 1C tool with such a name would be unreachable. Fail the + // registration LOUDLY (1C gets an exception) so the developer renames the + // tool instead of silently losing it. + std::string reserved; + for (const auto& t : tools) { + const std::string name = + t.is_object() ? t.value("name", std::string()) : std::string(); + if (isNativeTool(name)) { + if (!reserved.empty()) reserved += ", "; + reserved += name; + } + } + if (!reserved.empty()) { + const std::string msg = + "RegisterTools: tool name(s) reserved by the native search subsystem: " + + reserved + ". Rename them; registration rejected, tool cache unchanged."; + logToFile(msg); + AddError(MB2WCHAR(msg)); + return; + } + { std::lock_guard lock(toolsMutex); cachedToolsJson = utf8; @@ -1333,6 +1740,114 @@ void HttpServerComponent::doSetAuthToken(const std::u16string& token) std::string utf8 = WCHAR2MB(std::basic_string_view( reinterpret_cast(token.data()), token.size())); - authToken = utf8; + { + std::lock_guard lock(authTokenMutex); + authToken = utf8; + } logToFile("Auth token " + std::string(utf8.empty() ? "disabled" : "set")); } + +// Build the operational-status JSON. Shared by the Status property (sync) and +// the GetStatus function (async); the only difference is who reads it. +std::string HttpServerComponent::buildStatusJson() +{ + json status; + status["running"] = running.load(); + status["port"] = listenPort; + { + std::lock_guard lock(pendingMutex); + status["pending_requests"] = (int)pendingRequests.size(); + } + { + std::lock_guard lock(toolsMutex); + json tools = json::parse(cachedToolsJson, nullptr, false); + status["tools_registered"] = tools.is_array() ? (int)tools.size() : 0; + } + { + std::lock_guard lock(resourcesMutex); + json res = json::parse(cachedResourcesJson, nullptr, false); + status["resources_registered"] = res.is_array() ? (int)res.size() : 0; + } + { + std::lock_guard lock(promptsMutex); + json pr = json::parse(cachedPromptsJson, nullptr, false); + status["prompts_registered"] = pr.is_array() ? (int)pr.size() : 0; + } + { + std::lock_guard lock(sessionMutex); + status["active_sessions"] = (int)sessions.size(); + } + { + std::lock_guard lock(loggingMutex); + status["logging_enabled"] = loggingEnabled; + status["log_path"] = logPath; + } + status["auth_enabled"] = !authTokenCopy().empty(); + status["version"] = VERSION_SEMVER; + return status.dump(); +} + +// Apply a batch of configuration in one async call. Each field is optional; +// only present keys are touched. This mirrors the per-property setters but is +// reachable from async-only infobases where property assignment is forbidden. +void HttpServerComponent::doApplyConfig(const std::u16string& jsonStr) +{ + std::string utf8 = WCHAR2MB(std::basic_string_view( + reinterpret_cast(jsonStr.data()), jsonStr.size())); + + json cfg = json::parse(utf8, nullptr, false); + if (cfg.is_discarded() || !cfg.is_object()) { + AddError(u"ApplyConfig: expected a JSON object"); + return; + } + + if (cfg.contains("logging_enabled") && cfg["logging_enabled"].is_boolean()) { + bool enabled = cfg["logging_enabled"].get(); + { + std::lock_guard lock(loggingMutex); + loggingEnabled = enabled; + } + { + std::lock_guard lock(g_loggingMutex); + g_loggingEnabled = enabled; + } + logToFile("Logging " + std::string(enabled ? "enabled" : "disabled")); + } + + if (cfg.contains("log_path") && cfg["log_path"].is_string()) { + std::string p = cfg["log_path"].get(); + if (p.empty()) { + p = getDefaultLogPath(); + } + { + std::lock_guard lock(loggingMutex); + logPath = p; + } + { + std::lock_guard lock(g_loggingMutex); + g_logPath = p; + } + logToFile("Log path set to: " + p); + } + + if (cfg.contains("timeout") && cfg["timeout"].is_number_integer()) { + // Non-positive values would make every forwarded call "time out" + // instantly (wait_for(seconds(0)) returns immediately) — a + // hard-to-diagnose misconfiguration. Clamp to >= 1 second. + timeout.store(std::max(1, cfg["timeout"].get())); + } + + // Pre-serialized JSON arrays — forward verbatim to the existing handlers. + if (cfg.contains("tools_json") && cfg["tools_json"].is_string()) { + doRegisterTools(MB2WCHAR(cfg["tools_json"].get())); + } + if (cfg.contains("resources_json") && cfg["resources_json"].is_string()) { + doRegisterResources(MB2WCHAR(cfg["resources_json"].get())); + } + if (cfg.contains("prompts_json") && cfg["prompts_json"].is_string()) { + doRegisterPrompts(MB2WCHAR(cfg["prompts_json"].get())); + } + if (cfg.contains("auth_token") && cfg["auth_token"].is_string()) { + doSetAuthToken(MB2WCHAR(cfg["auth_token"].get())); + } +} diff --git a/http-1c-dll/src/HttpServerComponent.h b/http-1c-dll/src/HttpServerComponent.h index c60517f..435bc7e 100644 --- a/http-1c-dll/src/HttpServerComponent.h +++ b/http-1c-dll/src/HttpServerComponent.h @@ -62,7 +62,9 @@ class HttpServerComponent : public AddInNative httplib::Server* server = nullptr; std::thread serverThread; std::atomic running{false}; - int timeout = 30; + // Seconds to wait for 1C; written via ApplyConfig/property while HTTP + // worker threads read it in wait_for — hence atomic. + std::atomic timeout{30}; int listenPort = 0; public: @@ -109,6 +111,12 @@ class HttpServerComponent : public AddInNative std::mutex sessionMutex; std::map> sessions; + // Cap on tracked sessions: resurrection accepts arbitrary client-supplied + // ids, so without a bound a client could grow the map forever (memory DoS). + // When the cap is hit, the least-recently-active session is evicted. + static constexpr size_t MAX_SESSIONS = 256; + void evictSessionsIfFullLocked(); // requires sessionMutex to be held + std::string createSession(const std::string& protocolVersion); std::shared_ptr findSession(const std::string& sessionId); @@ -117,7 +125,15 @@ class HttpServerComponent : public AddInNative // ----------------------------------------------------------------------- // Security configuration. // ----------------------------------------------------------------------- - std::string authToken; // Bearer token; empty = no auth required. + // Bearer token; empty = no auth required. ApplyConfig may rewrite it while + // HTTP worker threads compare it in validateAuth — every access goes + // through authTokenMutex (readers take a copy via authTokenCopy()). + std::string authToken; + mutable std::mutex authTokenMutex; + std::string authTokenCopy() const { + std::lock_guard lock(authTokenMutex); + return authToken; + } RateLimiter rateLimiter; bool validateOrigin(const httplib::Request& req, httplib::Response& res); @@ -160,6 +176,13 @@ class HttpServerComponent : public AddInNative void doRegisterPrompts(const std::u16string& jsonStr); void doSetAuthToken(const std::u16string& token); + // Async-friendly config + status. 1C has no async property accessors, so in + // async-only infobases (synchronous extension calls disabled) properties + // cannot be set/read from the client. ApplyConfig/GetStatus expose the same + // capabilities through async methods (BeginCalling...). + void doApplyConfig(const std::u16string& jsonStr); + std::string buildStatusJson(); + // JSON-RPC endpoint handler for the MCP transport. void handleMcpRequest(const httplib::Request& req, httplib::Response& res); void handleMcpGet(const httplib::Request& req, httplib::Response& res); diff --git a/http-1c-dll/src/RustCore.h b/http-1c-dll/src/RustCore.h new file mode 100644 index 0000000..4526d68 --- /dev/null +++ b/http-1c-dll/src/RustCore.h @@ -0,0 +1,333 @@ +#ifndef HTTP1C_RUSTCORE_H +#define HTTP1C_RUSTCORE_H + +// --------------------------------------------------------------------------- +// RustCore.h — C++ surface for the Rust search core (rust-core/, crate `rcore`). +// +// The search core ships as a SEPARATE rcore.dll (cdylib, built /MD with a +// self-contained static onnxruntime). The C++ component is /MT and cannot +// compile under /MD (char16_t streams hit MSVC C2491), so it does NOT link the +// core: it loads rcore.dll at RUNTIME via LoadLibrary + GetProcAddress. One +// libhttp1cWin.dll therefore serves both packages: +// * "lite" — rcore.dll absent → RCore::available() == false → search tools +// return a structured "install RAG" result. +// * "full" — rcore.dll present (shipped next to libhttp1cWin.dll) → real +// fastembed search. +// +// rcore.dll exposes a minimal JSON-in / JSON-out C ABI (see rust-core/src): +// char* rcore_version(void); +// char* rcore_dispatch(const char* method, const char* payload_json); +// void rcore_free_string(char* s); +// void rcore_shutdown(void); +// +// MEMORY OWNERSHIP (must match rust-core/src/lib.rs): +// Every `char*` returned by an rcore_* function is allocated by Rust and MUST +// be released by calling rcore_free_string EXACTLY ONCE. Never call free() / +// delete on it — that would be a cross-CRT (and now cross-DLL) free → heap +// corruption. The RustString wrapper below frees via the LOADED +// rcore_free_string pointer, so ownership stays entirely inside rcore.dll. +// --------------------------------------------------------------------------- + +#include +#include // std::exchange + +// The Rust core ships as a Windows DLL only; on non-Windows builds (the CMake +// project still supports UNIX) this header degrades to inert stubs so the rest +// of the component compiles as the lite variant — no leakage. +#if defined(_WIN32) +#include // std::atomic ready (lock-free fast path) +#include // std::mutex serializing load attempts + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#endif // _WIN32 + +// --------------------------------------------------------------------------- +// RCore — lazy, thread-safe runtime loader for rcore.dll. +// +// The DLL is located NEXT TO THIS COMPONENT (libhttp1cWin.dll), not the process +// working directory: we resolve this module's own path via GetModuleHandleExW +// (FROM_ADDRESS of a function defined here) + GetModuleFileNameW, then load +// "\\rcore.dll". +// +// A FAILED load is retried on the next call (serialized by a mutex): the user +// can install rcore.dll next to libhttp1cWin.dll while 1C is running (e.g. the +// VA plugin's "install from archive" button) and the very next search call +// picks it up — no process restart. A failed probe is just LoadLibrary on a +// missing file (cheap), and it only happens on lite-mode search calls. Once +// loaded, the DLL stays loaded; the lock-free `ready` fast path makes the +// loaded case contention-free for concurrent httplib worker threads. +// +// If rcore.dll is missing or any of the 4 entry points is absent, available() +// returns false and the loader is an inert no-op (lite component) — never a +// crash. +// --------------------------------------------------------------------------- +class RustString; // fwd + +namespace RCore { + +#if defined(_WIN32) + +// ---- C ABI signatures of rcore.dll's exported entry points. ---- +using version_fn_t = char* (*)(void); +using dispatch_fn_t = char* (*)(const char*, const char*); +using free_fn_t = void (*)(char*); +using shutdown_fn_t = void (*)(void); + +namespace detail { + +// Loader state. Pointer fields are written only under loadMutex(); readers +// reach them through the `ready` acquire/release handshake in loaded(): the +// release store of ready=true happens AFTER the pointers are populated, so any +// thread that observes ready==true also observes the resolved pointers. +struct State { + HMODULE module = nullptr; + version_fn_t version = nullptr; + dispatch_fn_t dispatch = nullptr; + free_fn_t freeStr = nullptr; + shutdown_fn_t shutdown = nullptr; + std::atomic ready{false}; // module loaded AND all 4 symbols resolved +}; + +// An ordinary function whose address lives inside THIS module — used as the +// FROM_ADDRESS anchor so GetModuleHandleExW resolves libhttp1cWin.dll (not the +// host process). inline → one definition across all translation units. +inline void anchor() {} + +inline State& state() { + static State s; + return s; +} + +inline std::mutex& loadMutex() { + static std::mutex m; + return m; +} + +// Directory containing this component DLL, with a trailing backslash, or empty +// on failure. Uses the address of anchor() to identify the owning module. +inline std::wstring moduleDir() { + HMODULE self = nullptr; + if (!GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&anchor), + &self)) { + return std::wstring(); + } + + std::wstring path(MAX_PATH, L'\0'); + for (;;) { + DWORD len = GetModuleFileNameW(self, path.data(), + static_cast(path.size())); + if (len == 0) { + return std::wstring(); + } + if (len < path.size()) { + path.resize(len); + break; + } + // Buffer was too small (truncated) — grow and retry. + path.resize(path.size() * 2); + } + + std::wstring::size_type slash = path.find_last_of(L"\\/"); + if (slash == std::wstring::npos) { + return std::wstring(); + } + return path.substr(0, slash + 1); // keep the trailing separator +} + +inline void load(State& s) { + std::wstring dir = moduleDir(); + if (dir.empty()) { + return; + } + + s.module = LoadLibraryW((dir + L"rcore.dll").c_str()); + if (!s.module) { + return; // lite component: rcore.dll simply isn't installed. + } + + s.version = reinterpret_cast( + GetProcAddress(s.module, "rcore_version")); + s.dispatch = reinterpret_cast( + GetProcAddress(s.module, "rcore_dispatch")); + s.freeStr = reinterpret_cast( + GetProcAddress(s.module, "rcore_free_string")); + s.shutdown = reinterpret_cast( + GetProcAddress(s.module, "rcore_shutdown")); + + // Require the whole ABI: a partial/mismatched DLL is treated as "not there" + // so we degrade to the lite path instead of crashing on a null pointer. + // (A DLL caught mid-copy by "install from archive" lands here too — it is + // unloaded and the NEXT call retries the now-complete file.) + if (s.version && s.dispatch && s.freeStr && s.shutdown) { + s.ready.store(true, std::memory_order_release); + return; + } + FreeLibrary(s.module); + s.module = nullptr; + s.version = nullptr; + s.dispatch = nullptr; + s.freeStr = nullptr; + s.shutdown = nullptr; +} + +inline const State& loaded() { + State& s = state(); + // Fast path: already loaded — lock-free (acquire pairs with the release + // store in load(), making the resolved pointers visible). + if (s.ready.load(std::memory_order_acquire)) { + return s; + } + // Not loaded yet (or every previous attempt failed): retry under the lock + // so a late-installed rcore.dll is picked up without a process restart. + std::lock_guard lock(loadMutex()); + if (!s.ready.load(std::memory_order_relaxed)) { // re-check under the lock + load(s); + } + return s; +} + +} // namespace detail + +// True iff rcore.dll loaded AND all 4 entry points resolved (full component). +// Retries the load on every call until it succeeds (see loaded()), so this can +// flip lite -> full at runtime after the user installs rcore.dll. +inline bool available() { + return detail::loaded().ready.load(std::memory_order_acquire); +} + +// Free a char* returned by an rcore_* function, via the LOADED rcore_free_string +// pointer. Safe no-op if null or the core never loaded. +inline void freeString(char* s) { + if (!s) { + return; + } + const detail::State& st = detail::loaded(); + if (st.ready.load(std::memory_order_acquire) && st.freeStr) { + st.freeStr(s); + } + // If the core isn't loaded we cannot have a pointer it allocated, so there + // is nothing safe to free — intentionally leave it (never cross-CRT free()). +} + +// {"name","version","abi"} as JSON, or "" if the core isn't available. +RustString version(); + +// Generic JSON-in / JSON-out call. Returns an empty RustString if the core +// isn't available (callers must check available() first / treat empty as error). +RustString dispatch(const std::string& method, const std::string& payloadJson); + +// Best-effort teardown of the core's background workers. No-op if rcore.dll was +// never loaded. Idempotent; hook onto doStopListen()/form-close, NOT a +// component destructor — the Rust singleton outlives any single instance. +inline void shutdown() { + const detail::State& st = detail::loaded(); + if (st.ready.load(std::memory_order_acquire) && st.shutdown) { + st.shutdown(); + } +} + +#else // !_WIN32 — inert lite stubs (no Rust core on this platform) + +inline bool available() { return false; } +inline void freeString(char*) {} +inline void shutdown() {} + +RustString version(); +RustString dispatch(const std::string& method, const std::string& payloadJson); + +#endif // _WIN32 + +} // namespace RCore + +// --------------------------------------------------------------------------- +// RustString — RAII owner for a char* returned by the Rust core. +// +// Guarantees the string is freed via the loaded rcore_free_string exactly once. +// Move-only (copying would risk a double free). Use .c_str() to read, .str() to +// copy into a std::string. +// +// RustString r = RCore::dispatch("ping", "{}"); +// std::string body = r.str(); // copy out; r frees the buffer on scope exit +// --------------------------------------------------------------------------- +class RustString { +public: + // Take ownership of a char* returned by the Rust core. + static RustString adopt(char* raw) { return RustString(raw); } + + RustString() noexcept : ptr_(nullptr) {} + ~RustString() { reset(); } + + // Move-only: the wrapper is the unique owner of the buffer. + RustString(RustString&& other) noexcept + : ptr_(std::exchange(other.ptr_, nullptr)) {} + RustString& operator=(RustString&& other) noexcept { + if (this != &other) { + reset(); + ptr_ = std::exchange(other.ptr_, nullptr); + } + return *this; + } + + RustString(const RustString&) = delete; + RustString& operator=(const RustString&) = delete; + + // Raw, still-owned pointer (may be nullptr). Read-only use. + const char* c_str() const noexcept { return ptr_; } + + // Copy the contents into a std::string ("" if null). + std::string str() const { return ptr_ ? std::string(ptr_) : std::string(); } + + bool valid() const noexcept { return ptr_ != nullptr; } + + // Release ownership without freeing (rarely needed). + char* release() noexcept { return std::exchange(ptr_, nullptr); } + +private: + explicit RustString(char* raw) noexcept : ptr_(raw) {} + + void reset() noexcept { + if (ptr_) { + RCore::freeString(ptr_); + ptr_ = nullptr; + } + } + + char* ptr_; +}; + +namespace RCore { + +#if defined(_WIN32) + +inline RustString version() { + const detail::State& st = detail::loaded(); + if (!st.ready.load(std::memory_order_acquire)) { + return RustString(); + } + return RustString::adopt(st.version()); +} + +inline RustString dispatch(const std::string& method, const std::string& payloadJson) { + const detail::State& st = detail::loaded(); + if (!st.ready.load(std::memory_order_acquire)) { + return RustString(); + } + return RustString::adopt(st.dispatch(method.c_str(), payloadJson.c_str())); +} + +#else // !_WIN32 + +inline RustString version() { return RustString(); } +inline RustString dispatch(const std::string&, const std::string&) { return RustString(); } + +#endif // _WIN32 + +} // namespace RCore + +#endif // HTTP1C_RUSTCORE_H diff --git a/http-1c-dll/tests/smoke/main.cpp b/http-1c-dll/tests/smoke/main.cpp new file mode 100644 index 0000000..a111d66 --- /dev/null +++ b/http-1c-dll/tests/smoke/main.cpp @@ -0,0 +1,955 @@ +// =========================================================================== +// http1c_smoke — automated black-box smoke harness for the 1C Native API +// component (libhttp1cWin.dll). +// +// Loads the BUILT DLL exactly like 1C:Enterprise does: +// LoadLibrary -> GetClassObject("HttpServer") -> setMemManager -> Init +// with minimal stub implementations of the 1C-side interfaces +// (IMemoryManager, IAddInDefBase/IAddInDefBaseEx), drives the component +// over real HTTP (cpp-httplib client against the component's MCP endpoint) +// and asserts the behaviors: +// +// T1 load + init + GetProcessId plumbing +// T2 ApplyConfig(tools_json, timeout) + StartListen + MCP initialize +// T3 tools/list = 1C tools UNION native search tools, no duplicates +// T4 RegisterTools reserved-name rejection (AddError, cache unchanged) +// T5 unknown Mcp-Session-Id is transparently resurrected (200, not 404) +// T6 Bearer auth 401/200 + concurrent requests vs ApplyConfig token flips +// T7 tools/call timeout: ExternalEvent fired, "1 second" error, no hang +// T8 UTF-8 round-trip of Russian tool name/description +// T9 native list_collections callable (ok-collections OR rag_not_installed) +// T10 late rcore.dll install: lite copy answers rag_not_installed (message +// names all 4 native tools), then rcore.dll is copied in and the SAME +// loaded module flips to full — no process restart (loader retry) +// +// Usage: http1c_smoke.exe [path\to\libhttp1cWin.dll] +// Default DLL path: "..\bin\libhttp1cWin.dll" relative to the exe +// (falls back to "libhttp1cWin.dll" next to the exe). +// +// Exit code: 0 = all checks passed, otherwise the number of failed checks. +// =========================================================================== + +#ifndef _WINDOWS +#define _WINDOWS // SDK headers key off this: WCHAR_T = wchar_t, ADDIN_API = __stdcall +#endif + +#include "ComponentBase.h" +#include "AddInDefBase.h" +#include "IMemoryManager.h" + +#include "httplib.h" // also pulls in winsock2/ws2tcpip and runs WSAStartup +#include "json.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +// =========================================================================== +// Tiny assertion framework +// =========================================================================== + +static int g_checks = 0; +static int g_failures = 0; +static bool g_fatal = false; // set when continuing makes no sense (DLL/listen down) + +static bool checkImpl(bool ok, const std::string& msg, int line) { + ++g_checks; + if (!ok) { + ++g_failures; + std::printf(" CHECK FAILED (main.cpp:%d): %s\n", line, msg.c_str()); + std::fflush(stdout); + } + return ok; +} + +#define CHECK(cond, msg) checkImpl(!!(cond), (msg), __LINE__) +// Failing a REQUIRE marks the whole run fatal (subsequent tests are skipped). +#define REQUIRE(cond, msg) do { if (!checkImpl(!!(cond), (msg), __LINE__)) g_fatal = true; } while (0) + +template +static void runTest(const char* name, Fn&& fn) { + std::printf("=== %s ===\n", name); + std::fflush(stdout); + if (g_fatal) { + std::printf("SKIP: %s (previous fatal failure)\n", name); + std::fflush(stdout); + return; + } + const int before = g_failures; + try { + fn(); + } catch (const std::exception& e) { + CHECK(false, std::string("unhandled exception: ") + e.what()); + } catch (...) { + CHECK(false, "unhandled non-standard exception"); + } + std::printf("%s: %s\n", (g_failures == before) ? "PASS" : "FAIL", name); + std::fflush(stdout); +} + +// =========================================================================== +// UTF helpers — on Windows WCHAR_T is 16-bit wchar_t; convert via Win32. +// =========================================================================== + +static std::wstring utf8ToWide(const std::string& s) { + if (s.empty()) return std::wstring(); + const int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), (int)s.size(), nullptr, 0); + if (n <= 0) return std::wstring(); + std::wstring w((size_t)n, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s.data(), (int)s.size(), &w[0], n); + return w; +} + +static std::string wideToUtf8(const wchar_t* p) { + if (!p || !*p) return std::string(); + const int wlen = (int)wcslen(p); + const int n = WideCharToMultiByte(CP_UTF8, 0, p, wlen, nullptr, 0, nullptr, nullptr); + if (n <= 0) return std::string(); + std::string s((size_t)n, '\0'); + WideCharToMultiByte(CP_UTF8, 0, p, wlen, &s[0], n, nullptr, nullptr); + return s; +} + +// =========================================================================== +// 1C interface stubs +// =========================================================================== + +// IMemoryManager: malloc/free. The component allocates return-value strings +// through this; the harness (playing 1C) frees them after reading. +class MemoryManagerStub : public IMemoryManager { +public: + bool ADDIN_API AllocMemory(void** pMemory, unsigned long ulCountByte) override { + if (!pMemory) return false; + *pMemory = std::malloc(ulCountByte ? ulCountByte : 1); + return *pMemory != nullptr; + } + void ADDIN_API FreeMemory(void** pMemory) override { + if (pMemory && *pMemory) { + std::free(*pMemory); + *pMemory = nullptr; + } + } +}; + +// IAddInDefBaseEx: captures AddError calls and ExternalEvent notifications. +// ExternalEvent arrives from httplib worker threads -> mutex-guarded. +class ConnectionStub : public IAddInDefBaseEx { +public: + struct ErrorRecord { std::string source, descr; }; + struct EventRecord { std::string source, event, data; }; + + bool ADDIN_API AddError(unsigned short /*wcode*/, const WCHAR_T* source, + const WCHAR_T* descr, long /*scode*/) override { + std::lock_guard lock(mtx_); + errors_.push_back({ wideToUtf8(source), wideToUtf8(descr) }); + return true; + } + bool ADDIN_API Read(WCHAR_T*, tVariant*, long*, WCHAR_T**) override { return false; } + bool ADDIN_API Write(WCHAR_T*, tVariant*) override { return false; } + bool ADDIN_API RegisterProfileAs(WCHAR_T*) override { return true; } + bool ADDIN_API SetEventBufferDepth(long depth) override { depth_ = depth; return true; } + long ADDIN_API GetEventBufferDepth() override { return depth_; } + bool ADDIN_API ExternalEvent(WCHAR_T* source, WCHAR_T* message, WCHAR_T* data) override { + std::lock_guard lock(mtx_); + events_.push_back({ wideToUtf8(source), wideToUtf8(message), wideToUtf8(data) }); + return true; + } + void ADDIN_API CleanEventBuffer() override {} + bool ADDIN_API SetStatusLine(WCHAR_T*) override { return true; } + void ADDIN_API ResetStatusLine() override {} + IInterface* ADDIN_API GetInterface(Interfaces) override { return nullptr; } + + void clearErrors() { std::lock_guard l(mtx_); errors_.clear(); } + void clearEvents() { std::lock_guard l(mtx_); events_.clear(); } + std::vector errors() { std::lock_guard l(mtx_); return errors_; } + std::vector events() { std::lock_guard l(mtx_); return events_; } + +private: + std::mutex mtx_; + std::vector errors_; + std::vector events_; + long depth_ = 0; +}; + +// =========================================================================== +// tVariant helpers +// =========================================================================== + +static void setVariantString(tVariant& v, const std::wstring& s) { + tVarInit(&v); + v.vt = VTYPE_PWSTR; + v.pwstrVal = const_cast(s.c_str()); // null-terminated, harness-owned + v.wstrLen = (uint32_t)s.size(); +} + +static void setVariantInt(tVariant& v, int32_t value) { + tVarInit(&v); + v.vt = VTYPE_I4; + v.lVal = value; +} + +// Free a return value allocated by the component via MemoryManagerStub. +static void freeVariant(tVariant& v) { + if ((v.vt == VTYPE_PWSTR || v.vt == VTYPE_BLOB) && v.pwstrVal) { + std::free(v.pwstrVal); + } + tVarInit(&v); +} + +static std::string variantToUtf8(const tVariant& v) { + if (v.vt == VTYPE_PWSTR && v.pwstrVal) return wideToUtf8(v.pwstrVal); + return std::string(); +} + +static long long variantToInt(const tVariant& v) { + switch (v.vt) { + case VTYPE_I2: return v.shortVal; + case VTYPE_I4: return v.lVal; + case VTYPE_R8: return (long long)v.dblVal; + case VTYPE_I8: return v.llVal; + default: return 0; + } +} + +// =========================================================================== +// Component harness — loads the DLL and dispatches methods like 1C does. +// =========================================================================== + +struct Harness { + HMODULE dll = nullptr; + GetClassObjectPtr getClassObject = nullptr; + DestroyObjectPtr destroyObject = nullptr; + GetClassNamesPtr getClassNames = nullptr; + IComponentBase* comp = nullptr; + MemoryManagerStub mem; + ConnectionStub conn; + + bool load(const std::wstring& dllPath) { + dll = ::LoadLibraryW(dllPath.c_str()); + if (!dll) return false; + getClassObject = (GetClassObjectPtr)::GetProcAddress(dll, "GetClassObject"); + destroyObject = (DestroyObjectPtr)::GetProcAddress(dll, "DestroyObject"); + getClassNames = (GetClassNamesPtr)::GetProcAddress(dll, "GetClassNames"); + return getClassObject && destroyObject && getClassNames; + } + + long findMethod(const wchar_t* nameEn) { + return comp ? comp->FindMethod((const WCHAR_T*)nameEn) : -1; + } + + // Calls a method as a procedure (no return value). + bool callProc(const wchar_t* nameEn, tVariant* params, long count) { + const long num = findMethod(nameEn); + if (num < 0) return false; + return comp->CallAsProc(num, params, count); + } + + // Calls a method as a function; caller owns/frees *ret via freeVariant. + bool callFunc(const wchar_t* nameEn, tVariant* ret, tVariant* params, long count) { + const long num = findMethod(nameEn); + if (num < 0) return false; + tVarInit(ret); + return comp->CallAsFunc(num, ret, params, count); + } + + // ApplyConfig with a JSON object built by the caller. Async methods in this + // component execute synchronously inside CallAsProc (1C wraps the async + // part), so a plain call is the real production code path. + bool applyConfig(const json& cfg) { + const std::wstring w = utf8ToWide(cfg.dump()); + tVariant p; + setVariantString(p, w); + return callProc(L"ApplyConfig", &p, 1); + } + + void destroy() { + if (comp && destroyObject) { + destroyObject(&comp); + comp = nullptr; + } + // Intentionally no FreeLibrary: rcore.dll (if loaded) keeps background + // state; the process is about to exit anyway and the OS reclaims it. + } +}; + +// =========================================================================== +// HTTP helpers +// =========================================================================== + +static std::unique_ptr makeClient(int port) { + auto cli = std::make_unique("127.0.0.1", port); + cli->set_connection_timeout(5, 0); + cli->set_read_timeout(15, 0); + cli->set_write_timeout(5, 0); + return cli; +} + +struct RpcOptions { + std::string sessionId; // adds Mcp-Session-Id when non-empty + std::string bearerToken; // adds Authorization: Bearer ... when non-empty + std::string accept = "application/json"; +}; + +static httplib::Result mcpPost(httplib::Client& cli, const json& rpc, const RpcOptions& opt = {}) { + httplib::Headers headers; + headers.emplace("Accept", opt.accept); + if (!opt.sessionId.empty()) headers.emplace("Mcp-Session-Id", opt.sessionId); + if (!opt.bearerToken.empty()) headers.emplace("Authorization", "Bearer " + opt.bearerToken); + return cli.Post("/mcp", headers, rpc.dump(), "application/json"); +} + +static json rpcRequest(const std::string& method, int id, const json& params = json::object()) { + json rpc; + rpc["jsonrpc"] = "2.0"; + rpc["id"] = id; + rpc["method"] = method; + if (!params.empty()) rpc["params"] = params; + return rpc; +} + +// Bind-probe 127.0.0.1:; returns the first free port from the range. +static int pickFreePort() { + for (int port = 18876; port < 18896; ++port) { + SOCKET s = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (s == INVALID_SOCKET) continue; + sockaddr_in addr; + std::memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons((u_short)port); + ::inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr); + const int rc = ::bind(s, (sockaddr*)&addr, sizeof(addr)); + ::closesocket(s); + if (rc == 0) return port; + } + return 18876; // last resort; StartListen will fail loudly if busy +} + +// Extract tool names from a tools/list response body. +static std::vector toolNames(const json& body) { + std::vector names; + if (body.contains("result") && body["result"].contains("tools")) { + for (const auto& t : body["result"]["tools"]) { + names.push_back(t.value("name", std::string())); + } + } + return names; +} + +// Directory of the running exe, with trailing backslash. +static std::wstring exeDir() { + wchar_t buf[MAX_PATH * 4] = L""; + ::GetModuleFileNameW(nullptr, buf, (DWORD)(sizeof(buf) / sizeof(buf[0]))); + std::wstring path(buf); + const size_t slash = path.find_last_of(L"\\/"); + return (slash == std::wstring::npos) ? std::wstring() : path.substr(0, slash + 1); +} + +static bool fileExists(const std::wstring& p) { + const DWORD attrs = ::GetFileAttributesW(p.c_str()); + return attrs != INVALID_FILE_ATTRIBUTES && !(attrs & FILE_ATTRIBUTE_DIRECTORY); +} + +// =========================================================================== +// main — tests run sequentially against one component instance. +// =========================================================================== + +int wmain(int argc, wchar_t** argv) { + Harness h; + int port = 0; + std::string sessionId; + + std::wstring dllPath; + if (argc > 1) { + dllPath = argv[1]; + } else { + dllPath = exeDir() + L"..\\bin\\libhttp1cWin.dll"; + if (!fileExists(dllPath)) dllPath = exeDir() + L"libhttp1cWin.dll"; + } + std::printf("DLL under test: %s\n", wideToUtf8(dllPath.c_str()).c_str()); + + // ----------------------------------------------------------------------- + // T1: load + init + call plumbing sanity (GetProcessId). + // ----------------------------------------------------------------------- + runTest("T1 load+init", [&] { + REQUIRE(h.load(dllPath), "LoadLibrary + GetProcAddress(GetClassObject/DestroyObject/GetClassNames)"); + if (g_fatal) return; + + const std::string classNames = wideToUtf8(h.getClassNames()); + CHECK(classNames.find("HttpServer") != std::string::npos, + "GetClassNames contains HttpServer, got: " + classNames); + + h.comp = nullptr; + h.getClassObject(L"HttpServer", &h.comp); + REQUIRE(h.comp != nullptr, "GetClassObject(\"HttpServer\") returned an instance"); + if (g_fatal) return; + + REQUIRE(h.comp->setMemManager(&h.mem), "setMemManager accepted the stub"); + REQUIRE(h.comp->Init(static_cast(&h.conn)), "Init(IAddInDefBase*) returned true"); + CHECK(h.comp->GetInfo() == 2000, "GetInfo() == 2000"); + + tVariant ret; + REQUIRE(h.callFunc(L"GetProcessId", &ret, nullptr, 0), "CallAsFunc(GetProcessId)"); + CHECK((unsigned long long)variantToInt(ret) == (unsigned long long)::GetCurrentProcessId(), + "GetProcessId returned the current pid"); + freeVariant(ret); + }); + + // ----------------------------------------------------------------------- + // T2: ApplyConfig (tools_json + timeout, logging off) -> StartListen -> + // HTTP MCP initialize returns serverInfo + Mcp-Session-Id header. + // ----------------------------------------------------------------------- + runTest("T2 config+listen+initialize", [&] { + json tool = { + {"name", "my1cTool"}, + {"description", "d"}, + {"inputSchema", {{"type", "object"}, {"properties", json::object()}}} + }; + json cfg; + cfg["logging_enabled"] = false; // keep the harness from writing log files + cfg["tools_json"] = json::array({ tool }).dump(); + cfg["timeout"] = 2; + REQUIRE(h.applyConfig(cfg), "ApplyConfig(tools_json, timeout)"); + + port = pickFreePort(); + std::printf(" listening port: %d\n", port); + tVariant pPort, ret; + setVariantInt(pPort, port); + REQUIRE(h.callFunc(L"StartListen", &ret, &pPort, 1), "CallAsFunc(StartListen)"); + freeVariant(ret); + if (g_fatal) return; + + // The listener binds on a background thread — poll /health until live. + auto cli = makeClient(port); + bool up = false; + for (int i = 0; i < 100 && !up; ++i) { + auto res = cli->Get("/health"); + if (res && res->status == 200) up = true; + else std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + REQUIRE(up, "GET /health responds 200 after StartListen"); + if (g_fatal) return; + + json params = { + {"protocolVersion", "2025-03-26"}, + {"capabilities", json::object()}, + {"clientInfo", {{"name", "http1c_smoke"}, {"version", "1.0"}}} + }; + RpcOptions opt; + opt.accept = "application/json, text/event-stream"; + auto res = mcpPost(*cli, rpcRequest("initialize", 1, params), opt); + REQUIRE(res && res->status == 200, "POST /mcp initialize -> 200"); + if (g_fatal) return; + + const json body = json::parse(res->body, nullptr, false); + CHECK(!body.is_discarded() && body.contains("result"), "initialize: JSON-RPC result present"); + CHECK(body["result"].contains("serverInfo") && + body["result"]["serverInfo"].value("name", std::string()) == "1c-mcp-server", + "initialize: serverInfo.name == 1c-mcp-server"); + sessionId = res->get_header_value("Mcp-Session-Id"); + REQUIRE(!sessionId.empty(), "initialize: Mcp-Session-Id header present"); + }); + + // ----------------------------------------------------------------------- + // T3: tools/list = union of the 1C tool and the 4 native search tools, + // all names unique, count exactly 5. + // ----------------------------------------------------------------------- + runTest("T3 tools/list union, no duplicates", [&] { + auto cli = makeClient(port); + RpcOptions opt; opt.sessionId = sessionId; + auto res = mcpPost(*cli, rpcRequest("tools/list", 2), opt); + REQUIRE(res && res->status == 200, "tools/list -> 200"); + if (g_fatal) return; + + const json body = json::parse(res->body, nullptr, false); + const auto names = toolNames(body); + const std::set unique(names.begin(), names.end()); + + CHECK(names.size() == 5, "tools/list returns exactly 5 tools, got " + std::to_string(names.size())); + CHECK(unique.size() == names.size(), "all tool names are unique"); + for (const char* expected : {"my1cTool", "search", "grep", "get_segment", "list_collections"}) { + CHECK(unique.count(expected) == 1, std::string("tools/list contains ") + expected); + } + }); + + // ----------------------------------------------------------------------- + // T4: registering a tool named like a native one is rejected with AddError + // ("reserved...") and the previous cache stays intact. + // ----------------------------------------------------------------------- + runTest("T4 reserved-name rejection", [&] { + h.conn.clearErrors(); + json tool = { + {"name", "search"}, + {"description", "imposter"}, + {"inputSchema", {{"type", "object"}, {"properties", json::object()}}} + }; + json cfg; + cfg["tools_json"] = json::array({ tool }).dump(); + CHECK(h.applyConfig(cfg), "ApplyConfig itself succeeds (rejection is reported via AddError)"); + + bool sawReserved = false; + for (const auto& e : h.conn.errors()) { + if (e.descr.find("reserved") != std::string::npos) sawReserved = true; + } + CHECK(sawReserved, "AddError captured with message containing \"reserved\""); + + // Cache unchanged: still the previous 5 tools incl. my1cTool. + auto cli = makeClient(port); + RpcOptions opt; opt.sessionId = sessionId; + auto res = mcpPost(*cli, rpcRequest("tools/list", 3), opt); + REQUIRE(res && res->status == 200, "tools/list after rejection -> 200"); + if (g_fatal) return; + const auto names = toolNames(json::parse(res->body, nullptr, false)); + const std::set unique(names.begin(), names.end()); + CHECK(names.size() == 5, "tool cache unchanged: still 5 tools"); + CHECK(unique.count("my1cTool") == 1, "tool cache unchanged: my1cTool still present"); + }); + + // ----------------------------------------------------------------------- + // T5: unknown session id is transparently resurrected (no 404). + // ----------------------------------------------------------------------- + runTest("T5 session resurrection", [&] { + auto cli = makeClient(port); + RpcOptions opt; opt.sessionId = "bogus-id-123"; + auto res = mcpPost(*cli, rpcRequest("tools/list", 4), opt); + REQUIRE((bool)res, "tools/list with bogus session got a response"); + if (g_fatal) return; + CHECK(res->status == 200, "bogus Mcp-Session-Id -> HTTP 200 (resurrected), got " + std::to_string(res->status)); + const json body = json::parse(res->body, nullptr, false); + CHECK(!body.is_discarded() && body.contains("result") && + body["result"].contains("tools") && body["result"]["tools"].is_array() && + !body["result"]["tools"].empty(), + "resurrected session still serves a valid tools result"); + }); + + // ----------------------------------------------------------------------- + // T5b: the session map is capped — a flood of unknown ids cannot grow it + // without bound (LRU eviction at MAX_SESSIONS=256). Paced to respect + // the component's token-bucket rate limiter, so this test takes ~12s. + // ----------------------------------------------------------------------- + runTest("T5b session cap under id flood", [&] { + auto cli = makeClient(port); + int served = 0; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(45); + for (int i = 0; i < 300 && std::chrono::steady_clock::now() < deadline; ++i) { + RpcOptions opt; opt.sessionId = "flood-" + std::to_string(i); + for (;;) { + auto res = mcpPost(*cli, rpcRequest("tools/list", 1000 + i), opt); + if (res && res->status == 429) { // rate limiter — wait for refill + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (std::chrono::steady_clock::now() >= deadline) break; + continue; + } + if (res && res->status == 200) ++served; + break; + } + } + CHECK(served >= 280, "id flood mostly served, got " + std::to_string(served) + "/300"); + + tVariant ret; + REQUIRE(h.callFunc(L"GetStatus", &ret, nullptr, 0), "CallAsFunc(GetStatus)"); + if (g_fatal) return; + const json st = json::parse(variantToUtf8(ret), nullptr, false); + freeVariant(ret); + REQUIRE(!st.is_discarded() && st.contains("active_sessions"), + "GetStatus returns active_sessions"); + if (g_fatal) return; + const int active = st["active_sessions"].get(); + CHECK(active <= 256, "sessions capped at 256 (active=" + std::to_string(active) + ")"); + CHECK(active >= 200, "cap actually engaged near the limit (active=" + std::to_string(active) + ")"); + }); + + // ----------------------------------------------------------------------- + // T6: Bearer auth + concurrency smoke against ApplyConfig token flips. + // ----------------------------------------------------------------------- + runTest("T6 auth + concurrent token flips", [&] { + json cfg; cfg["auth_token"] = "sekret"; + REQUIRE(h.applyConfig(cfg), "ApplyConfig(auth_token=sekret)"); + + auto cli = makeClient(port); + // Earlier tests (T5b's id flood) may leave the token bucket drained, so + // these deterministic auth checks can hit 429 first — that is the rate + // limiter doing its job, not an auth verdict. Retry through 429s + // (refill is 20 tokens/s, settles in well under a second). + auto postRetrying429 = [&](int id, const RpcOptions& opt) { + auto r = mcpPost(*cli, rpcRequest("tools/list", id), opt); + for (int i = 0; i < 40 && r && r->status == 429; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + r = mcpPost(*cli, rpcRequest("tools/list", id), opt); + } + return r; + }; + + RpcOptions noAuth; noAuth.sessionId = sessionId; + auto res = postRetrying429(5, noAuth); + REQUIRE((bool)res, "request without Authorization got a response"); + if (!g_fatal) CHECK(res->status == 401, "no Authorization -> 401, got " + std::to_string(res->status)); + + RpcOptions withAuth; withAuth.sessionId = sessionId; withAuth.bearerToken = "sekret"; + res = postRetrying429(6, withAuth); + REQUIRE((bool)res, "request with Bearer sekret got a response"); + if (!g_fatal) CHECK(res->status == 200, "Bearer sekret -> 200, got " + std::to_string(res->status)); + if (g_fatal) return; + + // Concurrency smoke: 4 threads x 50 requests alternating valid token / + // no token, while the main thread flips auth_token between "sekret" and + // "" twenty times. Every response must be 200/401/429 — 429 comes from + // the component's own token-bucket rate limiter (60 burst, 20 rps) that + // 200 rapid-fire requests legitimately trip; anything else (5xx, + // connection error, hang) is a failure. + std::atomic bad{0}, ok200{0}, unauth401{0}, limited429{0}, connErr{0}; + std::vector threads; + for (int t = 0; t < 4; ++t) { + threads.emplace_back([&, t] { + auto threadCli = makeClient(port); + for (int i = 0; i < 50; ++i) { + RpcOptions opt; + opt.sessionId = sessionId; + if (i % 2 == 0) opt.bearerToken = "sekret"; + auto r = mcpPost(*threadCli, rpcRequest("tools/list", 100 + t * 50 + i), opt); + if (!r) { ++connErr; ++bad; continue; } + switch (r->status) { + case 200: ++ok200; break; + case 401: ++unauth401; break; + case 429: ++limited429; break; + default: ++bad; break; + } + } + }); + } + for (int i = 0; i < 20; ++i) { + json flip; flip["auth_token"] = (i % 2 == 0) ? "" : "sekret"; + h.applyConfig(flip); + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } + // End with auth ON so trailing no-token requests still see 401s. + { json flip; flip["auth_token"] = "sekret"; h.applyConfig(flip); } + for (auto& th : threads) th.join(); + + std::printf(" concurrency: 200=%d 401=%d 429=%d connErr=%d other=%d\n", + ok200.load(), unauth401.load(), limited429.load(), connErr.load(), + bad.load() - connErr.load()); + CHECK(bad.load() == 0, "no 5xx / connection errors / unexpected statuses under concurrent token flips"); + CHECK(ok200.load() > 0, "at least one 200 observed"); + CHECK(unauth401.load() > 0, "at least one 401 observed"); + + // Disable auth for the remaining tests. + json off; off["auth_token"] = ""; + REQUIRE(h.applyConfig(off), "ApplyConfig(auth_token=\"\") resets auth"); + }); + + // ----------------------------------------------------------------------- + // T7: tools/call of a 1C tool with timeout=1 — ExternalEvent fired, and + // with nobody answering the call returns the timeout error promptly. + // ----------------------------------------------------------------------- + runTest("T7 timeout + ExternalEvent", [&] { + // Non-positive timeouts are clamped to 1s (instant-timeout footgun); + // verified through the Timeout property round-trip. + auto readTimeoutProp = [&]() -> long long { + const long propNum = h.comp->FindProp((const WCHAR_T*)L"Timeout"); + tVariant val; + tVarInit(&val); + if (propNum < 0 || !h.comp->GetPropVal(propNum, &val)) return -1; + const long long v = variantToInt(val); + freeVariant(val); + return v; + }; + for (const int bad : {0, -5}) { + json cfgBad; cfgBad["timeout"] = bad; + CHECK(h.applyConfig(cfgBad), "ApplyConfig(timeout=" + std::to_string(bad) + ") accepted"); + CHECK(readTimeoutProp() == 1, + "timeout=" + std::to_string(bad) + " clamped to 1 second"); + } + + json cfg; cfg["timeout"] = 1; + REQUIRE(h.applyConfig(cfg), "ApplyConfig(timeout=1)"); + h.conn.clearEvents(); + + json params = { + {"name", "my1cTool"}, + {"arguments", json::object()} + }; + auto cli = makeClient(port); + RpcOptions opt; opt.sessionId = sessionId; // plain JSON (no SSE): no progressToken + const auto t0 = std::chrono::steady_clock::now(); + auto res = mcpPost(*cli, rpcRequest("tools/call", 7, params), opt); + const auto elapsedMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + + REQUIRE(res && res->status == 200, "tools/call my1cTool -> 200 (timeout is a tool-level error)"); + if (g_fatal) return; + CHECK(elapsedMs < 3000, "timeout response arrived within ~3s (took " + std::to_string(elapsedMs) + "ms)"); + + const json body = json::parse(res->body, nullptr, false); + bool isError = false; + std::string text; + if (!body.is_discarded() && body.contains("result")) { + isError = body["result"].value("isError", false); + if (body["result"].contains("content") && !body["result"]["content"].empty()) { + text = body["result"]["content"][0].value("text", std::string()); + } + } + CHECK(isError, "timeout reported as result.isError = true"); + CHECK(text.find("did not respond within 1 second") != std::string::npos, + "timeout message mentions \"1 second\", got: " + text); + + bool sawToolCall = false; + for (const auto& e : h.conn.events()) { + if (e.source == "HttpServer" && e.event == "ToolCall") { + const json data = json::parse(e.data, nullptr, false); + if (!data.is_discarded() && data.value("tool", std::string()) == "my1cTool") { + sawToolCall = true; + } + } + } + CHECK(sawToolCall, "ExternalEvent(HttpServer, ToolCall, {tool: my1cTool}) captured by the stub"); + }); + + // ----------------------------------------------------------------------- + // T8: UTF-8 round-trip — Russian tool name/description survive + // MB2WCHAR/WCHAR2MB byte-identically. + // ----------------------------------------------------------------------- + runTest("T8 UTF-8 round-trip", [&] { + const std::string ruName = (const char*)u8"инструмент_тест"; + const std::string ruDescr = (const char*)u8"Описание по-русски ёЁ"; + json tool = { + {"name", ruName}, + {"description", ruDescr}, + {"inputSchema", {{"type", "object"}, {"properties", json::object()}}} + }; + json cfg; + cfg["tools_json"] = json::array({ tool }).dump(); + REQUIRE(h.applyConfig(cfg), "ApplyConfig(tools_json with Russian strings)"); + + auto cli = makeClient(port); + RpcOptions opt; opt.sessionId = sessionId; + auto res = mcpPost(*cli, rpcRequest("tools/list", 8), opt); + REQUIRE(res && res->status == 200, "tools/list -> 200"); + if (g_fatal) return; + + const json body = json::parse(res->body, nullptr, false); + bool found = false; + std::string gotDescr; + if (!body.is_discarded() && body.contains("result") && body["result"].contains("tools")) { + for (const auto& t : body["result"]["tools"]) { + if (t.value("name", std::string()) == ruName) { + found = true; + gotDescr = t.value("description", std::string()); + } + } + } + CHECK(found, "tools/list returns the byte-identical UTF-8 tool name"); + CHECK(gotDescr == ruDescr, "tools/list returns the byte-identical UTF-8 description"); + }); + + // ----------------------------------------------------------------------- + // T9: native list_collections is callable end-to-end. Either real + // collections (rcore.dll present beside the DLL) or the structured + // rag_not_installed error (lite) — both are correct. + // ----------------------------------------------------------------------- + runTest("T9 native list_collections", [&] { + json params = { + {"name", "list_collections"}, + {"arguments", json::object()} + }; + auto cli = makeClient(port); + RpcOptions opt; opt.sessionId = sessionId; + auto res = mcpPost(*cli, rpcRequest("tools/call", 9, params), opt); + REQUIRE(res && res->status == 200, "tools/call list_collections -> 200"); + if (g_fatal) return; + + const json body = json::parse(res->body, nullptr, false); + REQUIRE(!body.is_discarded() && body.contains("result"), "list_collections: JSON-RPC result present"); + if (g_fatal) return; + + const bool isError = body["result"].value("isError", false); + std::string text; + if (body["result"].contains("content") && !body["result"]["content"].empty()) { + text = body["result"]["content"][0].value("text", std::string()); + } + const json payload = json::parse(text, nullptr, false); + + const bool okCollections = !isError && !payload.is_discarded() && + payload.is_object() && payload.contains("collections"); + const bool ragNotInstalled = isError && !payload.is_discarded() && + payload.is_object() && payload.value("code", std::string()) == "rag_not_installed"; + + std::printf(" list_collections variant: %s\n", + okCollections ? "ok-collections (rcore.dll present)" : + ragNotInstalled ? "rag_not_installed (lite)" : "UNEXPECTED"); + CHECK(okCollections || ragNotInstalled, + "list_collections returned ok-collections or rag_not_installed, got: " + text); + }); + + // ----------------------------------------------------------------------- + // T10: late rcore.dll install — lite flips to full WITHOUT a restart. + // A second copy of the component DLL is loaded from a temp dir with + // no rcore.dll beside it: native search must answer rag_not_installed + // and the message must name all four native tools. Then rcore.dll is + // copied in and the SAME loaded module is called again — the RCore + // loader retries the failed load and the answer must no longer be + // rag_not_installed. The flip stage needs the real rcore.dll from the + // full bundle, so on the lite CI build it is skipped (stage A — the + // rag_not_installed contract — still runs everywhere). + // ----------------------------------------------------------------------- + runTest("T10 late rcore.dll install (lite -> full, no restart)", [&] { + const size_t srcSlash = dllPath.find_last_of(L"\\/"); + const std::wstring srcDir = (srcSlash == std::wstring::npos) + ? std::wstring() : dllPath.substr(0, srcSlash + 1); + + wchar_t tmpBuf[MAX_PATH] = L""; + REQUIRE(::GetTempPathW(MAX_PATH, tmpBuf) != 0, "GetTempPath succeeded"); + if (g_fatal) return; + const std::wstring dir = std::wstring(tmpBuf) + L"http1c_smoke_t10_" + + std::to_wstring(::GetCurrentProcessId()) + L"\\"; + ::CreateDirectoryW(dir.c_str(), nullptr); + + // Force the lite layout: component DLL only. A stale rcore.dll left by + // a previous run (PID reuse) would defeat stage A — remove it first. + ::DeleteFileW((dir + L"rcore.dll").c_str()); + REQUIRE(::CopyFileW(dllPath.c_str(), (dir + L"libhttp1cWin.dll").c_str(), FALSE), + "copy the component DLL into an rcore-free temp dir"); + if (g_fatal) return; + + Harness h2; // separate module path => separate statics => fresh RCore state + REQUIRE(h2.load(dir + L"libhttp1cWin.dll"), "load the temp copy of the component"); + if (g_fatal) return; + h2.comp = nullptr; + h2.getClassObject(L"HttpServer", &h2.comp); + REQUIRE(h2.comp && h2.comp->setMemManager(&h2.mem) && + h2.comp->Init(static_cast(&h2.conn)), + "init the temp component instance"); + if (g_fatal) return; + + json cfg; + cfg["logging_enabled"] = false; + cfg["tools_json"] = "[]"; + cfg["timeout"] = 2; + CHECK(h2.applyConfig(cfg), "ApplyConfig on the temp instance"); + + const int port2 = pickFreePort(); + std::printf(" temp instance port: %d\n", port2); + tVariant pPort, ret; + setVariantInt(pPort, port2); + const bool listening = h2.callFunc(L"StartListen", &ret, &pPort, 1); + freeVariant(ret); + REQUIRE(listening, "StartListen on the temp instance"); + if (g_fatal) return; + + auto cli = makeClient(port2); + bool up = false; + for (int i = 0; i < 100 && !up; ++i) { + auto res = cli->Get("/health"); + if (res && res->status == 200) up = true; + else std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + REQUIRE(up, "temp instance /health responds 200"); + if (g_fatal) return; + + json initParams = { + {"protocolVersion", "2025-03-26"}, + {"capabilities", json::object()}, + {"clientInfo", {{"name", "http1c_smoke_t10"}, {"version", "1.0"}}} + }; + RpcOptions initOpt; + initOpt.accept = "application/json, text/event-stream"; + auto initRes = mcpPost(*cli, rpcRequest("initialize", 1, initParams), initOpt); + REQUIRE(initRes && initRes->status == 200, "temp instance initialize -> 200"); + if (g_fatal) return; + RpcOptions sess; + sess.sessionId = initRes->get_header_value("Mcp-Session-Id"); + + // tools/call search -> parsed inner payload (the text of content[0]). + auto callSearch = [&](int id, bool& isError) -> json { + json params = { + {"name", "search"}, + {"arguments", {{"query", "ping"}}} + }; + auto res = mcpPost(*cli, rpcRequest("tools/call", id, params), sess); + if (!res || res->status != 200) { isError = true; return json(); } + const json body = json::parse(res->body, nullptr, false); + if (body.is_discarded() || !body.contains("result")) { isError = true; return json(); } + isError = body["result"].value("isError", false); + std::string text; + if (body["result"].contains("content") && !body["result"]["content"].empty()) { + text = body["result"]["content"][0].value("text", std::string()); + } + return json::parse(text, nullptr, false); + }; + + // Stage A — without rcore.dll: structured rag_not_installed naming + // every native tool (so a caller knows the full surface it is missing). + bool isError = false; + json payload = callSearch(2, isError); + CHECK(isError && payload.is_object() && + payload.value("code", std::string()) == "rag_not_installed", + "without rcore.dll native search returns rag_not_installed"); + const std::string msg = payload.is_object() + ? payload.value("message", std::string()) : std::string(); + for (const char* native : {"search", "grep", "get_segment", "list_collections"}) { + CHECK(msg.find(native) != std::string::npos, + std::string("rag_not_installed message names ") + native); + } + + // Stage B — drop rcore.dll next to the RUNNING component and call again: + // the loader must retry and leave lite mode within the same process. + if (fileExists(srcDir + L"rcore.dll")) { + REQUIRE(::CopyFileW((srcDir + L"rcore.dll").c_str(), + (dir + L"rcore.dll").c_str(), FALSE), + "copy rcore.dll next to the running temp component"); + if (fileExists(srcDir + L"DirectML.dll")) { // lazy-loaded by ort; ship alongside + ::CopyFileW((srcDir + L"DirectML.dll").c_str(), + (dir + L"DirectML.dll").c_str(), FALSE); + } + if (!g_fatal) { + payload = callSearch(3, isError); + const std::string code = payload.is_object() + ? payload.value("code", std::string()) : std::string(""); + std::printf(" after install: search answered code=\"%s\" isError=%d\n", + code.c_str(), (int)isError); + CHECK(payload.is_object() && code != "rag_not_installed", + "after copying rcore.dll the loader picks it up without a restart"); + } + } else { + std::printf(" SKIP flip stage: no rcore.dll beside the primary DLL (lite build)\n"); + } + + tVariant none; + tVarInit(&none); + CHECK(h2.callProc(L"StopListen", &none, 0), "temp instance StopListen"); + h2.destroy(); + // Best-effort cleanup: both DLLs stay mapped until process exit (no + // FreeLibrary by design), so deletes may fail — that is fine, the next + // run gets a fresh PID-suffixed dir and scrubs stale files itself. + ::DeleteFileW((dir + L"libhttp1cWin.dll").c_str()); + ::DeleteFileW((dir + L"DirectML.dll").c_str()); + ::DeleteFileW((dir + L"rcore.dll").c_str()); + ::RemoveDirectoryW(dir.c_str()); + }); + + // ----------------------------------------------------------------------- + // Cleanup: stop the listener (joins the server thread + shuts down rcore) + // and destroy the component. The exe must exit on its own. + // ----------------------------------------------------------------------- + std::printf("=== cleanup ===\n"); + if (h.comp) { + tVariant none; + tVarInit(&none); + const bool stopped = h.callProc(L"StopListen", &none, 0); + CHECK(stopped, "StopListen succeeded"); + h.destroy(); + } + + std::printf("\n%d checks, %d failed\n", g_checks, g_failures); + std::printf(g_failures == 0 ? "ALL TESTS PASSED\n" : "TESTS FAILED\n"); + return g_failures > 255 ? 255 : g_failures; +} diff --git a/http-1c-dll/version.h b/http-1c-dll/version.h index c79aba2..eec710f 100644 --- a/http-1c-dll/version.h +++ b/http-1c-dll/version.h @@ -1,7 +1,7 @@ #define VER_FILENAME AddinNative -#define VERSION_FULL 1.4.2.0 -#define VERSION_SEMVER "1.4.2" +#define VERSION_FULL 1.5.0.0 +#define VERSION_SEMVER "1.5.0" #define VERSION_MAJOR 1 -#define VERSION_MINOR 4 -#define VERSION_REVISION 2 +#define VERSION_MINOR 5 +#define VERSION_REVISION 0 #define VERSION_BUILD 0 diff --git a/http-1c-dp/http1c/Forms/Form/Ext/Form.xml b/http-1c-dp/http1c/Forms/Form/Ext/Form.xml index 3c50d02..b6e7e01 100644 --- a/http-1c-dp/http1c/Forms/Form/Ext/Form.xml +++ b/http-1c-dp/http1c/Forms/Form/Ext/Form.xml @@ -17,11 +17,17 @@ Form.Command.GetStatus + ExternalEvent AddInDetachmentOnError + OnOpen @@ -113,6 +119,115 @@ + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>RAG / Search</v8:content> + </v8:item> + + Vertical + NormalSeparation + + + AlwaysHorizontal + None + + + RagModel + + + RagDevice + + + + + + AlwaysHorizontal + None + + + RagCollection + + + + + + + + AlwaysHorizontal + None + + + RagQuery + + + RagMode + + + + + + RagOutput + true + true + true + 12 + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Corpus sync (vector store)</v8:content> + </v8:item> + + Vertical + NormalSeparation + + + StepsPath + true + + + CorpusSources + + + CorpusSources.Path + + + CorpusSources.Collection + + + CorpusSources.Description + + + CorpusSources.BySubfolders + + +
+ + + @@ -251,6 +366,180 @@ xs:boolean + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Model / path</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Device</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Collection</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Query</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Mode (dense/keyword/hybrid)</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>RAG output</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Steps file (steps.json)</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Corpus sources</v8:content> + </v8:item> + + + v8:ValueTable + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Folder</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Collection</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Description (for AI)</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>By subfolders</v8:content> + </v8:item> + + + xs:boolean + + + + @@ -331,5 +620,80 @@ NextScreenshot + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Configure RAG</v8:content> + </v8:item> + + RagConfigure + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Index demo</v8:content> + </v8:item> + + RagIndexDemo + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Stats</v8:content> + </v8:item> + + RagStats + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Search</v8:content> + </v8:item> + + RagSearch + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Clear</v8:content> + </v8:item> + + RagClear + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Sync with vector store</v8:content> + </v8:item> + + + + en + Index the steps file + corpus sources into the vector store (real embedding) + + + SyncVector + + + + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>Install component</v8:content> + </v8:item> + + + + en + Register the http1c component into ExtCompT (shows the platform install dialog), then attach + + + InstallComponent + \ No newline at end of file diff --git a/http-1c-dp/http1c/Forms/Form/Ext/Form/Module.bsl b/http-1c-dp/http1c/Forms/Form/Ext/Form/Module.bsl index c96e1dd..287a7f6 100644 --- a/http-1c-dp/http1c/Forms/Form/Ext/Form/Module.bsl +++ b/http-1c-dp/http1c/Forms/Form/Ext/Form/Module.bsl @@ -57,6 +57,21 @@ Var ScreenshotDataArray; &AtClient Var ScreenshotCurrentIndex; +&AtClient +Var SelfTestCtx; + +&AtClient +Var SelfTestWaitTicks; + +&AtClient +Var SelfTestCfg; + +&AtClient +Var EmbedCtx; + +&AtClient +Var SyncCtx; + #EndRegion @@ -91,7 +106,233 @@ Procedure ExternalEvent(Source, Event, Data) Else Return; EndIf; - + +EndProcedure + +// --------------------------------------------------------------------------- +// OnOpen — prefill the RAG demo fields with sensible defaults. +// --------------------------------------------------------------------------- +&AtClient +Procedure OnOpen(Cancel) + + SelfTestCfg = ParseLaunchConfig(); + + // Trace that OnOpen fired + capture the launch parameter (best-effort). + Try + W = New TextWriter(SelfTestOutFile("onopen-trace.txt"), TextEncoding.UTF8); + W.Write("OnOpen fired " + String(CurrentDate()) + + " build=" + BuildVersion() + Chars.LF + + "LaunchParameter=[" + String(LaunchParameter) + "]"); + W.Close(); + Except + EndTry; + + EnsureRagDefaults(); + + // Attach the component on open (attach-only — no InstallAddIn modal). Shipping + // path attaches the declared template bundle (lite). The headless self-test + // instead attaches the on-disk DLL in ExtCompT IN-PLACE so rcore.dll is found + // beside libhttp1cWin.dll (real search) — its location comes from the launch + // config, never hardcoded. + If SelfTestCfg.selftest Then + AttachSelfTestComponent(0); + Else + AddInPath = GetDefaultAddInSource(); + BeginAttachingAddIn(New NotifyDescription("OnOpenAttachEnd", ThisObject), + AddInPath, "http1c", AddInType.Native); + EndIf; + +EndProcedure + +// Parse "ragselftest;model=;out=;extcompt=" from the launch +// parameter so every environment-specific absolute path stays OUT of the +// processor — the launcher (dev tooling) supplies them. +&AtClient +Function ParseLaunchConfig() + Cfg = New Structure("selftest, model, out, extcompt, perf, embedperf, batch, workers, synctest, steps, corpus, coll", + False, "", "", "", 0, "", 500, 1, False, "", "", ""); + LP = ""; + Try + LP = String(LaunchParameter); + Except + EndTry; + For Each Part In StrSplit(LP, ";", False) Do + Part = TrimAll(Part); + If Lower(Part) = "ragselftest" Then + Cfg.selftest = True; + ElsIf StrStartsWith(Part, "model=") Then + Cfg.model = Mid(Part, StrLen("model=") + 1); + ElsIf StrStartsWith(Part, "out=") Then + Cfg.out = Mid(Part, StrLen("out=") + 1); + ElsIf StrStartsWith(Part, "extcompt=") Then + Cfg.extcompt = Mid(Part, StrLen("extcompt=") + 1); + ElsIf StrStartsWith(Part, "perf=") Then + // perf=N → run the keyword-latency benchmark over an N-segment synthetic + // corpus instead of the normal assert contour. + Try + Cfg.perf = Number(Mid(Part, StrLen("perf=") + 1)); + Except + Cfg.perf = 0; + EndTry; + ElsIf StrStartsWith(Part, "embedperf=") Then + // embedperf= → read every *.feature under , chunk by scenario, + // index with REAL embedding and time the whole embedding build. + Cfg.embedperf = Mid(Part, StrLen("embedperf=") + 1); + ElsIf StrStartsWith(Part, "batch=") Then + Try + Cfg.batch = Number(Mid(Part, StrLen("batch=") + 1)); + Except + Cfg.batch = 500; + EndTry; + ElsIf StrStartsWith(Part, "workers=") Then + // workers=N → number of concurrent bulk-embedding sessions in the core + // (0 = auto ≈ ncpu/2; 1 = single worker; N = exactly N). + Try + Cfg.workers = Number(Mid(Part, StrLen("workers=") + 1)); + Except + Cfg.workers = 1; + EndTry; + ElsIf Lower(Part) = "synctest" Then + // Dev-only headless trigger for the Sync-with-vector button: prefills the + // form fields from steps=/corpus=/coll= and fires SyncVector on open. + Cfg.synctest = True; + ElsIf StrStartsWith(Part, "steps=") Then + Cfg.steps = Mid(Part, StrLen("steps=") + 1); + ElsIf StrStartsWith(Part, "corpus=") Then + Cfg.corpus = Mid(Part, StrLen("corpus=") + 1); + ElsIf StrStartsWith(Part, "coll=") Then + Cfg.coll = Mid(Part, StrLen("coll=") + 1); + EndIf; + EndDo; + Return Cfg; +EndFunction + +&AtClient +Function SelfTestOutDir() + Dir = ""; + Try + If TypeOf(SelfTestCfg) = Type("Structure") And ValueIsFilled(SelfTestCfg.out) Then + Dir = SelfTestCfg.out; + EndIf; + Except + EndTry; + If Not ValueIsFilled(Dir) Then + Dir = TempFilesDir(); + EndIf; + If Right(Dir, 1) <> "\" And Right(Dir, 1) <> "/" Then + Dir = Dir + "\"; + EndIf; + Return Dir; +EndFunction + +&AtClient +Function SelfTestOutFile(Name) + Return SelfTestOutDir() + Name; +EndFunction + +&AtClient +Function SelfTestAttachCandidates() + // In-place attach locations for the headless self-test (from launch config): + // loading the on-disk DLL in ExtCompT puts rcore.dll right beside it. + L = New Array; + If TypeOf(SelfTestCfg) = Type("Structure") And ValueIsFilled(SelfTestCfg.extcompt) Then + L.Add(SelfTestCfg.extcompt); + Sep = StrFind(SelfTestCfg.extcompt, "\", SearchDirection.FromEnd); + If Sep > 1 Then + L.Add(Left(SelfTestCfg.extcompt, Sep - 1)); + EndIf; + EndIf; + Return L; +EndFunction + +&AtClient +Procedure AttachSelfTestComponent(Index) + Cands = SelfTestAttachCandidates(); + If Index >= Cands.Count() Then + TraceLine("all self-test attach candidates failed (no extcompt in launch config?)"); + Return; + EndIf; + AddInPath = Cands[Index]; + TraceLine("self-test attach attempt " + String(Index) + " -> " + AddInPath); + BeginAttachingAddIn(New NotifyDescription("OnOpenAttachEnd", ThisObject, Index), + AddInPath, "http1c", AddInType.Native); +EndProcedure + +&AtClient +Function BuildVersion() + // Bump on EVERY source change so the log proves a fresh .epf is running. + Return "selftest-build-26-async-config"; +EndFunction + +&AtClient +Function Ms() + Return CurrentUniversalDateInMilliseconds(); +EndFunction + +&AtClient +Procedure TraceLine(Text) + Try + W = New TextWriter(SelfTestOutFile("onopen-trace.txt"), TextEncoding.UTF8, , True); + W.WriteLine("" + Text); + W.Close(); + Except + EndTry; +EndProcedure + +&AtClient +Procedure OnOpenAttachEnd(Connected, AdditionalParameters) Export + + TraceLine("OnOpenAttachEnd Connected=" + String(Connected) + " source=" + String(AddInPath)); + If Not Connected Then + // Self-test attaches pass the candidate index; try the next location. + If TypeOf(AdditionalParameters) = Type("Number") Then + AttachSelfTestComponent(AdditionalParameters + 1); + EndIf; + Return; + EndIf; + Try + Component = New("AddIn.http1c.HttpServer"); + Except + TraceLine("Component create EXCEPTION: " + ErrorDescription()); + Return; + EndTry; + + // Config via the async ApplyConfig method — property assignment is blocked + // in async-only infobases ("Cannot call synchronous methods on the client"). + // The self-test scheduling continues in OnOpenConfigEnd once config lands. + Try + Component.BeginCallingApplyConfig( + New NotifyDescription("OnOpenConfigEnd", ThisObject), + BuildConfigJson(False)); + Except + TraceLine("ApplyConfig dispatch EXCEPTION: " + ErrorDescription()); + // Still schedule the self-test — logging config is non-essential. + OnOpenConfigEnd(Undefined, Undefined, Undefined); + EndTry; + +EndProcedure + +&AtClient +Procedure OnOpenConfigEnd(ResultCall, ParametersCall, AdditionalParameters) Export + + // If launched for the headless self-test, run the RAG chain now that the + // component is attached and configured. + LP = ""; + Try + LP = Lower(String(LaunchParameter)); + Except + EndTry; + If SelfTestCfg.synctest Then + TraceLine("scheduling Sync_TestTrigger"); + AttachIdleHandler("Sync_TestTrigger", 1, True); + ElsIf ValueIsFilled(SelfTestCfg.embedperf) Then + TraceLine("scheduling RunEmbedPerfDeferred"); + AttachIdleHandler("RunEmbedPerfDeferred", 1, True); + ElsIf StrFind(LP, "ragselftest") > 0 Then + TraceLine("scheduling RunRagSelfTestDeferred"); + AttachIdleHandler("RunRagSelfTestDeferred", 1, True); + EndIf; + EndProcedure #EndRegion @@ -103,13 +344,29 @@ EndProcedure Procedure Connect(Command) EnsureLoggingDefaults(); - + + // Attach-only (no InstallAddIn — it pops a modal install dialog). The + // component ships in the declared template bundle, so attach loads it directly. AddInPath = GetDefaultAddInSource(); - BeginInstallAddIn( - New NotifyDescription("InstallAddInEnd", ThisObject), - AddInPath); - + BeginAttachingAddIn( + New NotifyDescription("AttachAddInEnd", ThisObject), + AddInPath, "http1c", AddInType.Native); + +EndProcedure + +// Explicit, USER-INITIATED install: registers the http1c component into the +// platform's ExtCompT (the standard `BeginInstallAddIn` flow that shows the +// install dialog), then attaches. Restored under its own button — kept OFF the +// open/Connect path so nothing pops a modal automatically (the attach-only +// default stays). Installs the component shipped as the .epf template; once +// registered it persists in ExtCompT across sessions. +&AtClient +Procedure InstallComponent(Command) + + AddInPath = GetDefaultAddInSource(); + BeginInstallAddIn(New NotifyDescription("InstallAddInEnd", ThisObject), AddInPath); + EndProcedure &AtClient @@ -387,28 +644,30 @@ Procedure AttachAddInEnd(Connected, AdditionalParameters) Export ResetRuntimeStatus(); EnsureLoggingDefaults(); - - // All configuration via synchronous property assignments. + + // All configuration (logging/timeout + MCP tool/resource/prompt catalogs) + // goes through the single async ApplyConfig method. Synchronous property + // assignment is forbidden in async-only infobases. StartListen continues in + // the AttachConfigEnd callback. Try - Component.LoggingEnabled = EnableLogging; - Component.LogPath = LogPath; - Component.Timeout = 120; + Component.BeginCallingApplyConfig( + New NotifyDescription("AttachConfigEnd", ThisObject), + BuildConfigJson(True)); Except ShowMessageBox(, "Configuration failed: " + ErrorDescription()); - Return; EndTry; - - // Register MCP primitives (synchronous property assignments). - RegisterMCPTools(); - RegisterMCPResources(); - RegisterMCPPrompts(); - + +EndProcedure + +&AtClient +Procedure AttachConfigEnd(ResultCall, ParametersCall, AdditionalParameters) Export + // Start listening (async — returns via callback). PortValue = Port; If PortValue = 0 Then PortValue = 8888; EndIf; - + Try Component.BeginCallingStartListen( New NotifyDescription("StartListenEnd", ThisObject), @@ -416,7 +675,7 @@ Procedure AttachAddInEnd(Connected, AdditionalParameters) Export Except ShowMessageBox(, "StartListen failed: " + ErrorDescription()); EndTry; - + EndProcedure &AtClient @@ -469,16 +728,19 @@ EndProcedure // // To add a new tool: // 1. Create a ToolXxx() function that returns the tool definition -// 2. Add it to the Tools array in RegisterMCPTools() +// 2. Add it to the Tools array in McpToolsJson() // 3. Add a handler in ProcessToolCall() dispatcher // 4. Implement HandleXxx() procedure with Begin* callbacks // ============================================================================ #Region ToolDefinitions +// Returns the MCP tool catalog as a JSON array string. Applied to the +// component via the async ApplyConfig method (property assignment is forbidden +// in async-only infobases). &AtClient -Procedure RegisterMCPTools() - +Function McpToolsJson() + Tools = New Array; Tools.Add(ToolGetStatus()); Tools.Add(ToolOpenForm()); @@ -487,10 +749,10 @@ Procedure RegisterMCPTools() Tools.Add(ToolRunLongTask()); Tools.Add(ToolTakeScreenshot()); Tools.Add(ToolTestScreenshot()); - - Component.Tools = SerializeToJson(Tools); - -EndProcedure + + Return SerializeToJson(Tools); + +EndFunction &AtClient Function NewTool(Name, Description) @@ -807,15 +1069,15 @@ EndProcedure // the resource content via SendResponse(). // // To add a new resource: -// 1. Add a resource definition to RegisterMCPResources() +// 1. Add a resource definition to McpResourcesJson() // 2. Add a handler in ProcessResourceRead() dispatcher // ============================================================================ #Region ResourceDefinitions &AtClient -Procedure RegisterMCPResources() - +Function McpResourcesJson() + Resources = New Array; // Example: expose 1C metadata catalog list as a resource @@ -835,10 +1097,10 @@ Procedure RegisterMCPResources() "JSON list of all document metadata objects in the current 1C infobase."); Resource.Insert("mimeType", "application/json"); Resources.Add(Resource); - - Component.Resources = SerializeToJson(Resources); - -EndProcedure + + Return SerializeToJson(Resources); + +EndFunction #EndRegion @@ -863,15 +1125,15 @@ EndProcedure // ExternalEvent. The handler returns a messages array per MCP spec. // // To add a new prompt: -// 1. Add a prompt definition to RegisterMCPPrompts() +// 1. Add a prompt definition to McpPromptsJson() // 2. Add a handler in ProcessPromptGet() dispatcher // ============================================================================ #Region PromptDefinitions &AtClient -Procedure RegisterMCPPrompts() - +Function McpPromptsJson() + Prompts = New Array; // Example: a prompt template for analyzing 1C data @@ -899,10 +1161,10 @@ Procedure RegisterMCPPrompts() PromptArgs.Add(Arg); Prompt.Insert("arguments", PromptArgs); Prompts.Add(Prompt); - - Component.Prompts = SerializeToJson(Prompts); - -EndProcedure + + Return SerializeToJson(Prompts); + +EndFunction #EndRegion @@ -1640,13 +1902,11 @@ Function GetMetadataSummaryOnServer() For Each Doc In Metadata.Documents Do Lines.Add(" - Document." + Doc.Name + " (" + String(Doc.Synonym) + ")"); EndDo; - - Result = ""; - For Each Line In Lines Do - Result = Result + Line + Chars.LF; - EndDo; - Return Result; - + + // One StrConcat over the ready array instead of +-in-a-loop (which reallocates + // the whole growing string on every iteration). Trailing LF preserved. + Return StrConcat(Lines, Chars.LF) + Chars.LF; + EndFunction &AtClient @@ -1779,22 +2039,52 @@ Procedure EnsureLoggingDefaults() EndProcedure +// Build the component configuration payload for the async ApplyConfig method. +// IncludeMcp adds the tool/resource/prompt catalogs (only needed when starting +// the MCP listener). All component config goes through this single async path +// because async-only infobases forbid synchronous property assignment. +&AtClient +Function BuildConfigJson(IncludeMcp) + + EnsureLoggingDefaults(); + + Cfg = New Structure; + Cfg.Insert("logging_enabled", EnableLogging = True); + Cfg.Insert("log_path", LogPath); + Cfg.Insert("timeout", 120); + + If IncludeMcp Then + Cfg.Insert("tools_json", McpToolsJson()); + Cfg.Insert("resources_json", McpResourcesJson()); + Cfg.Insert("prompts_json", McpPromptsJson()); + EndIf; + + Return SerializeToJson(Cfg); + +EndFunction + &AtClient Procedure ApplyLoggingSettings() - + EnsureLoggingDefaults(); - + If Component = Undefined Then Return; EndIf; - + Try - Component.LoggingEnabled = EnableLogging; - Component.LogPath = LogPath; + Component.BeginCallingApplyConfig( + New NotifyDescription("ApplyLoggingSettingsEnd", ThisObject), + BuildConfigJson(False)); Except ShowMessageBox(, "ConfigureLogging failed: " + ErrorDescription()); EndTry; - + +EndProcedure + +&AtClient +Procedure ApplyLoggingSettingsEnd(ResultCall, ParametersCall, AdditionalParameters) Export + // Fire-and-forget: logging config applied component-side. Nothing to do. EndProcedure &AtClient @@ -1807,12 +2097,14 @@ Procedure BuildCombinedStatusJSON(Callback) If Component <> Undefined Then Context = New Structure("Callback,StatusPayload", Callback, StatusPayload); + // Async status read — the Status property is unreadable in async-only + // infobases, so we use the GetStatus method instead. Try - StatusJson = Component.Status; + Component.BeginCallingGetStatus( + New NotifyDescription("BuildCombinedStatusGotStatus", ThisObject, Context)); Except - StatusJson = ""; + BuildCombinedStatusEnd("", Context); EndTry; - BuildCombinedStatusEnd(StatusJson, Context); Else StatusPayload.Insert("componentStatus", New Structure("running", False)); ExecuteNotifyProcessing(Callback, SerializeToJson(StatusPayload)); @@ -1820,9 +2112,20 @@ Procedure BuildCombinedStatusJSON(Callback) EndProcedure +&AtClient +Procedure BuildCombinedStatusGotStatus(ResultJson, ParametersCall, Context) Export + + StatusJson = ResultJson; + If StatusJson = Undefined Then + StatusJson = ""; + EndIf; + BuildCombinedStatusEnd(StatusJson, Context); + +EndProcedure + &AtClient Procedure BuildCombinedStatusEnd(ResultCall, AdditionalParameters) Export - + AdditionalParameters.StatusPayload.Insert("componentStatus", ParseJsonArgument(ResultCall, ResultCall)); ExecuteNotifyProcessing(AdditionalParameters.Callback, SerializeToJson(AdditionalParameters.StatusPayload)); @@ -1836,7 +2139,7 @@ Function GetDefaultAddInSource() Tmp = Obj.GetTemplate("http1c"); Addr = PutToTempStorage(Tmp, UUID); Return Addr; - + EndFunction #EndRegion @@ -2028,6 +2331,1452 @@ EndProcedure #EndRegion +// ============================================================================ +// RAG / SEARCH DEMO +// ============================================================================ +// +// Drives the Rust search core (rcore.dll) from 1C via the component's +// RagDispatch(method, payloadJson) method. Ingest/admin calls (configure, +// index_segments, stats, search, delete_collection) are issued from here; the +// search/grep/get_segment/list_collections MCP tools are served by the +// component itself (no 1C round-trip), so once data is indexed an MCP client +// can search it too. +// +// End-to-end test flow: +// 1. Connect - load the native component. +// 2. Configure RAG - select the e5 model (needs the FULL package + a +// staged model; the lite component answers +// rag_not_installed). +// 3. Index demo data - this base's catalog + document metadata as segments. +// 4. Stats - watch vector_status go building -> ready. +// 5. Search - dense / keyword / hybrid. +// ============================================================================ + +#Region RAGDemo + +&AtClient +Procedure EnsureRagDefaults() + + If Port = 0 Then + Port = 8888; + EndIf; + If Not ValueIsFilled(RagModel) Then + // Builtin model name (no hardcoded path). For an offline/local model put a + // directory path here; the headless self-test takes its model_path from the + // launch config instead. + RagModel = "multilingual-e5-small"; + EndIf; + If Not ValueIsFilled(RagDevice) Then + RagDevice = "auto"; + EndIf; + If Not ValueIsFilled(RagCollection) Then + RagCollection = "metadata"; + EndIf; + If Not ValueIsFilled(RagMode) Then + RagMode = "hybrid"; + EndIf; + +EndProcedure + +// ---- Command handlers ---- + +&AtClient +Procedure RagConfigure(Command) + + EnsureRagDefaults(); + + Payload = New Structure; + If RagModelLooksLikePath(RagModel) Then + Payload.Insert("model_path", RagModel); + Else + Payload.Insert("model", RagModel); + EndIf; + Payload.Insert("device", RagDevice); + + RagCall("configure", SerializeToJson(Payload), "RagConfigureEnd"); + +EndProcedure + +&AtClient +Procedure RagConfigureEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + ShowRagResult("configure", ResultJson); + +EndProcedure + +&AtClient +Procedure RagIndexDemo(Command) + + EnsureRagDefaults(); + + PayloadJson = BuildMetadataSegmentsJSON(RagCollection); + RagCall("index_segments", PayloadJson, "RagIndexDemoEnd"); + +EndProcedure + +&AtClient +Procedure RagIndexDemoEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + ShowRagResult("index_segments", ResultJson); + // Poll embedding progress and show a progress indicator in the form until the + // background worker has embedded every segment (vector_status = ready). + AttachIdleHandler("RagIndexPollTick", 1, True); + +EndProcedure + +&AtClient +Procedure RagIndexPollTick() Export + + RagCall("stats", "{}", "RagIndexPollEnd"); + +EndProcedure + +&AtClient +Procedure RagIndexPollEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + Emb = 0; + Total = 0; + VecStatus = ""; + Try + R = New JSONReader; + R.SetString(ResultJson); + Obj = ReadJSON(R, True); + Coll = Obj["result"]["collections"][RagCollection]; + If Coll <> Undefined Then + Emb = Coll["embedded"]; + Total = Coll["n_segments"]; + VecStatus = Coll["vector_status"]; + EndIf; + Except + EndTry; + + ShowEmbedProgress("Эмбеддинг векторов: " + RagCollection, Emb, Total); + + If VecStatus <> "ready" Then + AttachIdleHandler("RagIndexPollTick", 1, True); + Else + Status(); // clear the progress indicator + RagOutput = "Готово: эмбеддинг " + String(Emb) + " из " + String(Total) + " (100%)"; + EndIf; + +EndProcedure + +&AtClient +Procedure RagStats(Command) + + RagCall("stats", "{}", "RagStatsEnd"); + +EndProcedure + +// "Synchronize with the vector store": index the steps file + every corpus +// source row (real embedding) into named collections, each carrying a +// description (so an AI client can later list collections + descriptions and +// search a subset). Async (this config forbids sync component calls). Paths are +// taken from the form — nothing is prefilled or auto-loaded. +&AtClient +Procedure SyncVector(Command) + + If Component = Undefined Then + RagOutput = "Компонента не подключена. Нажмите Connect (нужна полная компонента с rcore.dll)."; + Return; + EndIf; + + Groups = BuildSyncGroups(); + Total = 0; + Names = New Array; + For Each G In Groups Do + Total = Total + G.segments.Count(); + Names.Add(G.collection); + EndDo; + If Total = 0 Then + RagOutput = "Нечего синхронизировать: укажите путь к шагам и/или строки таблицы корпуса (с существующими папками)."; + Return; + EndIf; + + SyncCtx = New Structure; + SyncCtx.Insert("Groups", Groups); + SyncCtx.Insert("GIndex", 0); + SyncCtx.Insert("Pos", 0); + SyncCtx.Insert("Batch", 500); + SyncCtx.Insert("Total", Total); + SyncCtx.Insert("CollNames", Names); + SyncCtx.Insert("T0", Ms()); + + RagOutput = "Синхронизация: " + String(Total) + " сегментов → " + String(Groups.Count()) + + " коллекций (" + StrConcat(Names, ", ") + "). Конфигурирую модель..."; + + Cfg = New Structure; + Cfg.Insert("model_path", RagModel); + Cfg.Insert("device", ?(ValueIsFilled(RagDevice), RagDevice, "auto")); + Cfg.Insert("embed_workers", 1); + Try + Component.BeginCallingRagDispatch( + New NotifyDescription("Sync_ConfigureEnd", ThisObject), "configure", SerializeToJson(Cfg)); + Except + RagOutput = "FAIL configure dispatch: " + ErrorDescription(); + EndTry; + +EndProcedure + +&AtClient +Procedure Sync_ConfigureEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + If StrFind(ResultJson, """ok"":true") = 0 Then + RagOutput = "FAIL configure (нужна полная компонента с rcore.dll): " + Left(ResultJson, 300); + Return; + EndIf; + SyncCtx.T0 = Ms(); + Sync_SubmitNext(); + +EndProcedure + +&AtClient +Procedure Sync_SubmitNext() + + // Skip past any finished groups. + While SyncCtx.GIndex < SyncCtx.Groups.Count() Do + G = SyncCtx.Groups[SyncCtx.GIndex]; + If SyncCtx.Pos < G.segments.Count() Then + Break; + EndIf; + SyncCtx.GIndex = SyncCtx.GIndex + 1; + SyncCtx.Pos = 0; + EndDo; + + If SyncCtx.GIndex >= SyncCtx.Groups.Count() Then + SelfTestWaitTicks = 0; + AttachIdleHandler("Sync_Tick", 1, True); + Return; + EndIf; + + G = SyncCtx.Groups[SyncCtx.GIndex]; + Upper = SyncCtx.Pos + SyncCtx.Batch - 1; + If Upper > G.segments.Count() - 1 Then + Upper = G.segments.Count() - 1; + EndIf; + Slice = New Array; + For i = SyncCtx.Pos To Upper Do + Slice.Add(G.segments[i]); + EndDo; + SyncCtx.Pos = Upper + 1; + + // The collection's description rides on the doc "name" (and is what a later + // list-collections surfaces to an AI client). + DocId = G.collection + "-batch-" + String(SyncCtx.Pos); + Payload = SegmentsPayload(G.collection, DocId, G.collection, Slice, G.description); + Try + Component.BeginCallingRagDispatch( + New NotifyDescription("Sync_BatchEnd", ThisObject), "index_segments", Payload); + Except + RagOutput = "FAIL index dispatch: " + ErrorDescription(); + EndTry; + +EndProcedure + +&AtClient +Procedure Sync_BatchEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + Sync_SubmitNext(); + +EndProcedure + +&AtClient +Procedure Sync_Tick() Export + + Component.BeginCallingRagDispatch( + New NotifyDescription("Sync_StatsEnd", ThisObject), "stats", "{}"); + +EndProcedure + +&AtClient +Procedure Sync_StatsEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + SelfTestWaitTicks = SelfTestWaitTicks + 1; + EmbDone = 0; + AllReady = True; + Lines = New Array; + Try + R = New JSONReader; + R.SetString(ResultJson); + Obj = ReadJSON(R, True); + Colls = Obj["result"]["collections"]; + For Each Name In SyncCtx.CollNames Do + C = ?(Colls = Undefined, Undefined, Colls[Name]); + If C = Undefined Then + AllReady = False; + Continue; + EndIf; + Emb = C["embedded"]; + NSeg = C["n_segments"]; + St = C["vector_status"]; + EmbDone = EmbDone + Emb; + If St <> "ready" Then + AllReady = False; + EndIf; + Lines.Add(" " + Name + ": " + String(Emb) + "/" + String(NSeg) + " (" + St + ")"); + EndDo; + Except + EndTry; + + ShowEmbedProgress("Эмбеддинг корпуса", EmbDone, SyncCtx.Total); + Head = "Синхронизация (" + String(Ms() - SyncCtx.T0) + " ms):"; + RagOutput = Head + Chars.LF + StrConcat(Lines, Chars.LF); + + If Not AllReady And SelfTestWaitTicks < 6000 Then + AttachIdleHandler("Sync_Tick", 1, True); + Return; + EndIf; + + RagOutput = RagOutput + Chars.LF + "ГОТОВО за " + String(Ms() - SyncCtx.T0) + + " ms — коллекции готовы к поиску."; + +EndProcedure + +// Build the list of {collection, description, segments[]} groups from the form: +// the steps file (one "steps" collection) + each corpus-source row (one +// collection, or one per subfolder when "By subfolders" is set). +&AtClient +Function BuildSyncGroups() + + Groups = New Array; + + If ValueIsFilled(StepsPath) Then + StepSegs = ReadVanessaSteps(StepsPath); + If StepSegs.Count() > 0 Then + Groups.Add(New Structure("collection, description, segments", + "steps", "Шаги Vanessa — определения шагов Gherkin (ИмяШага/ОписаниеШага)", StepSegs)); + EndIf; + EndIf; + + For Each Row In CorpusSources Do + If Not ValueIsFilled(Row.Path) Then + Continue; + EndIf; + Coll = ?(ValueIsFilled(Row.Collection), Row.Collection, "corpus"); + If Row.BySubfolders Then + For Each Sub In FindSubfolders(Row.Path) Do + Segs = ReadFeatureScenarios(Sub.FullName); + If Segs.Count() > 0 Then + Descr = ?(ValueIsFilled(Row.Description), Row.Description + " / " + Sub.Name, Sub.Name); + Groups.Add(New Structure("collection, description, segments", + Coll + "_" + Sub.Name, Descr, Segs)); + EndIf; + EndDo; + Else + Segs = ReadFeatureScenarios(Row.Path); + If Segs.Count() > 0 Then + Groups.Add(New Structure("collection, description, segments", Coll, Row.Description, Segs)); + EndIf; + EndIf; + EndDo; + + Return Groups; + +EndFunction + +// Read Vanessa step definitions from a steps.json array +// [{ИмяШага, ОписаниеШага, ПолныйТипШага}, ...] into segment structures. +&AtClient +Function ReadVanessaSteps(StepsFile) + + Segments = New Array; + Data = Undefined; + Try + F = New File(StepsFile); + If Not F.Exists() Then + Return Segments; + EndIf; + R = New JSONReader; + R.OpenFile(StepsFile); + Data = ReadJSON(R, True); + R.Close(); + Except + Return Segments; + EndTry; + If TypeOf(Data) <> Type("Array") Then + Return Segments; + EndIf; + For Each St In Data Do + Name = GetArg(St, "ИмяШага"); + Descr = GetArg(St, "ОписаниеШага"); + StepType = GetArg(St, "ПолныйТипШага"); + If Not ValueIsFilled(Name) Then + Continue; + EndIf; + Seg = New Structure; + Seg.Insert("text", Name); + Seg.Insert("embed_text", ?(ValueIsFilled(Descr), Name + " " + Descr, Name)); + Seg.Insert("meta", New Structure("type, stepType", "step", String(StepType))); + Segments.Add(Seg); + EndDo; + Return Segments; + +EndFunction + +&AtClient +Function FindSubfolders(Dir) + + Result = New Array; + Try + For Each F In FindFiles(Dir, "*", False) Do + If F.IsDirectory() Then + Result.Add(F); + EndIf; + EndDo; + Except + EndTry; + Return Result; + +EndFunction + +// Dev-only headless test of the Sync button: prefill the form fields from the +// launch config and fire SyncVector, then mirror RagOutput to the result file so +// the headless runner can observe progress + DONE. +&AtClient +Procedure Sync_TestTrigger() Export + + StepsPath = SelfTestCfg.steps; + If ValueIsFilled(SelfTestCfg.corpus) Then + Row = CorpusSources.Add(); + Row.Path = SelfTestCfg.corpus; + Row.Collection = ?(ValueIsFilled(SelfTestCfg.coll), SelfTestCfg.coll, "corpus"); + Row.Description = "synctest corpus"; + Row.BySubfolders = False; + EndIf; + RagModel = SelfTestCfg.model; + SyncVector(Undefined); + AttachIdleHandler("Sync_TestDump", 2, True); + +EndProcedure + +&AtClient +Procedure Sync_TestDump() Export + + Try + W = New TextWriter(SelfTestOutFile("ragselftest-result.txt"), TextEncoding.UTF8, Chars.LF, False); + W.WriteLine("build=" + BuildVersion() + " SYNCTEST"); + W.WriteLine(RagOutput); + W.Close(); + Except + EndTry; + If StrFind(RagOutput, "FAIL") > 0 Then + Try + W = New TextWriter(SelfTestOutFile("ragselftest-result.txt"), TextEncoding.UTF8, Chars.LF, True); + W.WriteLine("DONE"); + W.Close(); + Except + EndTry; + Return; + EndIf; + If StrFind(RagOutput, "ГОТОВО") > 0 Then + // End-to-end proof: read back the collection registry (name + description). + Component.BeginCallingRagDispatch( + New NotifyDescription("Sync_TestListEnd", ThisObject), "list_collections", "{}"); + Return; + EndIf; + AttachIdleHandler("Sync_TestDump", 2, True); + +EndProcedure + +&AtClient +Procedure Sync_TestListEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + Try + W = New TextWriter(SelfTestOutFile("ragselftest-result.txt"), TextEncoding.UTF8, Chars.LF, True); + W.WriteLine("list_collections -> " + Left(ResultJson, 800)); + W.WriteLine("DONE"); + W.Close(); + Except + EndTry; + +EndProcedure + +&AtClient +Procedure RagStatsEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + ShowRagResult("stats", ResultJson); + +EndProcedure + +&AtClient +Procedure RagSearch(Command) + + EnsureRagDefaults(); + + If Not ValueIsFilled(RagQuery) Then + ShowMessageBox(, "Enter a search query first."); + Return; + EndIf; + + Payload = New Structure; + Payload.Insert("query", RagQuery); + Payload.Insert("collection", RagCollection); + Payload.Insert("mode", RagMode); + Payload.Insert("k", 10); + Payload.Insert("include_text", True); + + RagCall("search", SerializeToJson(Payload), "RagSearchEnd"); + +EndProcedure + +&AtClient +Procedure RagSearchEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + ShowRagResult("search", ResultJson); + +EndProcedure + +&AtClient +Procedure RagClear(Command) + + EnsureRagDefaults(); + + Payload = New Structure("collection", RagCollection); + RagCall("delete_collection", SerializeToJson(Payload), "RagClearEnd"); + +EndProcedure + +&AtClient +Procedure RagClearEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + ShowRagResult("delete_collection", ResultJson); + +EndProcedure + +// ---- Helpers ---- + +&AtClient +Function RagModelLooksLikePath(Value) + + Return StrFind(Value, "\") > 0 Or StrFind(Value, "/") > 0 Or StrFind(Value, ":") > 0; + +EndFunction + +&AtClient +Procedure RagCall(Method, PayloadJson, CallbackName) + + If Component = Undefined Then + ShowMessageBox(, "Component is not connected. Press Connect first."); + Return; + EndIf; + + Try + Component.BeginCallingRagDispatch( + New NotifyDescription(CallbackName, ThisObject), + Method, PayloadJson); + Except + ShowMessageBox(, "RagDispatch('" + Method + "') failed: " + ErrorDescription()); + EndTry; + +EndProcedure + +&AtClient +Procedure ShowEmbedProgress(Stage, Emb, Total) + + Pct = ?(Total > 0, Int(Emb * 100 / Total), 0); + Descr = String(Emb) + " из " + String(Total) + " (" + String(Pct) + "%)"; + // Native 1C progress indicator: message + 0..100 bar + explanatory text. + Status(Stage, Pct, Descr); + // Also mirror into the visible output field on the form. + RagOutput = Stage + ": " + Descr; + +EndProcedure + +&AtClient +Procedure ShowRagResult(Label, ResultJson) + + Envelope = ParseJsonArgument(ResultJson, Undefined); + + Lines = New Array; + Lines.Add("=== " + Label + " ==="); + + If Envelope = Undefined Then + Lines.Add("(unparseable response)"); + Lines.Add(Left(ResultJson, 4000)); + RagOutput = JoinRagLines(Lines); + Return; + EndIf; + + If GetArg(Envelope, "ok") = True Then + ResultValue = GetArg(Envelope, "result"); + Hits = Undefined; + If TypeOf(ResultValue) = Type("Map") Then + Hits = ResultValue["hits"]; + EndIf; + + If Hits <> Undefined Then + Lines.Add("hits: " + String(Hits.Count())); + For Each Hit In Hits Do + DocId = GetArg(Hit, "doc_id"); + Score = GetArg(Hit, "score"); + Text = GetArg(Hit, "text"); + Snippet = ""; + If Text <> Undefined Then + Snippet = Left(StrReplace(String(Text), Chars.LF, " "), 100); + EndIf; + ScoreStr = ?(Score = Undefined, "", Format(Score, "NFD=4; NG=0")); + Lines.Add(" [" + ScoreStr + "] " + String(DocId) + " — " + Snippet); + EndDo; + Else + Lines.Add("OK: " + SerializeToJson(ResultValue)); + EndIf; + Else + ErrorObj = GetArg(Envelope, "error"); + Lines.Add("ERROR [" + String(GetArg(ErrorObj, "code")) + "]: " + + String(GetArg(ErrorObj, "message"))); + EndIf; + + RagOutput = JoinRagLines(Lines); + +EndProcedure + +&AtClient +Function JoinRagLines(Lines) + + // StrConcat over the ready array instead of +-in-a-loop (O(N) vs O(N^2)). + If Lines.Count() = 0 Then + Return ""; + EndIf; + Return StrConcat(Lines, Chars.LF) + Chars.LF; + +EndFunction + +&AtServer +Function BuildMetadataSegmentsJSON(Collection) + + // Index this base's catalog + document metadata as searchable segments + // (a single doc_id "metadata" with many segments). Capped so a very large + // configuration does not build an enormous payload for a demo. + MaxSegments = 500; + + Segments = New Array; + + For Each Cat In Metadata.Catalogs Do + If Segments.Count() >= MaxSegments Then + Break; + EndIf; + Segments.Add(New Structure("text", + "Справочник " + Cat.Name + " — " + String(Cat.Synonym))); + EndDo; + + For Each Doc In Metadata.Documents Do + If Segments.Count() >= MaxSegments Then + Break; + EndIf; + Segments.Add(New Structure("text", + "Документ " + Doc.Name + " — " + String(Doc.Synonym))); + EndDo; + + Payload = New Structure; + Payload.Insert("collection", Collection); + Payload.Insert("doc_id", "metadata"); + Payload.Insert("name", "1C metadata (catalogs + documents)"); + Payload.Insert("segments", Segments); + + JSONWriter = New JSONWriter; + JSONWriter.SetString(); + WriteJSON(JSONWriter, Payload); + Return JSONWriter.Close(); + +EndFunction + +#EndRegion + + +#Region RAGSelfTest + +// Headless self-test driven by the /C"ragselftest" launch parameter (see OnOpen). +// Synchronously attaches the component and round-trips RagDispatch, writing the +// envelopes to rust-core/target/ragselftest-result.txt. The launcher reads that +// file and terminates the 1C process it itself started (by its own PID). + +// Async full-chain self-test (this config forbids synchronous component calls). +// attach-only: the component is already cached in ExtCompT (the launcher stages +// libhttp1cWin.dll + rcore.dll + DirectML.dll there), so no InstallAddIn is +// needed and rcore.dll loaded beside the component enables real search. +// Flow: attach -> configure(local e5 model) -> index steps.json -> wait for +// embedding -> hybrid search. Each step is logged; the launcher waits for "DONE". + +&AtClient +Procedure RunRagSelfTestDeferred() + + Ctx = New Structure; + Ctx.Insert("Log", New Array); + Ctx.Insert("CaseIndex", 0); + Ctx.Insert("Pass", 0); + Ctx.Insert("Fail", 0); + SelfTestAppend(Ctx, "STARTED " + String(CurrentDate()) + " build=" + BuildVersion()); + + If Component = Undefined Then + SelfTestAppend(Ctx, "FAIL: component not attached on open"); + SelfTestAppend(Ctx, "RESULT: FAIL (0/0)"); + SelfTestAppend(Ctx, "DONE"); + Return; + EndIf; + + // Build the stub-adapter test cases (catch any builder error instead of + // letting an idle-handler exception pop a blocking modal). + Try + Ctx.Insert("Cases", SelfTestCases()); + Except + SelfTestAppend(Ctx, "FAIL: building test cases -> " + ErrorDescription()); + SelfTestAppend(Ctx, "RESULT: FAIL (0/0)"); + SelfTestAppend(Ctx, "DONE"); + Return; + EndTry; + + // Async GetStatus probe — proves the async config/status methods work and + // that the loaded DLL is the new build (no synchronous property access). + Try + Component.BeginCallingGetStatus( + New NotifyDescription("SelfTest_StatusEnd", ThisObject, Ctx)); + Except + SelfTestAppend(Ctx, "FAIL: GetStatus dispatch -> " + ErrorDescription()); + SelfTestAppend(Ctx, "RESULT: FAIL (0/" + String(Ctx.Cases.Count()) + ")"); + SelfTestAppend(Ctx, "DONE"); + EndTry; + +EndProcedure + +&AtClient +Procedure SelfTest_StatusEnd(StatusJson, ParametersCall, Ctx) Export + + // Hard gate (not a counted case): if the async GetStatus method is missing + // or returns junk, the loaded component is wrong — abort before the cases. + Probe = "" + StatusJson; + StatusOk = StrFind(Probe, """version""") > 0 And StrFind(Probe, """running""") > 0; + If StatusOk Then + SelfTestAppend(Ctx, "OK: GetStatus (async) -> " + Left(Probe, 200)); + Else + SelfTestAppend(Ctx, "FAIL: GetStatus (async) returned no status JSON -> " + Left(Probe, 200)); + SelfTestAppend(Ctx, "RESULT: FAIL (0/" + String(Ctx.Cases.Count()) + ")"); + SelfTestAppend(Ctx, "DONE"); + Return; + EndIf; + + // Offline model + device come from the launch config (no hardcoded paths). + // device "auto" = DirectML GPU with automatic CPU fallback. + Cfg = New Structure; + Cfg.Insert("model_path", SelfTestCfg.model); + Cfg.Insert("device", "auto"); + Ctx.Insert("TCfg", Ms()); + Try + Component.BeginCallingRagDispatch( + New NotifyDescription("SelfTest_ConfigureEnd", ThisObject, Ctx), + "configure", SerializeToJson(Cfg)); + Except + SelfTestAppend(Ctx, "FAIL: configure dispatch -> " + ErrorDescription()); + SelfTestAppend(Ctx, "RESULT: FAIL (0/" + String(Ctx.Cases.Count()) + ")"); + SelfTestAppend(Ctx, "DONE"); + EndTry; + +EndProcedure + +&AtClient +Procedure SelfTest_ConfigureEnd(ResultJson, ParametersCall, Ctx) Export + + SelfTestAppend(Ctx, "configure (" + String(Ms() - Ctx.TCfg) + " ms) -> " + Left(ResultJson, 240)); + If StrFind(ResultJson, """ok"":true") = 0 Then + SelfTestAppend(Ctx, "FAIL: configure — real embedder unavailable (rag_not_installed / mock)"); + SelfTestAppend(Ctx, "RESULT: FAIL (0/" + String(Ctx.Cases.Count()) + ")"); + SelfTestAppend(Ctx, "DONE"); + Return; + EndIf; + If StrFind(ResultJson, """dim"":384") = 0 Then + SelfTestAppend(Ctx, "WARN: model dim != 384 (mock fallback?) — semantic asserts may fail"); + EndIf; + SelfTest_RunCase(Ctx); + +EndProcedure + +// --- Generic asserting driver: each case is index -> poll-ready -> query -> assert --- + +&AtClient +Procedure SelfTest_RunCase(Ctx) + + If Ctx.CaseIndex >= Ctx.Cases.Count() Then + SelfTest_Finalize(Ctx); + Return; + EndIf; + C = Ctx.Cases[Ctx.CaseIndex]; + Ctx.Insert("CurCase", C); + Ctx.Insert("CaseSegments", 0); + SelfTestAppend(Ctx, ""); + SelfTestAppend(Ctx, "CASE " + String(Ctx.CaseIndex + 1) + "/" + String(Ctx.Cases.Count()) + + ": " + C.label + " (collection " + C.collection + ")"); + + If Not ValueIsFilled(C.indexMethod) Then + // No (re)index — query an already-indexed collection directly (find_step_usages). + Ctx.Insert("TQuery", Ms()); + Component.BeginCallingRagDispatch( + New NotifyDescription("SelfTest_CaseQueryEnd", ThisObject, Ctx), + C.queryMethod, C.queryPayload); + Return; + EndIf; + + Ctx.Insert("TEmbed", Ms()); + Component.BeginCallingRagDispatch( + New NotifyDescription("SelfTest_CaseIndexEnd", ThisObject, Ctx), + C.indexMethod, C.indexPayload); + +EndProcedure + +&AtClient +Procedure SelfTest_CaseIndexEnd(ResultJson, ParametersCall, Ctx) Export + + SelfTestAppend(Ctx, " " + Ctx.CurCase.indexMethod + " -> " + Left(ResultJson, 160)); + SelfTestCtx = Ctx; + SelfTestWaitTicks = 0; + AttachIdleHandler("SelfTest_CaseTick", 5, True); + +EndProcedure + +&AtClient +Procedure SelfTest_CaseTick() Export + + Component.BeginCallingRagDispatch( + New NotifyDescription("SelfTest_CaseStatsEnd", ThisObject, SelfTestCtx), "stats", "{}"); + +EndProcedure + +&AtClient +Procedure SelfTest_CaseStatsEnd(ResultJson, ParametersCall, Ctx) Export + + SelfTestWaitTicks = SelfTestWaitTicks + 1; + C = Ctx.CurCase; + Emb = 0; + Total = 0; + VecStatus = ""; + Try + R = New JSONReader; + R.SetString(ResultJson); + Obj = ReadJSON(R, True); + Coll = Obj["result"]["collections"][C.collection]; + If Coll <> Undefined Then + Emb = Coll["embedded"]; + Total = Coll["n_segments"]; + VecStatus = Coll["vector_status"]; + EndIf; + Except + EndTry; + Ctx.CaseSegments = Total; + ShowEmbedProgress(C.label, Emb, Total); + + If VecStatus <> "ready" And SelfTestWaitTicks < 60 Then + AttachIdleHandler("SelfTest_CaseTick", 5, True); + Return; + EndIf; + + SelfTestAppend(Ctx, " embedded " + String(Emb) + "/" + String(Total) + + " in " + String(Ms() - Ctx.TEmbed) + " ms; querying"); + Ctx.Insert("TQuery", Ms()); + Component.BeginCallingRagDispatch( + New NotifyDescription("SelfTest_CaseQueryEnd", ThisObject, Ctx), + C.queryMethod, C.queryPayload); + +EndProcedure + +&AtClient +Procedure SelfTest_CaseQueryEnd(ResultJson, ParametersCall, Ctx) Export + + C = Ctx.CurCase; + SelfTestAppend(Ctx, " " + C.queryMethod + " (" + String(Ms() - Ctx.TQuery) + " ms) -> " + + Left(ResultJson, 400)); + + OkPass = (StrFind(ResultJson, """ok"":true") > 0); + TextPass = (Not ValueIsFilled(C.expectText)) Or (StrFind(ResultJson, C.expectText) > 0); + SegPass = (C.expectSegments = 0) Or (Ctx.CaseSegments = C.expectSegments); + CasePass = OkPass And TextPass And SegPass; + + Detail = "ok=" + String(OkPass); + If ValueIsFilled(C.expectText) Then + Detail = Detail + " text<" + C.expectText + ">=" + String(TextPass); + EndIf; + If C.expectSegments > 0 Then + Detail = Detail + " segments(" + String(Ctx.CaseSegments) + "==" + String(C.expectSegments) + + ")=" + String(SegPass); + EndIf; + + If CasePass Then + Ctx.Pass = Ctx.Pass + 1; + SelfTestAppend(Ctx, " PASS: " + C.label + " [" + Detail + "]"); + Else + Ctx.Fail = Ctx.Fail + 1; + SelfTestAppend(Ctx, " FAIL: " + C.label + " [" + Detail + "]"); + EndIf; + + Ctx.CaseIndex = Ctx.CaseIndex + 1; + SelfTest_RunCase(Ctx); + +EndProcedure + +&AtClient +Procedure SelfTest_Finalize(Ctx) + + Total = Ctx.Cases.Count(); + SelfTestAppend(Ctx, ""); + If Ctx.Fail = 0 Then + SelfTestAppend(Ctx, "RESULT: ALL PASS (" + String(Ctx.Pass) + "/" + String(Total) + ")"); + Else + SelfTestAppend(Ctx, "RESULT: FAIL (" + String(Ctx.Pass) + "/" + String(Total) + + " passed, " + String(Ctx.Fail) + " failed)"); + EndIf; + SelfTestAppend(Ctx, "DONE"); + +EndProcedure + +// ============================================================================ +// Embedding benchmark over the whole .feature corpus (launch param +// embedperf=). Reads every *.feature under , chunks BY SCENARIO, +// indexes with REAL embedding in batches, polls until the worker has embedded +// everything, and reports the total embedding-build time + throughput. +// ============================================================================ + +&AtClient +Procedure RunEmbedPerfDeferred() + + EmbedCtx = New Structure; + EmbedCtx.Insert("Log", New Array); + SelfTestAppend(EmbedCtx, "STARTED " + String(CurrentDate()) + " build=" + BuildVersion() + " EMBEDPERF"); + If Component = Undefined Then + SelfTestAppend(EmbedCtx, "FAIL: component not attached on open"); + SelfTestAppend(EmbedCtx, "DONE"); + Return; + EndIf; + + Cfg = New Structure; + Cfg.Insert("model_path", SelfTestCfg.model); + Cfg.Insert("device", "auto"); + Cfg.Insert("embed_workers", SelfTestCfg.workers); + SelfTestAppend(EmbedCtx, "embed_workers requested = " + String(SelfTestCfg.workers) + + " (0=auto~ncpu/2, 1=single, N=exact)"); + EmbedCtx.Insert("TCfg", Ms()); + Try + Component.BeginCallingRagDispatch( + New NotifyDescription("EmbedPerf_ConfigureEnd", ThisObject), + "configure", SerializeToJson(Cfg)); + Except + SelfTestAppend(EmbedCtx, "FAIL: configure dispatch -> " + ErrorDescription()); + SelfTestAppend(EmbedCtx, "DONE"); + EndTry; + +EndProcedure + +&AtClient +Procedure EmbedPerf_ConfigureEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + SelfTestAppend(EmbedCtx, "configure (" + String(Ms() - EmbedCtx.TCfg) + " ms) -> " + Left(ResultJson, 240)); + If StrFind(ResultJson, """ok"":true") = 0 Then + SelfTestAppend(EmbedCtx, "FAIL: configure -> " + Left(ResultJson, 240)); + SelfTestAppend(EmbedCtx, "DONE"); + Return; + EndIf; + + TRead = Ms(); + Segments = ReadFeatureScenarios(SelfTestCfg.embedperf); + EmbedCtx.Insert("Segments", Segments); + EmbedCtx.Insert("Total", Segments.Count()); + EmbedCtx.Insert("Pos", 0); + EmbedCtx.Insert("Batch", ?(SelfTestCfg.batch > 0, SelfTestCfg.batch, 500)); + EmbedCtx.Insert("Collection", "features"); + EmbedCtx.Insert("Batches", 0); + SelfTestAppend(EmbedCtx, "scenarios=" + String(EmbedCtx.Total) + + " read+chunk in " + String(Ms() - TRead) + " ms; batch=" + String(EmbedCtx.Batch)); + + If EmbedCtx.Total = 0 Then + SelfTestAppend(EmbedCtx, "FAIL: no scenarios found under " + SelfTestCfg.embedperf); + SelfTestAppend(EmbedCtx, "DONE"); + Return; + EndIf; + + EmbedCtx.Insert("T0", Ms()); + EmbedPerf_SubmitNextBatch(); + +EndProcedure + +&AtClient +Procedure EmbedPerf_SubmitNextBatch() + + If EmbedCtx.Pos >= EmbedCtx.Total Then + SelfTestAppend(EmbedCtx, "submitted all " + String(EmbedCtx.Total) + " segments in " + + String(EmbedCtx.Batches) + " batches, accept took " + String(Ms() - EmbedCtx.T0) + + " ms; embedding in background..."); + SelfTestWaitTicks = 0; + AttachIdleHandler("EmbedPerf_Tick", 1, True); + Return; + EndIf; + + Upper = EmbedCtx.Pos + EmbedCtx.Batch - 1; + If Upper > EmbedCtx.Total - 1 Then + Upper = EmbedCtx.Total - 1; + EndIf; + Slice = New Array; + For i = EmbedCtx.Pos To Upper Do + Slice.Add(EmbedCtx.Segments[i]); + EndDo; + EmbedCtx.Pos = Upper + 1; + EmbedCtx.Batches = EmbedCtx.Batches + 1; + + Payload = SegmentsPayload(EmbedCtx.Collection, "features-batch-" + String(EmbedCtx.Batches), + "features batch " + String(EmbedCtx.Batches), Slice); + Try + Component.BeginCallingRagDispatch( + New NotifyDescription("EmbedPerf_BatchEnd", ThisObject), + "index_segments", Payload); + Except + SelfTestAppend(EmbedCtx, "FAIL: index_segments dispatch -> " + ErrorDescription()); + SelfTestAppend(EmbedCtx, "DONE"); + EndTry; + +EndProcedure + +&AtClient +Procedure EmbedPerf_BatchEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + If StrFind(ResultJson, """ok"":true") = 0 Then + SelfTestAppend(EmbedCtx, "batch " + String(EmbedCtx.Batches) + " -> " + Left(ResultJson, 200)); + EndIf; + EmbedPerf_SubmitNextBatch(); + +EndProcedure + +&AtClient +Procedure EmbedPerf_Tick() Export + + Component.BeginCallingRagDispatch( + New NotifyDescription("EmbedPerf_StatsEnd", ThisObject), "stats", "{}"); + +EndProcedure + +&AtClient +Procedure EmbedPerf_StatsEnd(ResultJson, ParametersCall, AdditionalParameters) Export + + SelfTestWaitTicks = SelfTestWaitTicks + 1; + Emb = 0; + Failed = 0; + Skipped = 0; + NSeg = 0; + VecStatus = ""; + Try + R = New JSONReader; + R.SetString(ResultJson); + Obj = ReadJSON(R, True); + Coll = Obj["result"]["collections"][EmbedCtx.Collection]; + If Coll <> Undefined Then + Emb = Coll["embedded"]; + Failed = Coll["failed"]; + Skipped = Coll["skipped"]; + NSeg = Coll["n_segments"]; + VecStatus = Coll["vector_status"]; + EndIf; + Except + EndTry; + + ShowEmbedProgress("Embedding features corpus", Emb, EmbedCtx.Total); + + Done = (Emb + Failed + Skipped >= EmbedCtx.Total) Or (VecStatus = "ready"); + If Not Done And SelfTestWaitTicks < 6000 Then + // Periodic progress line (~every 10s) so a long run is observable in the file. + If (SelfTestWaitTicks % 10) = 0 Then + SelfTestAppend(EmbedCtx, " progress: " + String(Emb) + "/" + String(EmbedCtx.Total) + + " embedded, " + String(Ms() - EmbedCtx.T0) + " ms elapsed"); + EndIf; + AttachIdleHandler("EmbedPerf_Tick", 1, True); + Return; + EndIf; + + Elapsed = Ms() - EmbedCtx.T0; + MsPerSeg = ?(Emb > 0, Elapsed / Emb, 0); + Rate = ?(Elapsed > 0, Int(Emb * 1000 / Elapsed), 0); + SelfTestAppend(EmbedCtx, ""); + SelfTestAppend(EmbedCtx, "EMBEDDED " + String(Emb) + "/" + String(EmbedCtx.Total) + + " (failed " + String(Failed) + ", skipped " + String(Skipped) + ", n_segments " + String(NSeg) + + ") in " + String(Elapsed) + " ms"); + SelfTestAppend(EmbedCtx, "throughput = " + String(Rate) + " seg/s, " + + String(Int(MsPerSeg * 100) / 100) + " ms/seg; vector_status=" + VecStatus); + If Not Done Then + SelfTestAppend(EmbedCtx, "WARN: stopped on tick cap (still building)"); + EndIf; + SelfTestAppend(EmbedCtx, "RESULT: EMBEDPERF DONE"); + SelfTestAppend(EmbedCtx, "DONE"); + +EndProcedure + +// Read every *.feature under Dir (recursive), chunk each BY SCENARIO, and return +// an array of segment structures {text, embed_text, meta}. embed_text = the +// scenario text so it is REALLY embedded (this is the whole point of the bench). +&AtClient +Function ReadFeatureScenarios(Dir) + + Segments = New Array; + Files = New Array; + Try + Files = FindFiles(Dir, "*.feature", True); + Except + EndTry; + For Each F In Files Do + If F.IsDirectory() Then + Continue; + EndIf; + Text = ""; + Try + TR = New TextReader(F.FullName, TextEncoding.UTF8); + Text = TR.Read(); + TR.Close(); + Except + Continue; + EndTry; + SplitFeatureIntoScenarios(Text, F.Name, Segments); + EndDo; + Return Segments; + +EndFunction + +&AtClient +Function IsScenarioHeader(TrimmedLine) + Return StrStartsWith(TrimmedLine, "Сценарий:") + Or StrStartsWith(TrimmedLine, "Scenario:") + Or StrStartsWith(TrimmedLine, "Scenario Outline:") + Or StrStartsWith(TrimmedLine, "Структура сценария:"); +EndFunction + +&AtClient +Procedure SplitFeatureIntoScenarios(Text, FileName, Segments) + + Lines = StrSplit(Text, Chars.LF, True); + Current = New Array; + InScenario = False; + For Each Line In Lines Do + Clean = StrReplace(Line, Chars.CR, ""); + If IsScenarioHeader(TrimL(Clean)) Then + If InScenario And Current.Count() > 0 Then + AddScenarioSegment(Segments, Current, FileName); + EndIf; + Current = New Array; + Current.Add(Clean); + InScenario = True; + ElsIf InScenario Then + Current.Add(Clean); + EndIf; + EndDo; + If InScenario And Current.Count() > 0 Then + AddScenarioSegment(Segments, Current, FileName); + EndIf; + +EndProcedure + +&AtClient +Procedure AddScenarioSegment(Segments, CurrentLines, FileName) + + Body = StrConcat(CurrentLines, Chars.LF); + Seg = New Structure; + Seg.Insert("text", Body); + Seg.Insert("embed_text", Body); + Seg.Insert("meta", New Structure("type, feature", "scenario", FileName)); + Segments.Add(Seg); + +EndProcedure + +// ============================================================================ +// Test cases — each exercises one adapter end-to-end and asserts the result. +// ============================================================================ + +&AtClient +Function SelfTestCases() + Cases = New Array; + + // Perf mode (launch param perf=N): benchmark keyword-search latency over an + // N-segment synthetic corpus. The first case indexes the corpus then queries; + // the rest re-query the SAME collection (indexMethod="") so each reports a clean + // per-query latency (the "search (X ms)" line) without re-indexing. embed_text + // is blank → segments are skipped by the embedder, isolating the keyword path. + If SelfTestCfg.perf > 0 Then + N = SelfTestCfg.perf; + Cases.Add(SelfTestCase("PERF index+query N=" + String(N), "perf", "index_segments", + PerfSegmentsPayload("perf", "perf-corpus", N), "search", + SearchJson("alpha", "perf", "keyword", 10), "alpha", 0)); + For i = 1 To 5 Do + Cases.Add(SelfTestCase("PERF query #" + String(i + 1) + " N=" + String(N), "perf", "", "", "search", + SearchJson("alpha", "perf", "keyword", 10), "alpha", 0)); + EndDo; + Return Cases; + EndIf; + + // 1. QA step catalog (anti-hallucination source): canonical phrases + descriptions. + Cases.Add(SelfTestCase("QA step catalog", "qa_steps", "index_segments", + StepCatalogPayload(), "search", + SearchJson("удаление пользователя", "qa_steps", "hybrid", 5), "удаля", 0)); + + // 2. QA scenarios: verbatim text + tags + line addressing; assert the real + // parameter value ("Феррон") is retrievable. + Cases.Add(SelfTestCase("QA scenarios", "qa_scenarios", "index_segments", + ScenariosPayload(), "search", + SearchJson("фильтр по компании", "qa_scenarios", "hybrid", 5), "Феррон", 0)); + + // 3. find_step_usages: reverse step -> scenario with the REAL parameter value + // (keyword scan over scenario text; qa_scenarios already indexed by case 2). + Cases.Add(SelfTestCase("find_step_usages", "qa_scenarios", "", "", "search", + SearchJson("я удаляю пользователя", "qa_scenarios", "keyword", 5), "VanessaUser1", 0)); + + // 4. Products adapter: catalog from a stub array; semantic intent (dense). + Cases.Add(SelfTestCase("Products (semantic)", "products", "index_segments", + ProductsPayload(), "search", + SearchJson("ноутбук", "products", "hybrid", 5), "Lenovo", 0)); + + // 5. Products by exact article/SKU via the keyword channel (already indexed). + Cases.Add(SelfTestCase("Products by article (keyword/SKU)", "products", "", "", "search", + SearchJson("ART-1003", "products", "keyword", 5), "DeLonghi", 0)); + + // 6. Clients adapter + dedup by exact INN: raw list has duplicate INNs; assert + // the indexed segment count equals the unique count (dedup happened). + Unique = ClientsDedup(StubClients()); + Cases.Add(SelfTestCase("Clients (dedup by INN)", "clients", "index_segments", + ClientsPayload(Unique), "search", + SearchJson("Ромашка", "clients", "keyword", 5), "Ромашка", Unique.Count())); + + Return Cases; +EndFunction + +&AtClient +Function SelfTestCase(Label, Collection, IndexMethod, IndexPayload, QueryMethod, QueryPayload, ExpectText, ExpectSegments) + C = New Structure; + C.Insert("label", Label); + C.Insert("collection", Collection); + C.Insert("indexMethod", IndexMethod); + C.Insert("indexPayload", IndexPayload); + C.Insert("queryMethod", QueryMethod); + C.Insert("queryPayload", QueryPayload); + C.Insert("expectText", ExpectText); + C.Insert("expectSegments", ExpectSegments); + Return C; +EndFunction + +&AtClient +Function SearchJson(Query, Collection, Mode, K) + Sp = New Structure; + Sp.Insert("query", Query); + Sp.Insert("collection", Collection); + Sp.Insert("mode", Mode); + Sp.Insert("k", K); + Sp.Insert("include_text", True); + Return SerializeToJson(Sp); +EndFunction + +&AtClient +Function PerfSegmentsPayload(Collection, DocId, Count) + // N synthetic segments. Every segment contains the common token "alpha" (so a + // keyword query for it matches ALL N — the worst case for the old build-a-full- + // Hit-per-match-then-full-sort path) plus a unique token so segments differ. + // embed_text="" marks each skip → no embedding, isolating the keyword channel. + Segments = New Array; + For i = 1 To Count Do + Seg = New Structure; + Seg.Insert("text", "alpha beta gamma segment number " + String(i) + " unique" + String(i)); + Seg.Insert("embed_text", ""); + Segments.Add(Seg); + EndDo; + Return SegmentsPayload(Collection, DocId, "perf corpus", Segments); +EndFunction + +&AtClient +Function SegmentsPayload(Collection, DocId, Name, Segments, Description = "") + Payload = New Structure; + Payload.Insert("collection", Collection); + Payload.Insert("doc_id", DocId); + Payload.Insert("name", Name); + Payload.Insert("segments", Segments); + // Labels the collection in the core (surfaced by list_collections / stats), + // so an AI client can discover collections by description. + If ValueIsFilled(Description) Then + Payload.Insert("collection_description", Description); + EndIf; + W = New JSONWriter; + W.SetString(); + WriteJSON(W, Payload); + Return W.Close(); +EndFunction + +// ============================================================================ +// Pluggable stub data sources (no file reads, no hardcoded corpora). Swap these +// for real adapters (Gherkin1C parse, step registry, product/client catalogs). +// ============================================================================ + +&AtClient +Function StubStep(Phrase, Description, ParamTypes) + Return New Structure("phrase, description, paramTypes", Phrase, Description, ParamTypes); +EndFunction + +&AtClient +Function StubStepCatalog() + S = New Array; + S.Add(StubStep("Я удаляю пользователя ""Имя""", "Удаление пользователя информационной базы по имени", "Строка")); + S.Add(StubStep("Я создаю элемент справочника ""Имя""", "Создание нового элемента справочника", "Строка")); + S.Add(StubStep("Я открываю форму ""Форма""", "Открытие управляемой формы по имени", "Строка")); + S.Add(StubStep("Я нажимаю кнопку ""Кнопка""", "Нажатие командной кнопки на форме", "Строка")); + S.Add(StubStep("Я проверяю фильтр по компании ""Компания""", "Проверка фильтра списка по реквизиту Компания", "Строка")); + S.Add(StubStep("Я провожу документ ""Документ""", "Проведение документа", "Строка")); + Return S; +EndFunction + +&AtClient +Function StubScenario(Name, Feature, Tags, LineStart, LineEnd, StepsText) + Return New Structure("name, feature, tags, lineStart, lineEnd, steps", + Name, Feature, Tags, LineStart, LineEnd, StepsText); +EndFunction + +&AtClient +Function StubScenarios() + S = New Array; + S.Add(StubScenario("Удаление пользователя", "Управление пользователями", "@smoke,@users", 5, 9, + "Дано я авторизован как ""Администратор""" + Chars.LF + + "Когда я удаляю пользователя ""VanessaUser1""" + Chars.LF + + "Тогда пользователь ""VanessaUser1"" отсутствует в базе")); + S.Add(StubScenario("Фильтр по компании в заказе поставщику", "Фильтры документов", "@filters,@regress", 12, 17, + "Дано я открываю список ""Заказ поставщику""" + Chars.LF + + "Когда я проверяю фильтр по компании ""Феррон""" + Chars.LF + + "Тогда в списке только документы компании ""Феррон""")); + S.Add(StubScenario("Проведение приходной накладной", "Складские документы", "@smoke,@warehouse", 20, 24, + "Дано открыта форма ""Приходная накладная""" + Chars.LF + + "Когда я провожу документ ""ПН-0001""" + Chars.LF + + "Тогда документ ""ПН-0001"" проведён")); + Return S; +EndFunction + +&AtClient +Function StubProduct(Name, Brand, Category, Sku, Article, Tags) + Return New Structure("name, brand, category, sku, article, tags", + Name, Brand, Category, Sku, Article, Tags); +EndFunction + +&AtClient +Function StubProducts() + P = New Array; + P.Add(StubProduct("Ноутбук Lenovo ThinkPad X1 Carbon", "Lenovo", "Ноутбуки", "SKU-NB-001", "ART-1001", "электроника,ноутбуки")); + P.Add(StubProduct("Смартфон Samsung Galaxy S24 Ultra", "Samsung", "Смартфоны", "SKU-SM-002", "ART-1002", "электроника,смартфоны")); + P.Add(StubProduct("Кофемашина DeLonghi Magnifica", "DeLonghi", "Кухонная техника", "SKU-KM-003", "ART-1003", "техника,кухня")); + P.Add(StubProduct("Монитор Dell UltraSharp 27", "Dell", "Мониторы", "SKU-MN-004", "ART-1004", "электроника,мониторы")); + P.Add(StubProduct("Наушники Sony WH-1000XM5", "Sony", "Аудио", "SKU-AU-005", "ART-1005", "электроника,аудио")); + Return P; +EndFunction + +&AtClient +Function StubClient(Name, Inn, City, Segment) + Return New Structure("name, inn, city, segment", Name, Inn, City, Segment); +EndFunction + +&AtClient +Function StubClients() + C = New Array; + C.Add(StubClient("ООО ""Ромашка""", "7701000001", "Москва", "опт")); + C.Add(StubClient("ООО ""Ромашка"" (филиал)", "7701000001", "Москва", "опт")); // same INN -> dup + C.Add(StubClient("ИП Иванов И.И.", "500100000010", "Химки", "розница")); + C.Add(StubClient("ООО ""Рога и Копыта""", "7702000002", "Казань", "опт")); + C.Add(StubClient("Иванов Иван (ИП)", "500100000010", "Химки", "розница")); // same INN -> dup + C.Add(StubClient("ООО ""Василёк""", "7703000003", "Тверь", "розница")); + Return C; +EndFunction + +&AtClient +Function ClientsDedup(Raw) + // Adapter-side dedup: exact INN match decides "one entity" (NOT in the core). + Seen = New Map; + Unique = New Array; + For Each Cl In Raw Do + NormKey = TrimAll(Cl.inn); + If Seen[NormKey] = Undefined Then + Seen.Insert(NormKey, True); + Unique.Add(Cl); + EndIf; + EndDo; + Return Unique; +EndFunction + +// ============================================================================ +// Adapters: map stub domain data -> index_segments payloads with rich metadata. +// ============================================================================ + +&AtClient +Function StepCatalogPayload() + Segments = New Array; + For Each Item In StubStepCatalog() Do + Seg = New Structure; + Seg.Insert("text", Item.phrase); + Seg.Insert("embed_text", Item.phrase + " | " + Item.description + " | параметры: " + Item.paramTypes); + Seg.Insert("meta", New Structure("type, params", "step", Item.paramTypes)); + Segments.Add(Seg); + EndDo; + Return SegmentsPayload("qa_steps", "qa-step-catalog", "QA step catalog", Segments); +EndFunction + +&AtClient +Function ScenariosPayload() + Segments = New Array; + For Each Sc In StubScenarios() Do + Verbatim = "Сценарий: " + Sc.name + Chars.LF + Sc.steps; + Seg = New Structure; + Seg.Insert("text", Verbatim); + Seg.Insert("embed_text", Sc.name + " " + StrReplace(Sc.tags, ",", " ") + " " + Sc.steps); + Seg.Insert("line_start", Sc.lineStart); + Seg.Insert("line_end", Sc.lineEnd); + Seg.Insert("meta", New Structure("type, feature, tags, name", "scenario", Sc.feature, Sc.tags, Sc.name)); + Segments.Add(Seg); + EndDo; + Return SegmentsPayload("qa_scenarios", "qa-scenarios", "QA scenarios", Segments); +EndFunction + +&AtClient +Function ProductsPayload() + Segments = New Array; + For Each Pr In StubProducts() Do + Seg = New Structure; + // text carries the article/SKU so the keyword channel finds exact ids; + // embed_text carries name+brand+category for semantic (dense) intent. + Seg.Insert("text", Pr.name + " (арт. " + Pr.article + ", " + Pr.sku + ")"); + Seg.Insert("embed_text", Pr.name + " | " + Pr.brand + " | " + Pr.category); + Seg.Insert("meta", New Structure("type, sku, article, category, brand, tags", + "product", Pr.sku, Pr.article, Pr.category, Pr.brand, Pr.tags)); + Segments.Add(Seg); + EndDo; + Return SegmentsPayload("products", "products-catalog", "Products", Segments); +EndFunction + +&AtClient +Function ClientsPayload(Unique) + Segments = New Array; + For Each Cl In Unique Do + Seg = New Structure; + Seg.Insert("text", Cl.name + " (ИНН " + Cl.inn + ")"); + Seg.Insert("embed_text", Cl.name + " " + Cl.city + " " + Cl.segment); + Seg.Insert("meta", New Structure("type, inn, city, segment", "client", Cl.inn, Cl.city, Cl.segment)); + Segments.Add(Seg); + EndDo; + Return SegmentsPayload("clients", "clients-catalog", "Clients", Segments); +EndFunction + +&AtClient +Procedure SelfTestAppend(Ctx, Line) + + Ctx.Log.Add(Line); + // Append ONLY the new line to the result file — O(1) per call. The previous + // implementation re-concatenated the entire log and truncate-rewrote the whole + // file on every call (O(N^2) in both string building and I/O). The first line + // truncates (Append=False) so each run starts clean; the rest append — the + // same pattern TraceLine uses. Ctx.Log is still kept in memory for any + // whole-log consumer. + Try + IsFirst = (Ctx.Log.Count() = 1); + Writer = New TextWriter(SelfTestOutFile("ragselftest-result.txt"), TextEncoding.UTF8, Chars.LF, Not IsFirst); + Writer.WriteLine(Line); + Writer.Close(); + Except + // best-effort; nothing to do if the file can't be written + EndTry; + +EndProcedure + +#EndRegion + + #Region Utilities &AtClient diff --git a/http-1c-dp/http1c/Templates/http1c/Ext/Template.bin b/http-1c-dp/http1c/Templates/http1c/Ext/Template.bin index eb064ca..b1fcaa3 100644 Binary files a/http-1c-dp/http1c/Templates/http1c/Ext/Template.bin and b/http-1c-dp/http1c/Templates/http1c/Ext/Template.bin differ diff --git a/rust-core/.cargo/config.toml b/rust-core/.cargo/config.toml new file mode 100644 index 0000000..879822a --- /dev/null +++ b/rust-core/.cargo/config.toml @@ -0,0 +1,19 @@ +# ----------------------------------------------------------------------------- +# CRT choice: rcore ships ONLY as a cdylib (rcore.dll) that the C++ component +# loads at RUNTIME (LoadLibrary, see http-1c-dll/src/RustCore.h) — nothing links +# it, so it does NOT have to match the C++ DLL's /MT. What it must match is +# ort/onnxruntime, which requires the DYNAMIC CRT (/MD); a static-CRT fastembed +# build produces a broken DLL. Memory never crosses the boundary raw either way: +# every rcore_* string is freed via rcore_free_string inside this DLL. +# +# `-crt-static` (the rustc default for *-msvc, pinned here explicitly) keeps a +# plain `cargo build --release --features fastembed` consistent with the +# official recipes (build/*.sh, CMakeLists, CI), which also pass +# RUSTFLAGS=-C target-feature=-crt-static — the env override and this config +# now agree instead of silently fighting (RUSTFLAGS replaces config rustflags). +# +# Note: this scopes the rustflag to the MSVC target only, so a non-Windows +# `cargo test`/`cargo check` (e.g. on a CI Linux box) is unaffected. +# ----------------------------------------------------------------------------- +[target.x86_64-pc-windows-msvc] +rustflags = ["-C", "target-feature=-crt-static"] diff --git a/rust-core/.gitignore b/rust-core/.gitignore new file mode 100644 index 0000000..ebda39f --- /dev/null +++ b/rust-core/.gitignore @@ -0,0 +1,7 @@ +# Rust build artifacts +/target/ + +# fastembed's downloaded/cached ONNX + tokenizer model files (created on the +# first real-embedder run / `cargo test --features fastembed`). Large binaries, +# never source — keep them out of git. +/.fastembed_cache/ diff --git a/rust-core/Cargo.lock b/rust-core/Cargo.lock new file mode 100644 index 0000000..74c8125 --- /dev/null +++ b/rust-core/Cargo.lock @@ -0,0 +1,3067 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anyhow" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey", + "rayon", + "thiserror", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "der" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "exr" +version = "1.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fastembed" +version = "5.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add59222e7bc3787285f993744b244cd454d78571845623606bdc45b22b23a4e" +dependencies = [ + "anyhow", + "hf-hub", + "image", + "ndarray", + "ort", + "safetensors", + "serde", + "serde_json", + "tokenizers", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", + "serde", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hf-hub" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" +dependencies = [ + "dirs", + "http", + "indicatif", + "libc", + "log", + "native-tls", + "rand", + "reqwest", + "serde", + "serde_json", + "thiserror", + "ureq", + "windows-sys 0.61.2", +] + +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89194689a993ab15268672e99e7b0e19da2da3268ac682e8f02d29d4d1434cd7" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indicatif" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "monostate" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafe1be9d0c75642e3e50fedc7ecadf1ef1cbce6eb66462153fc44245343fbee" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "openssl" +version = "0.10.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "pxfm" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand", + "rand_chacha", + "simd_helpers", + "thiserror", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rcore" +version = "0.1.0" +dependencies = [ + "anyhow", + "fastembed", + "once_cell", + "ort", + "regex", + "serde", + "serde_json", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "safetensors" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5" +dependencies = [ + "hashbrown 0.16.0", + "serde", + "serde_json", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "cookie_store", + "der", + "flate2", + "log", + "native-tls", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/rust-core/Cargo.toml b/rust-core/Cargo.toml new file mode 100644 index 0000000..6f6a4b3 --- /dev/null +++ b/rust-core/Cargo.toml @@ -0,0 +1,67 @@ +[package] +name = "rcore" +version = "0.1.0" +edition = "2021" +description = "Semantic + regex search core for the 1C MCP component, exposed over a C ABI (JSON in / JSON out)." +license = "MIT" +publish = false + +[lib] +# Single output shape: cdylib (rcore.dll). +# ort/onnxruntime requires the dynamic CRT (/MD), but the 1C C++ component +# cannot compile under /MD (char16_t streams hit MSVC C2491). So the real +# embedder ships as a separate rcore.dll (built /MD, self-contained static +# onnxruntime) that the /MT component does NOT link: it loads rcore.dll at +# runtime via LoadLibrary + GetProcAddress (see http-1c-dll/src/RustCore.h). +# One libhttp1cWin.dll therefore serves both packages — "lite" (no rcore.dll, +# search returns "install RAG") and "full" (with rcore.dll, real search). The +# FFI ownership convention (rcore_free_string frees Rust-side) keeps the +# cross-DLL string boundary safe. The full build is gated by RCORE_FASTEMBED. +name = "rcore" +crate-type = ["cdylib"] + +[dependencies] +# Keep the dependency surface minimal for Stage 0. Heavy ML deps +# (fastembed / ort / tokenizers) are intentionally deferred to Stage 1. +# Exact versions are pinned ("=x.y.z") so the skeleton build is reproducible. +serde = { version = "=1.0.219", features = ["derive"] } +serde_json = "=1.0.140" +once_cell = "=1.21.3" +# RE2-style regex engine (finite automata): linear-time matching, no +# backreferences/lookaround, so user-supplied `grep` patterns can't trigger +# catastrophic backtracking (ReDoS). Used by the `grep` dispatch method. +regex = "=1.12.3" + +# --- Real embedder (feature-gated) ------------------------------------------- +# fastembed pulls in onnxruntime (ort) + tokenizers — a heavy native stack we do +# NOT want compiled for the fast, mock-only `cargo test`. It is therefore an +# OPTIONAL dependency behind the `fastembed` feature (NOT default): the existing +# 83 unit tests compile/run without ort, and only `cargo test --features +# fastembed` (or a `--features fastembed` release build) compiles the real path. +# Pinned exactly to the versions verified working on this machine. +fastembed = { version = "=5.16.0", optional = true } +# fastembed's `embed` returns `anyhow::Result`; only needed on the real path. +anyhow = { version = "=1.0.99", optional = true } +# onnxruntime bindings, re-exported by fastembed. We name `ort` directly ONLY so +# we can spell the EP type `ort::ep::DirectML` when building the GPU execution- +# provider list. Pinned to the EXACT version fastembed depends on +# (=2.0.0-rc.12) so feature unification reuses a single ort build. Optional and +# behind the `fastembed` feature → the mock build never compiles ort. +ort = { version = "=2.0.0-rc.12", optional = true } + +[features] +# Off by default. Enabling it compiles the real FastEmbedder (ort + tokenizers) +# and wires `configure` to select it when the caller asks for a real model. +default = [] +# `fastembed/directml` turns on fastembed's DirectML path (it auto-sets the +# DML-required session opts — memory_pattern(false) + parallel_execution(false) +# — when a DirectML EP is registered). `dep:ort` + `ort/directml` make the +# `ort::ep::DirectML` type available so we can build the EP list ourselves. +# These are additive: fastembed already enables `ort/download-binaries`. +fastembed = ["dep:fastembed", "dep:anyhow", "dep:ort", "fastembed/directml", "ort/directml"] + +[profile.release] +# A search core is hot-path code; favour optimized release builds. +opt-level = 3 +lto = true +codegen-units = 1 diff --git a/rust-core/README.md b/rust-core/README.md new file mode 100644 index 0000000..848aeab --- /dev/null +++ b/rust-core/README.md @@ -0,0 +1,155 @@ +# rcore — Rust search core + +`rcore` provides semantic + keyword + hybrid search, regex (`grep`), and line +slicing (`get_segment`) for the 1C MCP component, over a JSON-in / JSON-out C +ABI. + +It compiles to a **cdylib** (`rcore.dll`) that the C++ component loads at +**runtime** (`LoadLibrary` + `GetProcAddress`, see +[`http-1c-dll/src/RustCore.h`](../http-1c-dll/src/RustCore.h)) — it is **not** +linked into the component. This is deliberate: ort/onnxruntime require the +dynamic CRT (`/MD`), but the C++ 1C component can't compile under `/MD` +(`std::basic_stringstream` hits MSVC C2491), so the search core ships +as a separate `/MD` DLL with a self-contained static onnxruntime. One +`libhttp1cWin.dll` then serves both distributions: + +- **lite** — `rcore.dll` absent → the `search`/`grep`/`get_segment`/ + `list_collections` tools return a structured `rag_not_installed` result. +- **full** — `rcore.dll` present (next to the component) + `DirectML.dll` → real + search. + +## Crate layout + +``` +rust-core/ +├── Cargo.toml # cdylib; pinned deps; feature-gated `fastembed` +├── .cargo/config.toml # +crt-static for the MOCK/staticlib path (see CRT note) +└── src/ + ├── lib.rs # C ABI + JSON dispatch + request parsing + e2e tests + ├── core.rs # process-global store (Lazy>) + async worker + ├── embed.rs # Embedder trait + deterministic MockEmbedder (tests) + ├── fastembed_embedder.rs # real FastEmbedder (feature `fastembed`; DirectML/CPU) + ├── filter.rs # combinable meta filters (`filter` field: any/all/tags) + ├── grep.rs # RE2-style regex scan over stored segment text + └── protocol.rs # JSON success/error envelope + error codes +``` + +## FFI contract + +All data crosses as **JSON strings**. Four `extern "C"` exports: + +| Function | Signature | +|----------|-----------| +| `rcore_version` | `char* rcore_version(void)` → `{"name","version","abi"}` | +| `rcore_dispatch` | `char* rcore_dispatch(const char* method, const char* payload_json)` → envelope | +| `rcore_free_string` | `void rcore_free_string(char* s)` | +| `rcore_shutdown` | `void rcore_shutdown(void)` | + +`rcore_dispatch` is the single generic entry point. Envelope: success +`{"ok":true,"result":...}`, failure `{"ok":false,"error":{"code","message"}}`. +Unknown method → `unknown_method`; bad JSON → `bad_payload`; bad regex → +`bad_pattern`. Panics are caught at the boundary (`catch_unwind`) and surface as +`internal` — they never unwind across the C ABI. + +**Methods:** `configure`, `index_segments`, `index_raw`, `search` +(`mode: dense|keyword|hybrid`, single / comma-list / all collections), `grep`, +`get_segment`, `stats`, `list_collections`, `list_models`, `reset`, +`delete_document`, `delete_collection`. Ingest is **async** (returns +immediately; a background worker pool embeds; collections expose a two-axis +`text_ready` / `vector_status` state, polled via `stats` / +`list_collections`). + +**Hit meta:** `search` hits echo the **effective** meta — the document-level +`meta` overlaid by the segment-level `meta`, segment winning on collision. It +is the same view the `filter` clauses match against, so what filtered a hit in +is exactly what the hit reports back (segment-level labels stay visible). + +**Collection registry:** each collection carries an optional `description`, +set via the `collection_description` field on `index_segments` **or** +`index_raw` (same semantics on both: last non-empty wins); +`list_collections` returns `{name, description, n_docs, n_segments, +vector_status, text_ready}` so a caller can discover what is searchable and +scope `search` to one or several collections. + +## Embedder: mock (tests) vs real (`fastembed` feature) + +The `Embedder` trait has two impls: + +- **`MockEmbedder`** (dim 64, deterministic token hash) — used by `cargo test` + and as a fallback. **Test-only**; not a production search backend. +- **`FastEmbedder`** (feature `fastembed`) — fastembed 5.16 + ort 2.0.0-rc.12 + + the multilingual-e5 family (L2-normalized, `query:`/`passage:` prefixes). + A separate `query` model instance plus a **bulk pool** of `embed_workers` + model sessions, because `fastembed::embed` is `&mut self` — a shared mutex + would let a bulk reindex block queries. Multiple bulk workers embed jobs + concurrently (CPU is forced when `embed_workers > 1`, since concurrent + DirectML sessions are unsupported); query embeddings stay prioritized. + +`configure` selects the backend: `model` / `model_path` non-empty (with the +feature compiled in) → `FastEmbedder`, else `MockEmbedder`. `embed_workers` +sizes the bulk pool (default auto ≈ `ncpu/2`, capped). + +**Model whitelist:** `model` must be one of the built-in names (matched +case-insensitively; empty/absent ⇒ the default): + +| name | dim | +|------|-----| +| `multilingual-e5-small` (default) | 384 | +| `multilingual-e5-base` | 768 | +| `multilingual-e5-large` | 1024 | + +`list_models` returns the same list — +`{"ok":true,"result":{"default":"multilingual-e5-small","models":[...]}}` — and +works before `configure` and in the lite/mock build, so a client can render a +model dropdown unconditionally. An unknown `model` name makes `configure` fail +with a structural `bad_model` error (message lists the supported names) before +any state is touched; this validation is identical with or without the +`fastembed` feature. A non-empty `model_path` (offline local files) bypasses +the whitelist — local files are loaded by path, not by name. + +**Model cache:** `configure`'s optional `cache_dir` sets the directory the +built-in model is downloaded into / loaded from (the fastembed/hf-hub cache +root, created on demand). Empty/absent ⇒ fastembed's default cache location. +Only meaningful for the builtin `model` path; `model_path` reads its files +directly and never touches a cache. `cache_dir` is echoed back by `configure` +alongside `model` / `model_path`. +**GPU:** `configure` `device: cpu | dml | auto` (default `auto`) registers the +DirectML execution provider with **automatic CPU fallback** (ort registers it +best-effort; no GPU/driver → CPU). The full package therefore ships +`DirectML.dll` (a hard import of the onnxruntime prebuilt). + +## Memory-ownership rule (critical) + +> Every `char*` returned to C is allocated by **Rust** (`CString::into_raw`) and +> must be freed **only** by `rcore_free_string`. **Never** call C's +> `free`/`delete` — that is a cross-CRT / cross-DLL free and corrupts the heap. + +Input pointers are borrowed (never freed by Rust). The C++ `RustString` RAII +wrapper frees via the **loaded** `rcore_free_string`, keeping ownership inside +`rcore.dll`. + +## CRT: mock `/MT` vs fastembed `/MD` + +- The **mock** path keeps the static CRT (`+crt-static` in + [`.cargo/config.toml`](.cargo/config.toml)) to match the C++ component's `/MT`. +- The **fastembed** `rcore.dll` needs the **dynamic** CRT (`/MD`): ort's prebuilt + onnxruntime is `/MD`, so `+crt-static` fails to link (`__imp_*` unresolved). + Build it with `RUSTFLAGS="-C target-feature=-crt-static"` (which CMake's + `RCORE_FASTEMBED=ON` path sets automatically). onnxruntime is statically linked + into `rcore.dll` — there is **no** `onnxruntime.dll`. + +## Build + +```sh +# Mock unit tests (fast, no ort): +cargo test + +# Real embedder tests (compiles ort; needs the dynamic CRT + network for the model): +RUSTFLAGS="-C target-feature=-crt-static" cargo test --features fastembed + +# Production rcore.dll (cdylib) — usually via CMake's full build: +# cmake .. -DRCORE_FASTEMBED=ON (drops rcore.dll next to the component) +RUSTFLAGS="-C target-feature=-crt-static" \ + cargo build --release --features fastembed --target x86_64-pc-windows-msvc +# → target/x86_64-pc-windows-msvc/release/rcore.dll +``` diff --git a/rust-core/src/core.rs b/rust-core/src/core.rs new file mode 100644 index 0000000..19a05dd --- /dev/null +++ b/rust-core/src/core.rs @@ -0,0 +1,3296 @@ +//! Process-global singleton store + the async ingest worker. +//! +//! A single [`Core`] instance is shared across *all* component instances in the +//! host process. The 1C runtime can open and close the MCP form many times (each +//! time constructing a fresh `HttpServerComponent` via the `AddComponent` +//! factory), but the search core — the loaded embedding model and the in-memory +//! index — must be built once and outlive any single component. It lives for the +//! lifetime of the process and dies with it. +//! +//! ## Concurrency model (Stage 1) +//! +//! The store is a `Lazy>`: +//! * readers = `search` / `stats`; +//! * writers = ingest-*apply* / `reset` / `configure`. +//! +//! The expensive part — embedding (15–60 s with a real model) — happens +//! **outside** the index lock, on background worker threads. CPU-heavy accept +//! work (token multisets, raw-doc normalization/offsets/chunking) is likewise +//! done in a lock-free *prepare* step; the write lock is held only for the +//! short *commit* (allocate ids, store text, mark Building) and for the short +//! apply (swap in finished vectors). This is the architectural heart of the +//! slice: an `index_segments` call from BSL returns immediately and never +//! freezes the 1C UI thread, and searches are never blocked behind a large +//! accept's text processing. +//! +//! The worker is a *pool* of `std::thread`s (one per embedder bulk session, so +//! a multi-session embedder embeds jobs in parallel; each thread is pinned to +//! its session by its spawn index) sharing one `mpsc` job queue. Large ingests +//! are split into [`EMBED_JOB_CHUNK`]-sized jobs so a single huge document +//! spreads across the pool instead of serializing on one thread. The pool is +//! spawned lazily on the first ingest and **resized** when a reconfigure +//! changes the embedder's bulk concurrency: the old threads get a `Stop` each +//! (after their FIFO queue drains) and are stashed in `retired_workers`; they +//! are joined — never under the index lock, which a worker may itself need to +//! apply a job — by [`shutdown`], alongside the current pool. + +use std::collections::HashMap; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::{Arc, Condvar, Mutex, RwLock}; +use std::thread::JoinHandle; + +use once_cell::sync::Lazy; + +use crate::embed::{dot, Embedder, MockEmbedder}; +use crate::filter::MetaFilter; + +// =========================================================================== +// Data model: collection → documents → segments +// =========================================================================== + +/// One indexed segment: a chunk of text plus its (lazily computed) vector. +/// +/// `embed_text` and segment-level `meta` are retained verbatim from ingest: +/// they are part of the stable data model (the real model card and future +/// meta-filter / re-embed cards read them) even though this dense-only slice +/// doesn't surface them yet. +#[allow(dead_code)] +#[derive(Clone)] +pub struct Segment { + /// Stable id, unique within the store. Lets callers reference a hit later. + pub segment_id: u64, + /// The human-facing text shown in search results. + pub text: String, + /// Optional alternate text to embed (e.g. a summary). Falls back to `text`. + pub embed_text: Option, + /// Optional source line range (offset table), echoed back in hits. + pub line_start: Option, + pub line_end: Option, + /// Arbitrary per-segment metadata (JSON object), echoed back in hits. + pub meta: serde_json::Value, + /// The normalized embedding. `None` until the worker fills it; stays `None` + /// for skipped (blank/non-finite) segments, which search simply ignores. + pub vector: Option>, + /// Lowercased token multiset of `text`, computed once at index time (by + /// [`token_multiset`], with the same tokenizer the query uses) so the keyword + /// channel scores a segment in O(query_terms) hash lookups instead of + /// re-tokenizing + allocating a `Vec` per segment on every query. Not + /// part of the wire model (never serialized); rebuilt on every (re)ingest of + /// the segment's text, so it can never drift from `text`. + pub kw_counts: HashMap, +} + +/// One document: a `doc_id`-keyed bundle of segments with doc-level metadata. +/// +/// For `index_raw` documents we additionally retain the **full normalized text** +/// plus a **line offset table** so `get_segment` can do an O(1) line-range slice. +/// Atomic `index_segments` records leave both `None` — they have no document-wide +/// text/line model, so `get_segment` returns a structural `no_line_index` error. +#[derive(Clone)] +pub struct Document { + /// Required stable id (a 1C reference / GUID). Everything that is later + /// updated or deleted is keyed by this. + pub doc_id: String, + pub name: String, + /// Doc-level metadata (JSON object), echoed back in hits. + pub meta: serde_json::Value, + pub segments: Vec, + /// Full CRLF→LF-normalized document text. `Some` only for `index_raw` docs. + pub full_text: Option, + /// Byte offset of the start of each line in `full_text` (1-based line N is at + /// `line_offsets[N-1]`). Parallel to `full_text`; `Some` only for raw docs. + /// Has exactly `line_count` entries (the number of `\n`-delimited lines). + pub line_offsets: Option>, +} + +/// Two-axis vector state. `text_ready` is a separate boolean on the collection. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum VectorStatus { + /// Text accepted; vectors still being computed by the worker. + Building, + /// All enqueued segments embedded (or skipped); vectors are complete. + Ready, +} + +impl VectorStatus { + /// Lower-case wire name used in JSON (`stats` / `search`). + pub fn as_str(self) -> &'static str { + match self { + VectorStatus::Building => "building", + VectorStatus::Ready => "ready", + } + } +} + +/// A named collection of documents plus its build state machine. +pub struct Collection { + /// Documents keyed by `doc_id` for atomic upsert/delete. + pub docs: HashMap, + /// Axis 1: text is queryable/visible the moment ingest is accepted. + pub text_ready: bool, + /// Axis 2: vectors transition Building → Ready as the worker finishes. + pub vector_status: VectorStatus, + /// Set only on a *fatal* fault. Per-segment problems do NOT set this. + pub error: Option, + /// How many jobs for this collection are still queued/in-flight. When it + /// drops to zero the collection flips to `Ready`. + pub pending_jobs: u32, + // ---- progress counters (the cold-start progress surfaced by `stats`) ---- + /// Segments that got a valid vector. + pub embedded: u64, + /// Segments that produced a non-finite/invalid vector. + pub failed: u64, + /// Segments skipped because their text was blank. + pub skipped: u64, + /// Human/AI-facing description of what this collection holds. Set via the + /// `collection_description` field on `index_segments` (last non-empty wins). + /// Surfaced by `stats` / `list_collections` so an MCP client can discover + /// which collections exist and pick a subset to search. + pub description: Option, +} + +impl Collection { + fn new() -> Self { + Collection { + docs: HashMap::new(), + text_ready: false, + vector_status: VectorStatus::Building, + error: None, + pending_jobs: 0, + embedded: 0, + failed: 0, + skipped: 0, + description: None, + } + } + + /// Total segment count across all docs. + pub fn n_segments(&self) -> usize { + self.docs.values().map(|d| d.segments.len()).sum() + } +} + +// =========================================================================== +// Configuration echoed back by `configure` / `stats` +// =========================================================================== + +/// Canonical wire name of the default built-in model (`configure` with an +/// empty/absent `model` that still wants the real embedder, e.g. via the +/// fastembed builtin path, loads this one). Surfaced by `list_models`. +pub const DEFAULT_MODEL: &str = "multilingual-e5-small"; + +/// Whitelist of built-in model names `configure` accepts in its `model` field, +/// in the order `list_models` reports them (default first). +/// +/// The NAMES live here — outside the `fastembed` feature — deliberately: +/// `list_models` must answer (so a client can render its model dropdown) and +/// `configure` must reject unknown names identically whether or not the real +/// embedder is compiled in. The name → `fastembed::EmbeddingModel` mapping +/// lives in `fastembed_embedder::supported_models` (feature-gated, since the +/// variant type only exists there); a feature-gated test pins the two lists +/// together so they can never drift. +pub const SUPPORTED_MODEL_NAMES: &[&str] = &[ + DEFAULT_MODEL, + "multilingual-e5-base", + "multilingual-e5-large", +]; + +/// Validate a `configure`-time built-in model name against +/// [`SUPPORTED_MODEL_NAMES`]. Matching is trimmed + ASCII-case-insensitive; +/// empty/whitespace means "the default" and is accepted. Unknown names return +/// the exact message the dispatcher embeds in its structural `bad_model` +/// error: `unknown model ''; supported: `. +pub fn validate_model_name(name: &str) -> Result<(), String> { + let trimmed = name.trim(); + if trimmed.is_empty() { + return Ok(()); // absent/blank ⇒ the default model + } + let lower = trimmed.to_ascii_lowercase(); + if SUPPORTED_MODEL_NAMES.iter().any(|m| *m == lower) { + Ok(()) + } else { + Err(format!( + "unknown model '{trimmed}'; supported: {}", + SUPPORTED_MODEL_NAMES.join(", ") + )) + } +} + +/// Echo of the knobs passed to `configure`. +/// +/// Embedder selection (see [`configure`]): a real fastembed model is built only +/// when the `fastembed` feature is compiled in AND the caller asked for it — +/// either by naming a built-in `model` (e.g. `"multilingual-e5-small"`) or by +/// supplying a non-empty `model_path` (offline files). Otherwise a +/// [`MockEmbedder`] is used (the default for the test suite and any build +/// without the feature). +#[derive(Clone, Default)] +pub struct Config { + /// Built-in model name, one of [`SUPPORTED_MODEL_NAMES`] (case-insensitive; + /// empty/absent ⇒ [`DEFAULT_MODEL`]). Selects the real embedder when the + /// `fastembed` feature is enabled. Unknown names are rejected by the + /// dispatcher with a structural `bad_model` error before `configure` runs. + pub model: Option, + /// Path to local ONNX + tokenizer files for the offline real embedder. + pub model_path: Option, + /// Directory the built-in model is downloaded into / loaded from (the + /// fastembed/HF cache root). `None`/blank ⇒ fastembed's default cache + /// location. Only meaningful for the builtin path (`model`); the offline + /// `model_path` route reads its files directly and never touches a cache. + pub cache_dir: Option, + pub normalize: bool, + pub max_seq_len: Option, + pub device: String, + pub intra_threads: Option, + /// Number of concurrent bulk-embedding worker threads / model sessions. + /// `None` ⇒ 1 (single worker, the default). `Some(0)` ⇒ auto (ncpu/2, + /// clamped to 1..=4 — multi-session scaling is memory-bandwidth bound and + /// saturates early, so more sessions would only waste RAM). + /// `Some(n)` ⇒ exactly `n`. More workers embed jobs concurrently (each on its + /// own model session with a smaller intra-op pool) at the cost of one model + /// copy in RAM per worker. Only meaningful for the real (fastembed) embedder. + pub embed_workers: Option, +} + +// =========================================================================== +// Background worker plumbing +// =========================================================================== + +/// A unit of embedding work handed to the worker thread. +/// +/// The doc's text segments are *already installed* in the store at accept time +/// (synchronously, with `vector: None`). This job carries only what the worker +/// needs to embed *outside* the lock and then, *inside* a short lock, fill the +/// already-present segments' vectors in place — matched by `(doc_id, segment_id)`: +/// * the collection + doc id to locate the segments; +/// * the stable `segment_id` of each segment, parallel to `embed_texts`; +/// * the text to embed for each segment (empty string marks a blank → skip). +struct EmbedJob { + collection: String, + doc_id: String, + /// Stable segment ids to fill, parallel to `embed_texts`. A stale id (the + /// doc was re-ingested while this job was in flight) simply won't be found + /// and is skipped — this is what keeps re-ingest atomic. + segment_ids: Vec, + /// Per-segment text to embed, parallel to `segment_ids`. Empty strings mark + /// blank segments to skip. + embed_texts: Vec, +} + +/// Messages on the worker queue. +enum WorkerMsg { + Job(EmbedJob), + /// Asks the worker to drain and exit. Sent by [`shutdown`]. + Stop, +} + +/// Owns the worker thread(s) + the shared job sender. Lives inside [`Core`]. +/// With more than one handle, several threads drain the queue concurrently, each +/// embedding on its own model session (see [`spawn_worker`]). +struct Worker { + tx: Sender, + handles: Vec>, +} + +// =========================================================================== +// The core +// =========================================================================== + +/// The process-global core state. +pub struct Core { + /// Whether `configure` has been called. + pub configured: bool, + /// The active embedder (mock for this slice). `None` until configured; + /// ingest auto-configures with a default mock if needed. + pub embedder: Option>, + /// Echoed config from the last `configure`. + pub config: Config, + /// Named collections of indexed documents. + pub collections: HashMap, + /// Source of stable, unique segment ids. + next_segment_id: u64, + /// The background worker pool, spawned lazily on first ingest and resized + /// (see [`enqueue_job`]) when the embedder's bulk concurrency changes. + worker: Option, + /// Join handles of worker threads retired by a pool resize. Retiring only + /// sends each old thread a `Stop` (their FIFO queue drains first) and + /// stashes the handle here; the actual `join()` happens in [`shutdown`], + /// OUTSIDE the index lock — joining under the write lock could deadlock, + /// because a retired thread may itself be blocked in `apply_job` waiting + /// for that very lock. + retired_workers: Vec>, + /// Condvar signalled whenever a collection's `vector_status`/counters change, + /// so tests (and callers) can `wait_until_ready` without polling/sleeping. + progress: Arc<(Mutex<()>, Condvar)>, +} + +impl Default for Core { + fn default() -> Self { + Core { + configured: false, + embedder: None, + config: Config::default(), + collections: HashMap::new(), + next_segment_id: 1, + worker: None, + retired_workers: Vec::new(), + progress: Arc::new((Mutex::new(()), Condvar::new())), + } + } +} + +impl Core { + /// Reset all mutable index state back to defaults. Used by `reset` and by + /// `configure` when the embedding space changes (dim mismatch). + /// + /// Note: does NOT tear down the worker thread — that is `shutdown`'s job and + /// must not happen on a routine `reset`. Any jobs still in flight for the + /// cleared collections become harmless: their apply step recreates the + /// (now-empty) collection only if it still… see `apply_job` for the guard. + pub fn reset(&mut self) { + self.configured = false; + self.embedder = None; + self.config = Config::default(); + self.collections.clear(); + self.next_segment_id = 1; + // (the dispatch counter lives in lib.rs as a lifetime-of-process atomic.) + // Wake any waiters so a `wait_until_ready` on a cleared collection + // returns promptly instead of hanging to its timeout. + self.progress.1.notify_all(); + } + + /// Allocate the next stable segment id. + fn alloc_segment_id(&mut self) -> u64 { + let id = self.next_segment_id; + self.next_segment_id = self.next_segment_id.saturating_add(1); + id + } + + /// Clone the progress condvar handle (shared with the worker thread). + pub fn progress_handle(&self) -> Arc<(Mutex<()>, Condvar)> { + Arc::clone(&self.progress) + } +} + +/// The process-global singleton. Acquire `.read()` for queries and `.write()` +/// for mutations. Never hold a lock across an FFI return. +pub static CORE: Lazy> = Lazy::new(|| RwLock::new(Core::default())); + +// =========================================================================== +// configure +// =========================================================================== + +/// Outcome of [`configure`], so the caller can build the JSON echo. +pub struct ConfigureResult { + pub dim: usize, + pub reset_due_to_dim_change: bool, +} + +/// Does this config ask for the *real* (fastembed) embedder? True when the +/// caller named a built-in `model` or supplied a non-empty `model_path`. When +/// false (the default, including the whole mock test suite) we build a +/// [`MockEmbedder`]. +fn wants_real_embedder(config: &Config) -> bool { + let named_model = config + .model + .as_deref() + .map(|m| !m.trim().is_empty()) + .unwrap_or(false); + let has_path = config + .model_path + .as_deref() + .map(|p| !p.trim().is_empty()) + .unwrap_or(false); + named_model || has_path +} + +/// Build the embedder this config selects, returning it as `Arc`. +/// +/// Selection: +/// * default (no `model` / `model_path`, or feature off) → [`MockEmbedder`] +/// (dim 64) — keeps the test suite and any no-feature build mock-only; +/// * `fastembed` feature on AND a real model requested → [`FastEmbedder`]: +/// a non-empty `model_path` loads local files (offline); otherwise the +/// built-in `model` name (one of [`SUPPORTED_MODEL_NAMES`], resolved by +/// `fastembed_embedder::resolve_model`; empty ⇒ [`DEFAULT_MODEL`]) is +/// downloaded/loaded from the HuggingFace cache — rooted at +/// `config.cache_dir` when given, else fastembed's default location. +/// +/// If the real model fails to load (e.g. offline with a cold cache, or +/// onnxruntime cannot be loaded) we **fall back to the mock** rather than +/// leaving the core unconfigured — this keeps the FFI boundary well-defined and +/// non-fatal. An *unknown model name* is normally rejected with a structural +/// `bad_model` error by the dispatcher BEFORE `configure` runs; a direct core +/// caller that skips that validation gets the same mock fallback here. Returns +/// `(embedder, real_in_effect)` so the caller can echo whether the real path is +/// actually active. +fn build_embedder(config: &Config) -> (Arc, bool) { + if !wants_real_embedder(config) { + return (Arc::new(MockEmbedder::new()), false); + } + + #[cfg(feature = "fastembed")] + { + use crate::fastembed_embedder::{resolve_model, Device, FastEmbedder}; + // Map the textual `device` knob to the execution-provider selection. + // Default (`auto`) = DirectML with ort's automatic CPU fallback, per the + // GPU-acceleration request; `cpu`/`dml` force the respective EP. Unknown + // strings fall back to `Auto` (safe: still works on a CPU-only machine). + let device = match config.device.trim().to_ascii_lowercase().as_str() { + "cpu" => Device::Cpu, + "dml" | "directml" => Device::DirectML, + _ => Device::Auto, // "auto" and any unrecognized value. + }; + // Prefer local files when a path is given (offline); else the built-in, + // resolved from the whitelisted `model` name (empty ⇒ default) and + // cached under `cache_dir` when one was given. `intra_threads` tunes the + // ONNX session pools (see `tunings`): query runs on a small CPU pool, + // bulk gets the configured device + remaining cores. + let built = match config.model_path.as_deref().filter(|p| !p.trim().is_empty()) { + Some(path) => { + FastEmbedder::new_local(path, device, config.intra_threads, config.embed_workers) + } + None => resolve_model(config.model.as_deref().unwrap_or("")).and_then(|model| { + FastEmbedder::new_builtin( + model, + config.cache_dir.as_deref().filter(|d| !d.trim().is_empty()), + device, + config.intra_threads, + config.embed_workers, + ) + }), + }; + match built { + Ok(fe) => return (Arc::new(fe), true), + Err(_e) => { + // Real model unavailable → degrade to mock (non-fatal). The + // dim then reverts to the mock's, which `stats` reports. + return (Arc::new(MockEmbedder::new()), false); + } + } + } + + // Feature not compiled in: a real request falls back to the mock. (The C++ + // build that needs the real model compiles with `--features fastembed`.) + #[cfg(not(feature = "fastembed"))] + { + (Arc::new(MockEmbedder::new()), false) + } +} + +/// Apply a `configure` request. Idempotent. If a different-dim embedder is +/// chosen while data is already indexed, the index is reset (old vectors live in +/// a different space and are invalid) — this is what handles the 64↔384 switch +/// between the mock and the real e5 model. +/// +/// The embedder is chosen by [`build_embedder`]: a [`MockEmbedder`] by default +/// (and for any build without the `fastembed` feature), or a real +/// [`FastEmbedder`] when the feature is on and the caller asked for a real model +/// via `model` / `model_path`. +pub fn configure(core: &mut Core, config: Config) -> ConfigureResult { + let (new_embedder, _real) = build_embedder(&config); + let new_dim = new_embedder.dim(); + + // If we already had an embedder of a different dim AND there is indexed + // data, the existing vectors are invalid in the new space → full reset. + // This covers the mock↔real (64↔384) transition: switching to the real + // model after indexing under the mock (or vice versa) clears the stale + // vectors so search never mixes incompatible vector spaces. + let old_dim = core.embedder.as_ref().map(|e| e.dim()); + let had_data = !core.collections.is_empty(); + let dim_changed = matches!(old_dim, Some(d) if d != new_dim); + let reset_due_to_dim_change = dim_changed && had_data; + + if reset_due_to_dim_change { + // Drop indexed collections; in-flight jobs for them become no-ops + // because their target collections no longer exist (see `apply_job`). + core.collections.clear(); + core.next_segment_id = 1; + } + + core.embedder = Some(new_embedder); + core.config = config; + core.configured = true; + core.progress.1.notify_all(); + + ConfigureResult { + dim: new_dim, + reset_due_to_dim_change, + } +} + +/// Ensure an embedder exists, auto-configuring a default mock if `configure` was +/// never called. Returns the embedder handle. Keeps ingest usable without a +/// mandatory explicit configure step (the dim is then the mock default). +fn ensure_embedder(core: &mut Core) -> Arc { + if core.embedder.is_none() { + core.embedder = Some(Arc::new(MockEmbedder::new())); + // Do NOT flip `configured` — that flag tracks an explicit configure. + } + Arc::clone(core.embedder.as_ref().expect("just set")) +} + +// =========================================================================== +// index_segments — the async accept +// =========================================================================== + +/// A parsed segment request from the `index_segments` payload (pre-embedding). +pub struct SegmentInput { + pub text: String, + pub embed_text: Option, + pub line_start: Option, + pub line_end: Option, + pub meta: serde_json::Value, +} + +/// A parsed `index_segments` request. +pub struct IndexRequest { + pub collection: String, + pub doc_id: String, + pub name: String, + pub meta: serde_json::Value, + pub segments: Vec, + /// Optional description of the *collection* (not the doc). Sets/updates + /// `Collection::description` so callers can label what a collection holds. + pub description: Option, +} + +/// Outcome of the synchronous accept. +pub struct AcceptResult { + pub collection: String, + pub segment_count: usize, +} + +/// The lock-free *prepare* half of an `index_segments` accept: everything that +/// is O(total text) — choosing embed texts, building the per-segment keyword +/// multisets — done WITHOUT `&mut Core`, so the dispatcher can run it before +/// taking the global write lock and searches are never blocked behind a large +/// accept's text processing. Segment ids are placeholders (`0`) here; the +/// commit step allocates the real ids under the lock. +pub struct PreparedIndex { + collection: String, + description: Option, + /// The fully-built doc (kw_counts computed, `vector: None`) with + /// placeholder `segment_id: 0` on every segment. + doc: Document, + /// Per-segment text to embed, parallel to `doc.segments`. Empty strings + /// mark blank segments (already counted in `skipped_now`). + embed_texts: Vec, + /// Blank segments counted during prepare; added to the collection's + /// `skipped` counter at commit. + skipped_now: u64, +} + +/// Build a [`PreparedIndex`] from an `index_segments` request — the CPU-heavy +/// part of accept (token multisets over every segment), pure of `&mut Core`. +/// Call this OUTSIDE the global write lock, then hand the result to +/// [`commit_index`] under the lock. +pub fn prepare_index(req: IndexRequest) -> PreparedIndex { + let mut doc = Document { + doc_id: req.doc_id.clone(), + name: req.name, + meta: req.meta, + segments: Vec::with_capacity(req.segments.len()), + // Atomic `index_segments` records carry no document-wide text/offset + // table; `get_segment` is intentionally unsupported for them. + full_text: None, + line_offsets: None, + }; + let mut embed_texts: Vec = Vec::with_capacity(req.segments.len()); + let mut skipped_now: u64 = 0; + + for s in req.segments { + // Choose embed_text-or-text; blank → mark skip (empty embed string). + let chosen = s.embed_text.clone().unwrap_or_else(|| s.text.clone()); + let is_blank = chosen.trim().is_empty(); + if is_blank { + skipped_now += 1; + } + embed_texts.push(if is_blank { String::new() } else { chosen }); + let kw_counts = token_multiset(&s.text); + doc.segments.push(Segment { + segment_id: 0, // placeholder; the real id is allocated at commit + text: s.text, + embed_text: s.embed_text, + line_start: s.line_start, + line_end: s.line_end, + meta: s.meta, + vector: None, + kw_counts, + }); + } + + PreparedIndex { + collection: req.collection, + description: req.description, + doc, + embed_texts, + skipped_now, + } +} + +/// The locked *commit* half of an `index_segments` accept. Does only the cheap +/// work under the write lock — allocate real segment ids, **install the doc's +/// text segments into the store with `vector: None`** (atomic upsert by +/// `doc_id`), mark the collection `text_ready=true` / `vector_status=Building`, +/// bump `pending_jobs` — then enqueues the [`EmbedJob`] chunks for the worker +/// pool. Returns immediately; embedding happens off-thread. +/// +/// This preserves the carried-forward fix from Stage 1 (§4.4 / Правка-2): +/// text-only operations (`grep` / `get_segment` / keyword) can see a doc's text +/// the moment commit returns, because the segments are present in the store +/// *before* the worker runs. The worker fills ONLY the vectors afterwards +/// (matched by `segment_id`), so dense search is the only operation that waits +/// for vectors. +/// +/// Blank-text segments were counted as `skipped` during prepare (they never get +/// embedded) so they don't keep the collection in Building forever. +pub fn commit_index(core: &mut Core, prepared: PreparedIndex) -> AcceptResult { + let _embedder = ensure_embedder(core); + + let PreparedIndex { + collection, + description, + mut doc, + embed_texts, + skipped_now, + } = prepared; + + // Allocate the real, store-unique segment ids (the only per-segment work + // that genuinely needs `&mut Core`), keeping the parallel id vec for the + // embed jobs. + let mut segment_ids: Vec = Vec::with_capacity(doc.segments.len()); + for seg in doc.segments.iter_mut() { + seg.segment_id = core.alloc_segment_id(); + segment_ids.push(seg.segment_id); + } + + let segment_count = doc.segments.len(); + let doc_id = doc.doc_id.clone(); + + // --- short write-locked accept: install text, mark Building --- + let coll = core + .collections + .entry(collection.clone()) + .or_insert_with(Collection::new); + coll.text_ready = true; + coll.vector_status = VectorStatus::Building; + coll.error = None; + coll.skipped = coll.skipped.saturating_add(skipped_now); + // Label the collection (last non-empty description wins). + if let Some(desc) = description { + if !desc.trim().is_empty() { + coll.description = Some(desc); + } + } + + // Atomic upsert by doc_id: install the doc's text segments NOW (vectors are + // `None`). Re-ingesting the same doc_id replaces all of its segments here, in + // one map insert — so text-readers never see a torn doc, and any embed job + // still in flight for the *previous* generation of this doc has stale + // `segment_id`s that simply won't match (see `apply_job`). + coll.docs.insert(doc_id.clone(), doc); + + // Enqueue the embed work, chunked across the worker pool (bumps + // `pending_jobs` by the chunk count — see `enqueue_embed_chunks`). + enqueue_embed_chunks(core, collection.clone(), doc_id, segment_ids, embed_texts); + + AcceptResult { + collection, + segment_count, + } +} + +/// Synchronous accept for `index_segments`: thin prepare+commit wrapper, kept +/// so existing callers/tests that already hold the write lock keep working. +/// The dispatcher (`lib.rs`) instead runs [`prepare_index`] BEFORE taking the +/// lock and [`commit_index`] under it, so the O(total text) work never blocks +/// concurrent searches. (Test-only in non-test builds, hence the allow.) +#[cfg_attr(not(test), allow(dead_code))] +pub fn accept_index(core: &mut Core, req: IndexRequest) -> AcceptResult { + commit_index(core, prepare_index(req)) +} + +/// Job quantum: the maximum number of segments carried by one [`EmbedJob`]. +/// A large ingest (one `index_segments`/`index_raw` call) is split into jobs of +/// at most this many segments so a single huge document spreads across the +/// whole worker pool instead of keeping one worker busy while the rest idle. +/// 256 matches the ONNX sub-batch scale (`FastEmbedder::embed_batch` defaults +/// to 256/m), so a chunk is a few sub-batches of real work — big enough to +/// amortize per-job overhead, small enough to parallelize. +const EMBED_JOB_CHUNK: usize = 256; + +/// Split one ingest's parallel `(segment_ids, embed_texts)` arrays into +/// [`EMBED_JOB_CHUNK`]-sized [`EmbedJob`]s, bump the collection's +/// `pending_jobs` by the chunk count, and enqueue them. The bump and the +/// enqueue live in this ONE helper (used by both accept paths) so they can +/// never drift apart — `pending_jobs` reaching zero is what flips a collection +/// to `Ready`, so an off-by-one here would strand it `Building` forever. +/// +/// EDGE CASE: an empty segment list still enqueues exactly ONE (empty) job — +/// that empty job's apply is what decrements `pending_jobs` and flips the +/// collection to `Ready`; zero jobs would leave it `Building` forever. Hence +/// chunk count = `max(1, ceil(len / EMBED_JOB_CHUNK))`. +fn enqueue_embed_chunks( + core: &mut Core, + collection: String, + doc_id: String, + segment_ids: Vec, + embed_texts: Vec, +) { + debug_assert_eq!(segment_ids.len(), embed_texts.len()); + let n = segment_ids.len(); + let n_chunks = ((n + EMBED_JOB_CHUNK - 1) / EMBED_JOB_CHUNK).max(1); + + // Bump pending_jobs by the number of chunks BEFORE enqueueing, so a fast + // worker applying the first chunk can never observe a zero counter and flip + // the collection Ready prematurely. (We hold the write lock here, but the + // ordering keeps the invariant obvious and robust.) + if let Some(coll) = core.collections.get_mut(&collection) { + coll.pending_jobs = coll.pending_jobs.saturating_add(n_chunks as u32); + } + + // Walk the parallel arrays front-to-back, splitting off ≤CHUNK-sized heads. + // The first iteration of the loop also covers the empty-list case (one + // empty job), keeping the max(1, …) contract without a special branch. + let mut rest_ids = segment_ids; + let mut rest_texts = embed_texts; + loop { + let take = rest_ids.len().min(EMBED_JOB_CHUNK); + let tail_ids = rest_ids.split_off(take); + let tail_texts = rest_texts.split_off(take); + enqueue_job( + core, + EmbedJob { + collection: collection.clone(), + doc_id: doc_id.clone(), + segment_ids: rest_ids, + embed_texts: rest_texts, + }, + ); + if tail_ids.is_empty() { + break; + } + rest_ids = tail_ids; + rest_texts = tail_texts; + } +} + +/// Push a job onto the worker pool's queue, lazily spawning — or **resizing** — +/// the pool first. The number of worker threads matches the embedder's bulk +/// concurrency (one thread per model session), so a multi-session embedder +/// embeds jobs in parallel. +/// +/// Resizing: a reconfigure can swap in an embedder with a different +/// `bulk_concurrency` (e.g. `embed_workers: 2` → `6`). The old pool would then +/// silently mismatch the new session count — no parallelism gain, wasted RAM — +/// so when the live pool size differs from what the current embedder wants we +/// retire it: send one `Stop` per old thread on the OLD queue (it is FIFO, so +/// any jobs still pending there drain before the Stops are seen) and stash the +/// handles in `retired_workers`, then spawn a fresh pool of the right size. +/// CRITICAL: we never `join()` here — we hold the CORE write lock, and an old +/// worker may be blocked in `apply_job` waiting for that very lock (deadlock). +/// The retired handles are joined by [`shutdown`], outside the lock. +fn enqueue_job(core: &mut Core, job: EmbedJob) { + let want = core + .embedder + .as_ref() + .map(|e| e.bulk_concurrency()) + .unwrap_or(1) + .max(1); + + if core.worker.as_ref().map(|w| w.handles.len()) != Some(want) { + // Retire a wrong-sized pool (Stops queue behind its pending jobs). + if let Some(old) = core.worker.take() { + for _ in 0..old.handles.len() { + let _ = old.tx.send(WorkerMsg::Stop); + } + core.retired_workers.extend(old.handles); + } + // Spawn the fresh pool sized to the current embedder. + core.worker = Some(spawn_worker(core.progress_handle(), want)); + } + if let Some(w) = core.worker.as_ref() { + // If the receiver is gone (worker stopped), the job is dropped and the + // collection would stay Building; that only happens post-shutdown. + let _ = w.tx.send(WorkerMsg::Job(job)); + } +} + +/// Spawn `workers` background worker threads sharing one job queue. Each pulls +/// jobs, embeds OUTSIDE the index lock, then applies finished vectors under a +/// short write lock. A single mpsc receiver is shared behind a `Mutex`: a thread +/// holds it only for the brief dequeue (and while blocked on an empty queue — +/// harmless, there is no work), releasing it before the expensive embed, so up +/// to `workers` embeds run concurrently. `progress` wakes `wait_until_ready`. +/// +/// Each thread carries its stable spawn index `i` as its embedding *slot*: the +/// worker embeds via `embed_passages_at(i, …)`, which pins thread `i` to model +/// session `i` for the thread's whole life. Pinning (vs the old shared +/// round-robin cursor) means a slow job on one session can never get a sibling +/// worker queued up behind that same session's lock while other sessions idle. +fn spawn_worker(progress: Arc<(Mutex<()>, Condvar)>, workers: usize) -> Worker { + let (tx, rx): (Sender, Receiver) = mpsc::channel(); + let shared_rx = Arc::new(Mutex::new(rx)); + let mut handles = Vec::with_capacity(workers); + for i in 0..workers.max(1) { + let rx = Arc::clone(&shared_rx); + let progress = Arc::clone(&progress); + let handle = std::thread::Builder::new() + .name(format!("rcore-ingest-{i}")) + .spawn(move || worker_loop(i, rx, progress)) + .expect("failed to spawn ingest worker"); + handles.push(handle); + } + Worker { tx, handles } +} + +/// The worker thread body. Dequeues under the shared receiver lock (released +/// before the heavy embed so siblings run concurrently), embeds outside the +/// index lock, then applies under a short lock. Exits on `Stop` or when the +/// sender is dropped. `slot` is this thread's stable spawn index, used to pin +/// it to one embedder session (see [`spawn_worker`]). +fn worker_loop(slot: usize, rx: Arc>>, progress: Arc<(Mutex<()>, Condvar)>) { + loop { + // Hold the receiver lock ONLY for the dequeue; a poisoned lock or a + // dropped/closed sender ends this thread. + let msg = match rx.lock() { + Ok(guard) => guard.recv(), + Err(_) => break, + }; + let msg = match msg { + Ok(m) => m, + Err(_) => break, // sender dropped → drain done + }; + match msg { + WorkerMsg::Stop => break, + WorkerMsg::Job(job) => { + // --- embed OUTSIDE the index lock (the expensive part) --- + // Grab a handle to the embedder under a brief read lock, then + // release it before doing the heavy embedding work. + let embedder = match CORE.read() { + Ok(c) => c.embedder.clone(), + Err(_) => None, + }; + let embedder = match embedder { + Some(e) => e, + None => { + // No embedder (e.g. reset mid-flight) — apply with the + // collection still bookkept so it doesn't hang Building. + apply_job(job, None); + notify(&progress); + continue; + } + }; + + // Only embed the non-blank texts; blanks already counted as + // skipped at accept. We embed all in one batch call, pinned to + // this thread's model session via `slot` (a retired thread's + // high slot is taken modulo the pool size by the embedder). + let vectors = embedder.embed_passages_at(slot, &job.embed_texts); + + // --- apply finished vectors under a SHORT write lock --- + apply_job(job, Some(vectors)); + notify(&progress); + } + } + } +} + +/// Notify all waiters that progress changed. +fn notify(progress: &Arc<(Mutex<()>, Condvar)>) { + let _guard = progress.0.lock().expect("progress mutex"); + progress.1.notify_all(); +} + +/// Apply one finished job under a short write lock: fill the *already-installed* +/// segments' vectors in place (matched by `segment_id`), update counters, and +/// flip the collection to `Ready` when it has no more pending jobs. +/// +/// The doc's text was installed synchronously at accept time, so here we only +/// fill vectors — we never insert/replace segments. `vectors`, when `Some`, is +/// parallel to `job.segment_ids` / `job.embed_texts`: +/// * a blank embed text (empty string) → skip (vector stays `None`); +/// * a non-finite / all-zero vector → fail; +/// * a missing segment id (the doc was re-ingested while this job ran, so the +/// id is stale) → silently skipped, which is what makes re-ingest atomic; +/// * otherwise the vector is installed and `embedded` bumped. +fn apply_job(job: EmbedJob, vectors: Option>>) { + let mut core = match CORE.write() { + Ok(c) => c, + Err(_) => return, // poisoned lock: nothing safe to do + }; + + let EmbedJob { + collection, + doc_id, + segment_ids, + embed_texts, + } = job; + + // If the collection was cleared (reset / dim-change) while this job was in + // flight, drop the result on the floor. The job is now meaningless. + let coll = match core.collections.get_mut(&collection) { + Some(c) => c, + None => return, + }; + + let mut embedded = 0u64; + let mut failed = 0u64; + + if let Some(mut vectors) = vectors { + // Locate the (possibly re-ingested) doc once; if it's gone, every id is + // stale and we just fall through to the bookkeeping below. + if let Some(doc) = coll.docs.get_mut(&doc_id) { + // Index this doc's current segments by id for an O(1) lookup, so a + // re-ingest that shuffled/replaced segments still fills correctly and + // stale ids from a superseded generation are simply absent. + let mut by_id: HashMap = doc + .segments + .iter_mut() + .map(|s| (s.segment_id, s)) + .collect(); + + for (i, &sid) in segment_ids.iter().enumerate() { + // Blank text was marked skip at accept; leave vector None. + if embed_texts.get(i).map(|t| t.is_empty()).unwrap_or(true) { + continue; + } + // Stale id (doc re-ingested) → segment absent → skip silently. + let seg = match by_id.get_mut(&sid) { + Some(s) => s, + None => continue, + }; + // MOVE the vector out of the job (leaving an empty Vec behind) + // instead of cloning it under the write lock — each embedding is + // `dim` f32s and we hold the exclusive lock here. + match vectors.get_mut(i).map(std::mem::take) { + Some(v) if v.iter().all(|x| x.is_finite()) && v.iter().any(|&x| x != 0.0) => { + seg.vector = Some(v); + embedded += 1; + } + _ => { + // Non-finite / all-zero / absent vector → fail, keep going. + failed += 1; + } + } + } + } + } + + coll.embedded = coll.embedded.saturating_add(embedded); + coll.failed = coll.failed.saturating_add(failed); + coll.pending_jobs = coll.pending_jobs.saturating_sub(1); + if coll.pending_jobs == 0 { + coll.vector_status = VectorStatus::Ready; + } +} + +// =========================================================================== +// index_raw — normalize + offset table + chunker + async accept +// =========================================================================== + +/// Default token target per chunk when the request omits `chunk_cfg`. The +/// chunker aims for chunks around this many estimated tokens, snapping to whole +/// lines, so a chunk is usually slightly under or over this figure. +pub const DEFAULT_CHUNK_TARGET_TOKENS: usize = 300; + +/// Default number of overlapping lines carried into the next chunk so a match +/// straddling a chunk boundary is still wholly present in at least one chunk. +const DEFAULT_OVERLAP_LINES: usize = 2; + +/// Estimate the token count of a piece of text. +/// +/// IMPORTANT — heuristic placeholder. There is no real tokenizer in this slice +/// (the [`MockEmbedder`] is a bag-of-hashed-tokens model with no vocabulary). We +/// approximate a transformer subword count with `max(whitespace_words, chars/4)`: +/// whitespace words give a sane floor for normal prose, while `chars/4` (the +/// common "~4 chars per token" rule of thumb) dominates for long/CJK/no-space +/// runs so we never wildly under-count. This is the SINGLE place that defines +/// "token count"; when the real embedder + tokenizer land, swap this body for a +/// call into the tokenizer and everything downstream (budget, hard cap, oversized +/// detection) keeps working unchanged. +pub fn estimate_tokens(text: &str) -> usize { + let words = text.split_whitespace().count(); + let chars = text.chars().count(); + words.max(chars / 4) +} + +/// Knobs controlling how [`chunk_text`] splits a document. All optional on the +/// wire (`chunk_cfg`); sensible defaults are filled in by [`ChunkConfig::resolve`]. +#[derive(Clone, Copy)] +pub struct ChunkConfig { + /// Soft per-chunk token budget the chunker aims for (snapped to line ends). + pub target_tokens: usize, + /// Hard cap on tokens per chunk; defaults to the configured `max_seq_len`. + /// A single line over this cap becomes one *oversized* chunk on its own. + pub max_tokens: usize, + /// Number of trailing lines of one chunk repeated at the start of the next. + pub overlap_lines: usize, +} + +impl ChunkConfig { + /// Resolve a (possibly partial) request config against the store's + /// `max_seq_len`, clamping to keep invariants: target ≥ 1, max ≥ target, + /// overlap < target so chunking always makes forward progress. + pub fn resolve( + target_tokens: Option, + max_tokens: Option, + overlap_lines: Option, + config_max_seq_len: Option, + ) -> ChunkConfig { + let target = target_tokens.unwrap_or(DEFAULT_CHUNK_TARGET_TOKENS).max(1); + // Hard cap = explicit value, else configured max_seq_len, else fall back + // to the target itself. Never below the target (that would make every + // chunk oversized). + let cap = max_tokens + .or_else(|| config_max_seq_len.map(|v| v as usize)) + .unwrap_or(target) + .max(target); + // Overlap must stay strictly below the target line count so successive + // chunks always advance; we cap it defensively here too. + let overlap = overlap_lines.unwrap_or(DEFAULT_OVERLAP_LINES); + ChunkConfig { + target_tokens: target, + max_tokens: cap, + overlap_lines: overlap, + } + } +} + +/// One chunk produced by [`chunk_text`]: its 1-based inclusive line range, the +/// exact text of those lines, and whether it is an *oversized* single line. +pub struct Chunk { + /// 1-based inclusive first line of the chunk. + pub line_start: u64, + /// 1-based inclusive last line of the chunk. + pub line_end: u64, + /// The chunk's text (the joined lines, LF-separated, no trailing newline). + pub text: String, + /// True when this chunk is a single line that alone exceeds `max_tokens`. + /// Such a chunk is truncated for *embedding* but returned whole by + /// `get_segment`; we surface the flag so the caller (accept) can truncate + /// only the embed text and keep the stored segment text intact. + pub oversized: bool, +} + +/// Build the line offset table for `text`: the byte offset at which each +/// `\n`-delimited line begins. `text` MUST already be CRLF→LF normalized. +/// +/// The table has exactly one entry per line, where "line count" follows the same +/// model grep uses: `split('\n')` yields `(number of '\n') + 1` pieces. So a +/// trailing `\n` produces a final empty line with its own offset (== text.len()). +/// This lets `get_segment` map a 1-based line number to a byte range in O(1). +pub fn build_line_offsets(text: &str) -> Vec { + let mut offsets = vec![0usize]; // line 1 always starts at byte 0 + for (i, b) in text.bytes().enumerate() { + if b == b'\n' { + // The next line starts right after this '\n'. + offsets.push(i + 1); + } + } + offsets +} + +/// Split a normalized document into line-snapped chunks by token budget. +/// +/// Algorithm (greedy, line-granular, with overlap): +/// * Walk the document's lines. Accumulate lines into the current chunk while +/// the running token estimate stays under `target_tokens`. +/// * When adding a line would exceed the target, the current chunk is emitted +/// (snapped to whole line boundaries — we never split mid-line), and the next +/// chunk *backs up* `overlap_lines` lines so consecutive chunks overlap. +/// * Edge case — a single line that alone exceeds `max_tokens` is emitted as +/// its own **oversized** chunk (it cannot be combined or split further at the +/// line granularity). The caller truncates only its *embed* text to the cap; +/// `get_segment` still returns it whole from the stored full text. +/// +/// Returns chunks with 1-based inclusive line ranges. An empty document yields a +/// single empty chunk covering line 1 so the doc is always representable. +pub fn chunk_text(text: &str, cfg: &ChunkConfig) -> Vec { + // Line model matches `build_line_offsets` / grep: split on '\n'. This keeps + // line numbers consistent across chunking, the offset table, and get_segment. + let lines: Vec<&str> = text.split('\n').collect(); + let n = lines.len(); + + let mut chunks: Vec = Vec::new(); + if n == 0 { + return chunks; // unreachable (split always yields ≥1), but be safe. + } + + let mut start = 0usize; // 0-based index of the current chunk's first line + while start < n { + let mut end = start; // 0-based inclusive index of last line included + let mut tokens = estimate_tokens(lines[start]); + + // A single first line already over the hard cap → oversized chunk of + // exactly that line. Don't try to grow it. + let oversized = tokens > cfg.max_tokens; + if !oversized { + // Grow the chunk one line at a time while we stay within the target + // budget. We always keep at least the starting line (end == start). + while end + 1 < n { + let next_tokens = estimate_tokens(lines[end + 1]); + if tokens + next_tokens > cfg.target_tokens { + break; + } + tokens += next_tokens; + end += 1; + } + } + + // Materialize the chunk text from the joined lines (LF-separated). + let text = lines[start..=end].join("\n"); + chunks.push(Chunk { + line_start: (start + 1) as u64, // 1-based inclusive + line_end: (end + 1) as u64, + text, + oversized, + }); + + // Advance. Normally we step to `end + 1`, but back up `overlap_lines` so + // consecutive chunks share context. Never let overlap stall progress: + // the next start must be strictly greater than the current start. + let next_start = (end + 1).saturating_sub(cfg.overlap_lines); + start = next_start.max(start + 1); + } + + chunks +} + +/// A parsed `index_raw` request (the raw-document ingest path). +pub struct RawIndexRequest { + pub collection: String, + /// Optional description of the *collection* (not the doc) — same + /// `collection_description` semantics as `index_segments` (last non-empty wins). + pub description: Option, + /// Caller-supplied id; `None` ⇒ auto-assign a stable id and return it. + pub doc_id: Option, + pub name: String, + pub meta: serde_json::Value, + /// The raw document text (any newline convention; normalized on accept). + pub text: String, + /// Optional chunker overrides (`target_tokens` / `max_tokens` / `overlap_lines`). + pub target_tokens: Option, + pub max_tokens: Option, + pub overlap_lines: Option, +} + +/// Outcome of the synchronous `index_raw` accept. `doc_id` is always populated +/// (auto-assigned when the request omitted it) so the caller can upsert/delete. +pub struct RawAcceptResult { + pub collection: String, + pub doc_id: String, + pub segment_count: usize, +} + +/// Auto-assign a stable, collision-resistant `doc_id` for a fire-and-forget +/// `index_raw` with no caller id. Reuses the monotonic segment-id source so the +/// value is unique within the process run. The `raw:` prefix makes auto-ids +/// distinguishable from caller GUIDs in logs/debugging. +fn auto_doc_id(core: &mut Core) -> String { + format!("raw:{}", core.alloc_segment_id()) +} + +/// The lock-free *prepare* half of an `index_raw` accept: CRLF normalization, +/// the line offset table, chunking, kw multisets — all the O(total text) work, +/// done WITHOUT `&mut Core` so the dispatcher can run it before taking the +/// global write lock. `max_seq_len` is passed in as a plain value (the +/// dispatcher reads it under a brief read lock first); segment ids are +/// placeholders (`0`) until [`commit_index_raw`] allocates the real ones. +pub struct PreparedRawIndex { + collection: String, + /// Collection description carried to commit (last non-empty wins). + description: Option, + /// Caller-supplied id, or `None` ⇒ auto-assigned at commit (needs the + /// store's id source). + doc_id: Option, + name: String, + meta: serde_json::Value, + /// The fully-normalized document text (CRLF→LF), stored on the doc. + full_text: String, + /// Offset table over `full_text` (see [`build_line_offsets`]). + line_offsets: Vec, + /// Chunked segments (kw_counts computed, `vector: None`, placeholder ids). + segments: Vec, + /// Per-segment embed text, parallel to `segments` (empty string = blank). + embed_texts: Vec, + /// Blank chunks counted during prepare; added to `skipped` at commit. + skipped_now: u64, +} + +/// Build a [`PreparedRawIndex`] from an `index_raw` request — steps (1), (2) +/// and (4) of the raw accept (normalize, offsets, chunk + kw multisets), pure +/// of `&mut Core`. Call this OUTSIDE the global write lock, then hand the +/// result to [`commit_index_raw`] under the lock. +pub fn prepare_index_raw(req: RawIndexRequest, max_seq_len: Option) -> PreparedRawIndex { + // (1) Normalize newlines once, up front. Everything downstream — offsets, + // chunk line numbers, stored text — is in terms of this LF-only text. + let full_text = normalize_newlines(&req.text); + + // (2) Offset table over the full normalized document. + let line_offsets = build_line_offsets(&full_text); + + // Resolve chunk config against the configured max_seq_len (the hard cap), + // received here as a plain parameter so prepare needs no `Core` at all. + let cfg = ChunkConfig::resolve( + req.target_tokens, + req.max_tokens, + req.overlap_lines, + max_seq_len, + ); + + // (4) Chunk the normalized text into line-snapped segments. + let chunks = chunk_text(&full_text, &cfg); + + // Build the doc's segments + the parallel embed-text list. + let mut segments: Vec = Vec::with_capacity(chunks.len()); + let mut embed_texts: Vec = Vec::with_capacity(chunks.len()); + let mut skipped_now: u64 = 0; + + for chunk in chunks { + // Oversized single line → truncate ONLY the embed text to the hard cap. + // The stored segment text stays whole (get_segment returns it intact). + let embed_full = if chunk.oversized { + truncate_to_tokens(&chunk.text, cfg.max_tokens) + } else { + chunk.text.clone() + }; + let is_blank = embed_full.trim().is_empty(); + if is_blank { + skipped_now += 1; + } + embed_texts.push(if is_blank { String::new() } else { embed_full }); + let kw_counts = token_multiset(&chunk.text); + segments.push(Segment { + segment_id: 0, // placeholder; the real id is allocated at commit + text: chunk.text, + embed_text: None, + line_start: Some(chunk.line_start), + line_end: Some(chunk.line_end), + meta: serde_json::json!({}), + vector: None, + kw_counts, + }); + } + + PreparedRawIndex { + collection: req.collection, + description: req.description, + doc_id: req.doc_id, + name: req.name, + meta: req.meta, + full_text, + line_offsets, + segments, + embed_texts, + skipped_now, + } +} + +/// The locked *commit* half of an `index_raw` accept — steps (3) and (5): +/// allocate real segment ids, store the full normalized text + offset table on +/// the doc (so `get_segment` works the instant this returns — before any +/// embedding), mark the collection `text_ready` / `Building`, upsert the doc, +/// and enqueue the chunked embed jobs (which later fill only the vectors). +/// +/// Returns immediately; embedding happens off-thread, exactly like +/// `index_segments`. The doc is greppable and `get_segment`-able right away. +pub fn commit_index_raw(core: &mut Core, prepared: PreparedRawIndex) -> RawAcceptResult { + let _embedder = ensure_embedder(core); + + let PreparedRawIndex { + collection, + description, + doc_id, + name, + meta, + full_text, + line_offsets, + mut segments, + embed_texts, + skipped_now, + } = prepared; + + // doc_id: caller-supplied or auto-assigned (returned in the ack). + let doc_id = match doc_id { + Some(id) => id, + None => auto_doc_id(core), + }; + + // Allocate the real, store-unique segment ids, keeping the parallel id vec + // for the embed jobs. + let mut segment_ids: Vec = Vec::with_capacity(segments.len()); + for seg in segments.iter_mut() { + seg.segment_id = core.alloc_segment_id(); + segment_ids.push(seg.segment_id); + } + + let segment_count = segments.len(); + + // (3) Build the doc carrying the full text + offset table. + let doc = Document { + doc_id: doc_id.clone(), + name, + meta, + segments, + full_text: Some(full_text), + line_offsets: Some(line_offsets), + }; + + // (5) Short write-locked accept: install text+offsets, mark Building, upsert. + let coll = core + .collections + .entry(collection.clone()) + .or_insert_with(Collection::new); + coll.text_ready = true; + coll.vector_status = VectorStatus::Building; + coll.error = None; + coll.skipped = coll.skipped.saturating_add(skipped_now); + // Label the collection (last non-empty description wins) — same as index_segments. + if let Some(desc) = description { + if !desc.trim().is_empty() { + coll.description = Some(desc); + } + } + // Atomic upsert by doc_id (same semantics as index_segments). + coll.docs.insert(doc_id.clone(), doc); + + // Enqueue the embed work, chunked across the worker pool (bumps + // `pending_jobs` by the chunk count — see `enqueue_embed_chunks`). + enqueue_embed_chunks( + core, + collection.clone(), + doc_id.clone(), + segment_ids, + embed_texts, + ); + + RawAcceptResult { + collection, + doc_id, + segment_count, + } +} + +/// Synchronous accept for `index_raw`: thin prepare+commit wrapper (reading +/// `max_seq_len` from the core it already holds), kept so existing +/// callers/tests keep working. The dispatcher (`lib.rs`) instead runs +/// [`prepare_index_raw`] BEFORE taking the write lock and [`commit_index_raw`] +/// under it, so normalization/offsets/chunking never block concurrent searches. +/// (Test-only in non-test builds, hence the allow.) +#[cfg_attr(not(test), allow(dead_code))] +pub fn accept_index_raw(core: &mut Core, req: RawIndexRequest) -> RawAcceptResult { + let max_seq_len = core.config.max_seq_len; + commit_index_raw(core, prepare_index_raw(req, max_seq_len)) +} + +/// Truncate `text` to roughly `max_tokens` for *embedding only*. Uses the same +/// "~4 chars per token" rule of thumb as [`estimate_tokens`]; cut on a char +/// boundary so we never split a UTF-8 sequence. The original line is preserved +/// elsewhere (stored segment text) — this only bounds what we hand the embedder. +fn truncate_to_tokens(text: &str, max_tokens: usize) -> String { + let max_chars = max_tokens.saturating_mul(4); + if text.chars().count() <= max_chars { + return text.to_string(); + } + text.chars().take(max_chars).collect() +} + +// =========================================================================== +// get_segment — O(1) line-range slice over the offset table +// =========================================================================== + +/// A parsed `get_segment` request. +pub struct GetSegmentRequest { + pub doc_id: String, + pub line_start: u64, + pub line_end: u64, + /// Optional hard cap on how many lines are returned (after clamping). + pub max_lines: Option, +} + +/// Why `get_segment` could not produce a slice. Surfaced as a structural error. +pub enum GetSegmentError { + /// No document with that `doc_id` exists in any collection. + NotFound, + /// The document exists but has no full-text/offset table (an atomic + /// `index_segments` record). Line slicing is not supported for it. + NoLineIndex, +} + +/// Successful `get_segment` result: the sliced text plus the *actual* (clamped) +/// 1-based inclusive line range that was returned. +pub struct GetSegmentResult { + pub doc_id: String, + pub line_start: u64, + pub line_end: u64, + pub text: String, + /// Total number of lines in the document (so the caller can page). + pub line_count: u64, +} + +/// Return the text of lines `[line_start, line_end]` (1-based inclusive) from a +/// document's stored full text via its offset table — O(1) per boundary, then a +/// single substring copy. +/// +/// Clamping: an out-of-range request is clamped to `[1, line_count]` and the +/// *actual* range used is returned. `max_lines`, when set, further caps the +/// returned line count (trimming from the end). A reversed request +/// (`line_start > line_end`) is normalized to a single line at the start. +/// +/// Errors: an unknown `doc_id` → [`GetSegmentError::NotFound`]; a document with +/// no offset table (atomic `index_segments` record) → [`GetSegmentError::NoLineIndex`]. +pub fn get_segment(core: &Core, req: &GetSegmentRequest) -> Result { + // Find the doc across all collections (doc_id is unique per ingest path; the + // first match wins). We scan collections because doc_id is the addressing + // key and the caller doesn't pass a collection here. + let doc = core + .collections + .values() + .find_map(|coll| coll.docs.get(&req.doc_id)) + .ok_or(GetSegmentError::NotFound)?; + + // Must be a raw doc with a full-text/offset table. + let (full_text, offsets) = match (&doc.full_text, &doc.line_offsets) { + (Some(t), Some(o)) => (t, o), + _ => return Err(GetSegmentError::NoLineIndex), + }; + + let line_count = offsets.len() as u64; // ≥ 1 (offsets always has line 1) + + // Clamp the requested range to [1, line_count]. A 0 or reversed request is + // normalized: start is clamped to ≥1, end to ≥start, both to ≤ line_count. + let mut start = req.line_start.clamp(1, line_count); + let mut end = req.line_end.clamp(start, line_count); + + // Respect max_lines (cap the returned line count, trimming from the end). + if let Some(max) = req.max_lines { + let max = max.max(1) as u64; + if end - start + 1 > max { + end = start + max - 1; + } + } + let _ = &mut start; // start is final; kept mutable above for clarity. + + // O(1) byte range: line N starts at offsets[N-1]; the slice runs from the + // start of `start` to the end of `end`. The end of line `end` is the start + // of line `end+1` minus its '\n' (i.e. offsets[end] - 1), or text.len() for + // the last line. + let byte_start = offsets[(start - 1) as usize]; + let byte_end = if (end as usize) < offsets.len() { + // Start of the next line minus the separating '\n'. + offsets[end as usize] - 1 + } else { + full_text.len() + }; + let text = full_text[byte_start..byte_end].to_string(); + + Ok(GetSegmentResult { + doc_id: req.doc_id.clone(), + line_start: start, + line_end: end, + text, + line_count, + }) +} + +/// Normalize line endings to `\n` (CRLF and bare CR → LF). Mirrors the grep +/// module's normalization so the line model is identical everywhere: chunking, +/// the offset table, grep, and get_segment all agree on what a "line" is. +fn normalize_newlines(s: &str) -> String { + s.replace("\r\n", "\n").replace('\r', "\n") +} + +// =========================================================================== +// search (dense) +// =========================================================================== + +/// Which retrieval channel(s) `search` uses. The wire field is `mode`; an +/// absent or unrecognized value defaults to [`SearchMode::Dense`] (unchanged +/// Stage 1 behavior). +/// +/// ## `min_score` semantics differ per mode (read before changing) +/// The three modes produce scores on **incomparable scales**, so `min_score` +/// cannot mean the same thing across them — applying one cosine threshold to an +/// RRF score would be a category error. We therefore define it per mode: +/// * **Dense** — `score` is cosine similarity over L2-normalized vectors, an +/// absolute, meaningful number in `[-1, 1]`. `min_score` is an absolute +/// cosine floor (the Stage 1 behavior, unchanged). +/// * **Keyword** — `score` is the lexical match score (see [`keyword_score`]): +/// a small positive number combining distinct-term coverage and total +/// occurrences. `min_score` is a floor on *that* score (e.g. `min_score: 1.0` +/// keeps only segments matching at least one whole query term). It is NOT a +/// cosine. +/// * **Hybrid** — `score` is the Reciprocal-Rank-Fusion score, `Σ 1/(k+rank)` +/// over the dense and keyword rank lists (`k = 60`). Its magnitude is +/// relative (roughly `0 .. 2/61 ≈ 0.033`) and carries no absolute meaning, +/// so comparing it to a cosine threshold is meaningless. `min_score`, when +/// given, is applied as a floor on the *fused RRF score* — useful only to +/// drop the long tail, never as a similarity cutoff. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SearchMode { + /// Top-k by dot product over normalized vectors (needs embedded vectors). + Dense, + /// Full lexical scan over stored segment text (needs no vectors; works + /// while `vector_status == Building`). + Keyword, + /// Dense + keyword fused with Reciprocal Rank Fusion. + Hybrid, +} + +impl SearchMode { + /// Parse the wire `mode` string. Unknown / absent ⇒ `Dense` (back-compat). + pub fn parse(s: Option<&str>) -> SearchMode { + match s.map(|x| x.trim().to_ascii_lowercase()).as_deref() { + Some("keyword") => SearchMode::Keyword, + Some("hybrid") => SearchMode::Hybrid, + _ => SearchMode::Dense, + } + } +} + +/// Parsed `search` request. +pub struct SearchRequest { + pub query: String, + pub collection: Option, + /// Retrieval channel(s). Defaults to `Dense`. + pub mode: SearchMode, + pub k: usize, + pub min_score: Option, + pub max_per_doc: Option, + pub include_text: bool, + /// Combinable meta filters over the hit's effective meta (empty ⇒ no-op). + pub filter: MetaFilter, +} + +/// One search hit. +pub struct Hit { + pub doc_id: String, + pub name: String, + pub collection: String, + pub meta: serde_json::Value, + pub segment_id: u64, + pub line_start: Option, + pub line_end: Option, + pub score: f32, + pub text: String, +} + +/// Result of a search: ranked hits + a partiality flag (true if any searched +/// collection is still Building, so vectors are still filling in). +pub struct SearchResult { + pub hits: Vec, + pub partial: bool, +} + +/// Standard Reciprocal-Rank-Fusion constant. The canonical value from the RRF +/// paper (Cormack et al. 2009): a larger `k` flattens the contribution of rank +/// position so a high rank in either channel still meaningfully lifts a hit. +const RRF_K: f32 = 60.0; + +/// Tokenize a query for keyword matching: lowercase, then split on any +/// non-alphanumeric character (Unicode-aware), dropping empties. This keeps +/// alphanumeric runs together so exact identifiers survive intact — +/// `"ABC-123"` → `["abc", "123"]`, `"ИНН7701234567"` stays one token — which is +/// exactly the SKU/ИНН case keyword mode exists to serve. The same tokenizer is +/// applied to both the query and the segment text so they match symmetrically. +fn keyword_tokens(text: &str) -> Vec { + text.to_lowercase() + .split(|c: char| !c.is_alphanumeric()) + .filter(|t| !t.is_empty()) + .map(|t| t.to_string()) + .collect() +} + +/// Token *multiset* of a segment's text — `token -> occurrence count` — built +/// once at index time and cached on the [`Segment`]. Uses the exact same +/// tokenizer as [`keyword_tokens`] so cached segment tokens and live query terms +/// match symmetrically. Lets [`keyword_score`] avoid re-tokenizing the segment +/// (and allocating a `Vec`) on every query. +fn token_multiset(text: &str) -> HashMap { + let mut counts: HashMap = HashMap::new(); + for tok in keyword_tokens(text) { + *counts.entry(tok).or_insert(0) += 1; + } + counts +} + +/// Lexical match score of one segment against the (already tokenized) distinct +/// query terms. Returns `0.0` when nothing matches (the segment is then not a +/// keyword hit). +/// +/// Scoring scheme (simple and documented, BM25-lite-ish but deliberately not +/// length-normalized so short exact-identifier segments are not penalized): +/// +/// ```text +/// score = (# distinct query terms present in the segment) +/// + 0.1 * (total occurrences of any query term in the segment) +/// ``` +/// +/// The integer part is **distinct-term coverage** — the dominant signal, so a +/// segment containing more of the query's *distinct* terms always outranks one +/// that merely repeats a single term. The fractional `0.1 * occurrences` part is +/// a small tie-breaker that rewards density without ever letting raw repetition +/// overtake an extra distinct term (since 10 repeats of one term add only 1.0, +/// the same as covering one more distinct term — and coverage is counted first). +/// Tokens are compared for *equality* (whole-token match), so `"db"` does not +/// match inside `"database"`; this is the precise-identifier behavior we want +/// (use `grep` for substring/regex search). +fn keyword_score(seg_counts: &HashMap, query_terms: &[String]) -> f32 { + if query_terms.is_empty() || seg_counts.is_empty() { + return 0.0; + } + let mut distinct_present = 0u32; + let mut total_occurrences = 0u32; + for term in query_terms { + if let Some(&occ) = seg_counts.get(term) { + distinct_present += 1; + total_occurrences += occ; + } + } + distinct_present as f32 + 0.1 * total_occurrences as f32 +} + +/// Character budget of the `preview` the FFI layer emits when `include_text` is +/// false (mirror of the truncation in `lib.rs::hit_to_json`). When a caller does +/// not want full text we materialize only this prefix, so a long segment's text +/// is never cloned in full just to be thrown away at serialization. +const PREVIEW_CHARS: usize = 120; + +/// A scored segment, held by reference — the lightweight intermediate the +/// scanners produce. Carrying borrows (not clones) means scoring the *whole* +/// corpus allocates nothing per segment; the expensive [`Hit`] (which clones +/// `doc.meta` + the segment text) is built only for the ≤k survivors, in +/// [`make_hit`], after top-k selection. The borrow is tied to the `&Core` the +/// search holds for its whole duration, so the refs stay valid until hits are +/// materialized. +struct Scored<'a> { + score: f32, + segment_id: u64, + doc: &'a Document, + seg: &'a Segment, + collection: &'a str, +} + +/// Descending score comparator (NaN treated as lowest), shared by every ranking +/// step so the order semantics are identical across channels. +fn cmp_score_desc(a: f32, b: f32) -> std::cmp::Ordering { + b.partial_cmp(&a).unwrap_or(std::cmp::Ordering::Equal) +} + +/// Materialize one [`Hit`] from a scored ref. This is the ONLY place that clones +/// meta and segment text, and it now runs only for the final survivors. +/// When `include_text` is false we clone just the `PREVIEW_CHARS`-char prefix the +/// wire payload will show, not the whole (possibly large) segment text. +/// +/// `meta` is the *effective* meta (doc overlaid by segment, segment wins) — the +/// same view the meta FILTERS match against, so what filtered a hit in is what +/// the hit echoes back (segment-level labels like a step's `type` stay visible). +fn make_hit(s: &Scored, include_text: bool) -> Hit { + let text = if include_text { + s.seg.text.clone() + } else { + s.seg.text.chars().take(PREVIEW_CHARS).collect() + }; + Hit { + doc_id: s.doc.doc_id.clone(), + name: s.doc.name.clone(), + collection: s.collection.to_string(), + meta: crate::filter::effective_meta(&s.doc.meta, &s.seg.meta), + segment_id: s.seg.segment_id, + line_start: s.seg.line_start, + line_end: s.seg.line_end, + score: s.score, + text, + } +} + +/// Does this collection fall within the request's scope? The scope is a +/// comma-separated list of collection names (the wire side accepts either a +/// single `collection` string or a `collections` array and joins it here): an +/// exact match against any listed name passes. `None` (no filter) ⇒ all +/// collections. Empty names in the list are ignored. Collection names therefore +/// must not contain commas (they don't — they're caller-chosen identifiers). +fn in_scope(want: &Option, cname: &str) -> bool { + match want { + None => true, + Some(list) => list + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .any(|name| name == cname), + } +} + +/// Descending sort of scored refs by score (NaN lowest). Cheap: it reorders +/// `Scored` (a few words each), never `Hit`. +fn sort_scored_desc(scored: &mut [Scored]) { + scored.sort_by(|a, b| cmp_score_desc(a.score, b.score)); +} + +/// Select the top-`k` scored refs **without** a full sort of the candidate set. +/// `select_nth_unstable_by` partitions in O(N) so the k highest land in `[..k]` +/// (unordered); we then sort just those k. For N ≫ k this replaces O(N log N) +/// with O(N + k log k). Tie order among equal scores is unspecified — which is +/// already true of the existing path, since the corpus is walked in `HashMap` +/// iteration order — so no observable contract changes. +fn top_k_scored(mut scored: Vec, k: usize) -> Vec { + if k == 0 { + return Vec::new(); + } + if scored.len() > k { + scored.select_nth_unstable_by(k - 1, |a, b| cmp_score_desc(a.score, b.score)); + scored.truncate(k); + } + sort_scored_desc(&mut scored); + scored +} + +/// Dense channel: embed the query and score every *vectorized* segment in scope +/// by dot product (== cosine over normalized vectors). Applies the meta filter. +/// `min_score`, when given, is an absolute cosine floor. Returns the scored refs +/// (UNSORTED — selection/sorting happens once in [`finalize`]) plus whether any +/// in-scope collection is still `Building`. +/// +/// Segments without a vector (not embedded yet, or skipped) are ignored — dense +/// is the only channel that needs vectors. +fn dense_channel<'a>(core: &'a Core, req: &SearchRequest, apply_min: bool) -> (Vec>, bool) { + let embedder = match core.embedder.as_ref() { + Some(e) => e, + None => return (Vec::new(), false), + }; + let qvec = embedder.embed_query(&req.query); + + let mut partial = false; + let mut scored: Vec = Vec::new(); + for (cname, coll) in core.collections.iter() { + if !in_scope(&req.collection, cname) { + continue; + } + if coll.vector_status == VectorStatus::Building { + partial = true; + } + for doc in coll.docs.values() { + for seg in &doc.segments { + let v = match &seg.vector { + Some(v) => v, + None => continue, // not embedded (or skipped) → ignore + }; + if !req.filter.matches_doc_seg(&doc.meta, &seg.meta) { + continue; + } + let score = dot(&qvec, v); + // Absolute cosine floor — only applied when this is the active + // mode (dense). For hybrid we want the *full* rank list as input + // to fusion, so the caller passes apply_min = false. + if apply_min { + if let Some(min) = req.min_score { + if score < min { + continue; + } + } + } + scored.push(Scored { + score, + segment_id: seg.segment_id, + doc, + seg, + collection: cname, + }); + } + } + } + (scored, partial) +} + +/// Keyword channel: full lexical scan over stored segment **text** — no vectors +/// needed, so it works the instant a doc is accepted (even while `Building`). +/// Scores by [`keyword_score`], drops zero-score (non-matching) segments, and +/// applies the meta filter. `min_score`, when given and `apply_min`, is a floor +/// on the *match score* (not a cosine). Returns the scored refs (UNSORTED). +/// +/// Note `partial` is irrelevant to keyword (it never reads vectors), so this +/// returns only the list; the caller decides the result-level `partial` flag. +fn keyword_channel<'a>(core: &'a Core, req: &SearchRequest, apply_min: bool) -> Vec> { + let query_terms = distinct(keyword_tokens(&req.query)); + if query_terms.is_empty() { + return Vec::new(); // no terms ⇒ nothing can match + } + let mut scored: Vec = Vec::new(); + for (cname, coll) in core.collections.iter() { + if !in_scope(&req.collection, cname) { + continue; + } + for doc in coll.docs.values() { + for seg in &doc.segments { + if !req.filter.matches_doc_seg(&doc.meta, &seg.meta) { + continue; + } + let score = keyword_score(&seg.kw_counts, &query_terms); + if score <= 0.0 { + continue; // no query term present → not a hit + } + if apply_min { + if let Some(min) = req.min_score { + if score < min { + continue; + } + } + } + scored.push(Scored { + score, + segment_id: seg.segment_id, + doc, + seg, + collection: cname, + }); + } + } + } + scored +} + +/// De-duplicate a token list, preserving first-seen order. +fn distinct(tokens: Vec) -> Vec { + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut out = Vec::with_capacity(tokens.len()); + for t in tokens { + if seen.insert(t.clone()) { + out.push(t); + } + } + out +} + +/// Fuse two ranked lists with Reciprocal Rank Fusion. Each segment (identified +/// by its stable `segment_id`) accrues `1/(RRF_K + rank)` for every channel it +/// appears in (`rank` is 0-based here, matching the common RRF formulation). +/// A segment strong in *only one* channel still surfaces — that is the whole +/// point: an exact-identifier match that dense misses, or a semantic match the +/// keyword scan misses, both get fused in. The fused `score` replaces the raw +/// per-channel score; its magnitude is relative (see [`SearchMode`] docs). +fn rrf_fuse<'a>(mut dense: Vec>, mut keyword: Vec>) -> Vec> { + // RRF scores by *rank* within each channel, so each list must be ranked + // descending first. These sorts are over lightweight `Scored` refs (no + // clones), and RRF inherently needs the full per-channel ranking. + sort_scored_desc(&mut dense); + sort_scored_desc(&mut keyword); + + // Accumulate fused scores keyed by segment_id, keeping one representative ref + // per segment (the first we encounter carries the displayable fields). + let mut fused: HashMap = HashMap::new(); + let mut rep: HashMap = HashMap::new(); + + for (rank, s) in dense.into_iter().enumerate() { + *fused.entry(s.segment_id).or_insert(0.0) += 1.0 / (RRF_K + rank as f32); + rep.entry(s.segment_id).or_insert(s); + } + for (rank, s) in keyword.into_iter().enumerate() { + *fused.entry(s.segment_id).or_insert(0.0) += 1.0 / (RRF_K + rank as f32); + rep.entry(s.segment_id).or_insert(s); + } + + // Overwrite each representative's score with its fused RRF score. Left + // UNSORTED — `finalize` does the single top-k selection for the whole search. + rep.into_iter() + .map(|(sid, mut s)| { + s.score = fused.get(&sid).copied().unwrap_or(0.0); + s + }) + .collect() +} + +/// Final stage shared by every mode: select the top `k` (honoring `max_per_doc`) +/// from the scored refs, then materialize a [`Hit`] for each survivor — so the +/// `doc.meta` / text clones happen only for what is actually returned. +/// +/// Without `max_per_doc` we take a bounded top-k via [`top_k_scored`] (no full +/// sort). With `max_per_doc`, capping needs the ranked superset, so we sort the +/// (lightweight) refs once and walk them applying the per-doc cap and the `k` +/// limit — identical semantics to the previous `limit_hits`. +fn finalize(scored: Vec, k: usize, max_per_doc: Option, include_text: bool) -> Vec { + let selected: Vec = match max_per_doc { + None => top_k_scored(scored, k), + Some(max_per_doc) => { + let mut scored = scored; + sort_scored_desc(&mut scored); + let mut per_doc: HashMap<(&str, &str), usize> = HashMap::new(); + let mut kept: Vec = Vec::new(); + for s in scored { + let key = (s.collection, s.doc.doc_id.as_str()); + let count = per_doc.entry(key).or_insert(0); + if *count < max_per_doc { + *count += 1; + kept.push(s); + } + if kept.len() >= k { + break; + } + } + kept + } + }; + selected.iter().map(|s| make_hit(s, include_text)).collect() +} + +/// Mode-aware search over the store. Honors `mode` (`dense` | `keyword` | +/// `hybrid`, default `dense`); all modes respect `collection`, `k`, +/// `max_per_doc`, `include_text`, and the meta filter. +/// +/// `min_score` is applied per the active mode — an absolute cosine floor in +/// dense, a match-score floor in keyword, and a fused-RRF-score floor in hybrid +/// (see [`SearchMode`] for why these are necessarily different scales). +/// +/// `partial` (vectors still filling in) is meaningful only where vectors are +/// read: it is reported for `dense` and `hybrid` when an in-scope collection is +/// still `Building`. For `keyword` it is always `false` — keyword reads only +/// text, which is present the instant a doc is accepted. +pub fn search(core: &Core, req: SearchRequest) -> SearchResult { + let (scored, partial) = match req.mode { + SearchMode::Dense => dense_channel(core, &req, true), + SearchMode::Keyword => { + // Keyword reads only text → never partial, works while Building. + (keyword_channel(core, &req, true), false) + } + SearchMode::Hybrid => { + // Run both channels WITHOUT each one's own min_score (we want the + // full rank lists as fusion input), fuse with RRF, then apply + // min_score as a floor on the fused score below. + let (dense, partial) = dense_channel(core, &req, false); + let keyword = keyword_channel(core, &req, false); + let mut fused = rrf_fuse(dense, keyword); + if let Some(min) = req.min_score { + fused.retain(|h| h.score >= min); + } + (fused, partial) + } + }; + + let hits = finalize(scored, req.k, req.max_per_doc, req.include_text); + SearchResult { hits, partial } +} + +// =========================================================================== +// delete_document / delete_collection — incremental, atomic removal +// =========================================================================== + +/// Outcome of [`delete_document`]. +/// +/// `deleted` is the structural answer (chosen over an error so a delete of an +/// already-absent doc — a common, benign idempotent retry — is a normal `false` +/// result, not a failure the caller must special-case). When `true`, the doc and +/// all its segments are gone; `removed_segments` reports how many segments went +/// away, and `collection_dropped` says whether the doc's collection became empty +/// and was therefore removed (see the empty-collection policy below). +pub struct DeleteDocResult { + pub deleted: bool, + pub collection: Option, + pub removed_segments: usize, + pub collection_dropped: bool, +} + +/// Remove a document (and ALL its segments) by `doc_id`, atomically. +/// +/// Atomicity: the whole removal happens under the single short write lock the +/// dispatcher already holds for this call — readers never observe a torn doc +/// (half its segments gone). Because a `doc_id` is unique across collections +/// (it is the addressing key, as `get_segment` relies on), we scan collections +/// for the first one that owns it and remove it there. +/// +/// In-flight embeds: if the background worker is mid-embedding this doc when the +/// delete lands, its later `apply_job` simply finds the doc absent and drops the +/// result on the floor (the same stale-id path that makes re-ingest atomic) — so +/// a deleted doc never reappears with late-arriving vectors. +/// +/// Empty-collection policy: when removing the doc empties its collection, we +/// **drop the now-empty collection** entirely (rather than keep a ghost entry in +/// `stats`). This keeps `stats` honest — a collection with zero docs and zero +/// segments carries no state worth surfacing — and matches what a caller deleting +/// the last document of a collection intuitively expects. (`delete_collection` +/// exists for the explicit whole-collection case.) +/// +/// Unknown `doc_id` → `deleted: false` (a no-op, not an error). Counters are not +/// touched on a miss; on a hit, the per-collection progress counters are left as +/// historical totals (they describe embedding *attempts*, not current contents), +/// while the surfaced `n_docs` / `n_segments` derive from the live maps and so +/// drop immediately. +pub fn delete_document(core: &mut Core, doc_id: &str) -> DeleteDocResult { + // Find the collection that owns this doc (first match wins; ids are unique). + let owner = core + .collections + .iter() + .find(|(_, coll)| coll.docs.contains_key(doc_id)) + .map(|(name, _)| name.clone()); + + let cname = match owner { + Some(c) => c, + None => { + // Unknown doc → structural no-op result. + return DeleteDocResult { + deleted: false, + collection: None, + removed_segments: 0, + collection_dropped: false, + }; + } + }; + + // Remove the doc; count its segments for the result/stats. + let coll = core + .collections + .get_mut(&cname) + .expect("owner collection just located"); + let removed = coll + .docs + .remove(doc_id) + .map(|d| d.segments.len()) + .unwrap_or(0); + + // Drop the collection if it is now empty (empty-collection policy above). + let collection_dropped = coll.docs.is_empty(); + if collection_dropped { + core.collections.remove(&cname); + } + + // Wake any `wait_until_ready` waiter so a pending wait on a now-dropped + // collection returns promptly instead of hanging to its timeout. + core.progress.1.notify_all(); + + DeleteDocResult { + deleted: true, + collection: Some(cname), + removed_segments: removed, + collection_dropped, + } +} + +/// Outcome of [`delete_collection`]. +/// +/// `deleted` is the structural answer (an unknown collection → `false`, a benign +/// no-op rather than an error). When `true`, `removed_docs` / `removed_segments` +/// report what was cleared. +pub struct DeleteCollectionResult { + pub deleted: bool, + pub removed_docs: usize, + pub removed_segments: usize, +} + +/// Remove an entire collection — all its documents and segments — atomically +/// under the dispatcher's single write lock. +/// +/// In-flight embeds for the collection become harmless no-ops: their +/// `apply_job` finds the collection absent and returns early (the existing +/// reset/dim-change guard). Unknown collection → `deleted: false` (a no-op, not +/// an error), consistent with [`delete_document`]. +pub fn delete_collection(core: &mut Core, collection: &str) -> DeleteCollectionResult { + match core.collections.remove(collection) { + Some(coll) => { + let removed_segments = coll.n_segments(); + let removed_docs = coll.docs.len(); + // Wake waiters so a pending `wait_until_ready` on this collection + // returns promptly (it is gone → never going to be Ready). + core.progress.1.notify_all(); + DeleteCollectionResult { + deleted: true, + removed_docs, + removed_segments, + } + } + None => DeleteCollectionResult { + deleted: false, + removed_docs: 0, + removed_segments: 0, + }, + } +} + +// =========================================================================== +// stats helpers +// =========================================================================== + +/// Aggregate totals for `stats`: (total_docs, total_segments). +pub fn totals(core: &Core) -> (usize, usize) { + let mut docs = 0usize; + let mut segs = 0usize; + for coll in core.collections.values() { + docs += coll.docs.len(); + segs += coll.n_segments(); + } + (docs, segs) +} + +/// The embedder dim, or 0 if not configured. +pub fn current_dim(core: &Core) -> usize { + core.embedder.as_ref().map(|e| e.dim()).unwrap_or(0) +} + +// =========================================================================== +// wait_until_ready — deterministic test/caller helper +// =========================================================================== + +/// Block until `collection`'s `vector_status` is `Ready` (or it vanishes / hits +/// `error`), or until `timeout` elapses. Returns `true` if it observed Ready. +/// +/// Uses the shared progress condvar so it wakes the instant the worker applies a +/// job — no polling, no sleeps. Safe to call from tests and from callers that +/// want a synchronous "indexing done" point. (Used by the test suite today; a +/// future `await_collection` dispatch arm can expose it over FFI.) +#[cfg_attr(not(test), allow(dead_code))] +pub fn wait_until_ready(collection: &str, timeout: std::time::Duration) -> bool { + // Snapshot the condvar handle without holding the index lock across waits. + let progress = { + match CORE.read() { + Ok(c) => c.progress_handle(), + Err(_) => return false, + } + }; + let deadline = std::time::Instant::now() + timeout; + let (lock, cvar) = &*progress; + let mut guard = lock.lock().expect("progress mutex"); + loop { + // Check current state under the index read lock. + if let Ok(core) = CORE.read() { + match core.collections.get(collection) { + Some(c) if c.vector_status == VectorStatus::Ready => return true, + Some(c) if c.error.is_some() => return false, + None => return false, // gone (reset) → never going to be ready + _ => {} + } + } + let now = std::time::Instant::now(); + if now >= deadline { + return false; + } + let remaining = deadline - now; + let (g, timed_out) = cvar + .wait_timeout(guard, remaining) + .expect("progress condvar wait"); + guard = g; + if timed_out.timed_out() { + // Final check after timeout before giving up. + if let Ok(core) = CORE.read() { + if let Some(c) = core.collections.get(collection) { + return c.vector_status == VectorStatus::Ready; + } + } + return false; + } + } +} + +// =========================================================================== +// shutdown +// =========================================================================== + +/// Stop and join the background worker pool — including any threads retired by +/// a pool resize (see [`enqueue_job`]), which already received their Stops and +/// only await the join. Idempotent and safe to call from a C++ teardown path. +/// After this the worker is gone; a later ingest spawns a fresh one lazily. +pub fn shutdown() { + // Take the worker AND the retired handles out of the singleton under the + // write lock, then join them OUTSIDE the lock (joining holds no index lock, + // so a job applying mid-join can still acquire the write lock and finish + // cleanly — joining under the lock would deadlock on exactly that). + let (worker, retired) = match CORE.write() { + Ok(mut c) => (c.worker.take(), std::mem::take(&mut c.retired_workers)), + Err(_) => return, + }; + if let Some(w) = worker { + // One Stop per thread so every worker dequeues exactly one and exits. + for _ in 0..w.handles.len() { + let _ = w.tx.send(WorkerMsg::Stop); + } + for handle in w.handles { + let _ = handle.join(); + } + } + // Retired threads were sent their Stops at retirement time; their queue has + // long since drained, so these joins are (at worst) waiting out one job. + for handle in retired { + let _ = handle.join(); + } +} + +/// Process-global mutex shared by ALL test modules (this one and `lib.rs`). +/// Because every test mutates the single `CORE` singleton, they must run +/// serialized; using one shared lock prevents cross-module races (a `reset` in +/// one test wiping another test's in-flight collection). +#[cfg(test)] +pub static TEST_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::time::Duration; + + /// Acquire the global test mutex so the shared `CORE` singleton isn't raced + /// by tests running in parallel. Each test resets state at the start. + /// + /// Crucially we also **drain the background worker** (`shutdown` joins it) + /// BEFORE resetting: the worker is a process-global thread shared by every + /// test, so a job enqueued by a *previous* test could otherwise apply to a + /// later test's same-named collection and flip it `Ready` prematurely (a + /// latent cross-test race). Draining first guarantees no stale job survives + /// into the next test; the worker respawns lazily on the next ingest. This is + /// test-harness hygiene only — production behaviour is unchanged. + fn test_lock() -> std::sync::MutexGuard<'static, ()> { + let g = TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + // Join any in-flight worker so no stale job bleeds into this test, then + // reset to a clean slate. + shutdown(); + if let Ok(mut c) = CORE.write() { + c.reset(); + } + g + } + + fn seg(text: &str) -> SegmentInput { + SegmentInput { + text: text.to_string(), + embed_text: None, + line_start: None, + line_end: None, + meta: json!({}), + } + } + + fn index(collection: &str, doc_id: &str, texts: &[&str]) { + let req = IndexRequest { + collection: collection.to_string(), + doc_id: doc_id.to_string(), + name: format!("doc {doc_id}"), + meta: json!({}), + segments: texts.iter().map(|t| seg(t)).collect(), + description: None, + }; + let mut c = CORE.write().unwrap(); + accept_index(&mut c, req); + } + + /// Keyword-search latency vs corpus size. `#[ignore]`d (not a correctness + /// test) — run explicitly: + /// cargo test --release perf_keyword_scaling -- --ignored --nocapture + /// + /// Uses only `accept_index`/`search` (signatures common to the pre-perf code) + /// so the SAME test runs in a worktree at the old commit for a true + /// before/after. `embed_text=""` marks every segment skip → no embedding, so + /// this isolates the keyword path (token-multiset cache + top-k selection + + /// deferred Hit materialization). The common term `"alpha"` matches ALL N + /// segments — the worst case for the old "build a full Hit per match, then + /// full-sort" path. + #[test] + #[ignore] + fn perf_keyword_scaling() { + use std::time::Instant; + let _g = test_lock(); + for &n in &[2_000usize, 10_000, 50_000] { + { + let mut c = CORE.write().unwrap(); + c.reset(); + c.embedder = Some(Arc::new(MockEmbedder::new())); + let segments: Vec = (0..n) + .map(|i| SegmentInput { + text: format!("alpha beta gamma segment number {i} unique{i}"), + embed_text: Some(String::new()), + line_start: None, + line_end: None, + meta: json!({}), + }) + .collect(); + accept_index( + &mut c, + IndexRequest { + collection: "perf".to_string(), + doc_id: "d".to_string(), + name: "d".to_string(), + meta: json!({}), + segments, + description: None, + }, + ); + } + let c = CORE.read().unwrap(); + let q = 50u32; + let t0 = Instant::now(); + let mut hits = 0usize; + for _ in 0..q { + let res = search( + &c, + SearchRequest { + query: "alpha".to_string(), + collection: Some("perf".to_string()), + mode: SearchMode::Keyword, + k: 10, + min_score: None, + max_per_doc: None, + include_text: true, + filter: MetaFilter::default(), + }, + ); + hits += res.hits.len(); + } + let el = t0.elapsed(); + println!( + "PERF keyword N={:>6} q={} total={:>11.3?} avg={:>9.3?}/q hits/q={}", + n, + q, + el, + el / q, + hits / q as usize + ); + } + } + + #[test] + fn configure_sets_dim() { + let _g = test_lock(); + let mut c = CORE.write().unwrap(); + let res = configure(&mut c, Config::default()); + assert_eq!(res.dim, MockEmbedder::DEFAULT_DIM); + assert!(c.configured); + assert!(!res.reset_due_to_dim_change); + } + + #[test] + fn reconfigure_different_dim_resets() { + let _g = test_lock(); + // Force a small-dim embedder + some data, then reconfigure to default. + { + let mut c = CORE.write().unwrap(); + c.embedder = Some(Arc::new(MockEmbedder::with_dim(8))); + let coll = c.collections.entry("k".to_string()).or_insert_with(Collection::new); + coll.text_ready = true; + coll.docs.insert( + "d1".to_string(), + Document { + doc_id: "d1".to_string(), + name: "n".to_string(), + meta: json!({}), + segments: vec![], + full_text: None, + line_offsets: None, + }, + ); + } + let mut c = CORE.write().unwrap(); + let res = configure(&mut c, Config::default()); + assert!(res.reset_due_to_dim_change, "dim 8 → 64 with data must reset"); + assert!(c.collections.is_empty(), "index must be cleared on dim change"); + } + + #[test] + fn index_returns_immediately_then_becomes_ready() { + let _g = test_lock(); + index("docs", "d1", &["semantic search over vectors", "another segment here"]); + // Right after accept the collection exists and is Building. + { + let c = CORE.read().unwrap(); + let coll = c.collections.get("docs").unwrap(); + assert!(coll.text_ready); + assert_eq!(coll.vector_status, VectorStatus::Building); + } + // After the worker finishes it flips to Ready. + assert!( + wait_until_ready("docs", Duration::from_secs(5)), + "collection should reach Ready" + ); + let c = CORE.read().unwrap(); + let coll = c.collections.get("docs").unwrap(); + assert_eq!(coll.vector_status, VectorStatus::Ready); + assert_eq!(coll.embedded, 2); + assert_eq!(coll.docs.len(), 1); + } + + #[test] + fn text_is_installed_synchronously_at_accept() { + // Carried-forward fix (Task 1): the doc's TEXT segments must be present + // in the store the instant accept returns — BEFORE the worker embeds. + let _g = test_lock(); + { + // Hold the WRITE lock across accept + inspection. The worker needs the + // write lock to apply vectors, so while we hold it the worker is + // blocked — making "text present, vectors still None, Building" a + // deterministic observation rather than a race. + let mut c = CORE.write().unwrap(); + let req = IndexRequest { + collection: "docs".to_string(), + doc_id: "d1".to_string(), + name: "doc d1".to_string(), + meta: json!({}), + segments: vec![seg("alpha text here"), seg("beta text here")], + description: None, + }; + accept_index(&mut c, req); + + let coll = c.collections.get("docs").unwrap(); + assert!(coll.text_ready, "text must be ready at accept"); + let doc = coll.docs.get("d1").expect("doc installed synchronously"); + assert_eq!(doc.segments.len(), 2, "both text segments present"); + assert!(doc.segments.iter().any(|s| s.text == "alpha text here")); + assert!(doc.segments.iter().any(|s| s.text == "beta text here")); + // Vectors are NOT yet filled — that is the worker's job. + assert!( + doc.segments.iter().all(|s| s.vector.is_none()), + "vectors must still be None right after accept" + ); + // The collection is Building until the worker finishes. + assert_eq!(coll.vector_status, VectorStatus::Building); + } // write lock released here → worker can now apply. + + // And the worker still fills vectors in place afterwards. + assert!(wait_until_ready("docs", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + let doc = c.collections.get("docs").unwrap().docs.get("d1").unwrap(); + assert!( + doc.segments.iter().all(|s| s.vector.is_some()), + "worker must fill the already-installed segments' vectors" + ); + } + + #[test] + fn search_finds_shared_token_doc() { + let _g = test_lock(); + index("docs", "db", &["database connection pool tuning"]); + index("docs", "fruit", &["banana orange apple smoothie"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + + let c = CORE.read().unwrap(); + let res = search( + &c, + SearchRequest { + query: "database connection".to_string(), + collection: Some("docs".to_string()), + mode: SearchMode::Dense, + k: 10, + min_score: None, + max_per_doc: None, + include_text: true, + filter: MetaFilter::default(), + }, + ); + assert!(!res.partial); + assert!(!res.hits.is_empty()); + assert_eq!(res.hits[0].doc_id, "db", "db doc should rank first"); + } + + #[test] + fn search_respects_k_min_score_max_per_doc() { + let _g = test_lock(); + index( + "docs", + "d1", + &["alpha beta gamma", "alpha beta delta", "alpha epsilon"], + ); + index("docs", "d2", &["alpha beta gamma"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + + // k limits total hits. + let res = search( + &c, + SearchRequest { + query: "alpha beta gamma".to_string(), + collection: None, + mode: SearchMode::Dense, + k: 2, + min_score: None, + max_per_doc: None, + include_text: true, + filter: MetaFilter::default(), + }, + ); + assert_eq!(res.hits.len(), 2, "k must cap hits"); + + // max_per_doc limits per document. + let res = search( + &c, + SearchRequest { + query: "alpha beta gamma".to_string(), + collection: None, + mode: SearchMode::Dense, + k: 10, + min_score: None, + max_per_doc: Some(1), + include_text: true, + filter: MetaFilter::default(), + }, + ); + let mut counts: HashMap = HashMap::new(); + for h in &res.hits { + *counts.entry(h.doc_id.clone()).or_insert(0) += 1; + } + assert!(counts.values().all(|&n| n <= 1), "max_per_doc must hold"); + + // min_score filters out low scorers. + let res = search( + &c, + SearchRequest { + query: "alpha beta gamma".to_string(), + collection: None, + mode: SearchMode::Dense, + k: 10, + min_score: Some(0.99), + max_per_doc: None, + include_text: true, + filter: MetaFilter::default(), + }, + ); + assert!( + res.hits.iter().all(|h| h.score >= 0.99), + "all hits must meet min_score" + ); + } + + #[test] + fn top_k_selection_is_descending_and_bounded() { + // Locks in the bounded top-k path (no max_per_doc, N > k): results must be + // exactly k, ranked strictly non-increasing by score — i.e. selection + + // sort preserve ordering even though the candidate set is never fully + // sorted. + let _g = test_lock(); + index( + "docs", + "d1", + &[ + "alpha beta gamma delta", + "alpha beta gamma", + "alpha beta", + "alpha", + "unrelated tokens entirely", + ], + ); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + let res = search( + &c, + SearchRequest { + query: "alpha beta gamma delta".to_string(), + collection: Some("docs".to_string()), + mode: SearchMode::Keyword, + k: 3, + min_score: None, + max_per_doc: None, + include_text: true, + filter: MetaFilter::default(), + }, + ); + assert_eq!(res.hits.len(), 3, "k must bound the result"); + for w in res.hits.windows(2) { + assert!( + w[0].score >= w[1].score, + "hits must be ranked descending: {} then {}", + w[0].score, + w[1].score + ); + } + } + + #[test] + fn include_text_false_caps_hit_text_to_preview() { + // The new `make_hit` clones only the preview-length prefix when the caller + // doesn't want full text, so a long segment's text is never copied whole. + let _g = test_lock(); + let long: String = "слово ".repeat(200); // ~1200 chars, well past PREVIEW_CHARS + index("docs", "d1", &[long.as_str()]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + let res = search( + &c, + SearchRequest { + query: "слово".to_string(), + collection: Some("docs".to_string()), + mode: SearchMode::Keyword, + k: 1, + min_score: None, + max_per_doc: None, + include_text: false, + filter: MetaFilter::default(), + }, + ); + assert_eq!(res.hits.len(), 1); + assert_eq!( + res.hits[0].text.chars().count(), + PREVIEW_CHARS, + "include_text:false must materialize only the preview prefix" + ); + } + + #[test] + fn search_during_building_is_partial() { + let _g = test_lock(); + // Make the collection exist + Building without letting the worker run by + // constructing the Building state directly (no job enqueued). + { + let mut c = CORE.write().unwrap(); + c.embedder = Some(Arc::new(MockEmbedder::new())); + let coll = c.collections.entry("docs".to_string()).or_insert_with(Collection::new); + coll.text_ready = true; + coll.vector_status = VectorStatus::Building; + coll.pending_jobs = 1; + } + let c = CORE.read().unwrap(); + let res = search( + &c, + SearchRequest { + query: "anything".to_string(), + collection: Some("docs".to_string()), + mode: SearchMode::Dense, + k: 5, + min_score: None, + max_per_doc: None, + include_text: true, + filter: MetaFilter::default(), + }, + ); + assert!(res.partial, "search over a Building collection must be partial"); + } + + #[test] + fn upsert_replaces_doc_segments() { + let _g = test_lock(); + index("docs", "d1", &["original unique_alpha_token content"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + // Re-ingest the same doc_id with different text. + index("docs", "d1", &["replaced unique_beta_token content"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + + let c = CORE.read().unwrap(); + let coll = c.collections.get("docs").unwrap(); + assert_eq!(coll.docs.len(), 1, "still one doc"); + let texts: Vec<&str> = coll + .docs + .get("d1") + .unwrap() + .segments + .iter() + .map(|s| s.text.as_str()) + .collect(); + assert!( + texts.iter().any(|t| t.contains("unique_beta_token")), + "new segment present" + ); + assert!( + !texts.iter().any(|t| t.contains("unique_alpha_token")), + "old segment gone" + ); + } + + #[test] + fn blank_segment_is_skipped_and_collection_still_ready() { + let _g = test_lock(); + let req = IndexRequest { + collection: "docs".to_string(), + doc_id: "d1".to_string(), + name: "n".to_string(), + meta: json!({}), + segments: vec![seg("real content here"), seg(" "), seg("")], + description: None, + }; + { + let mut c = CORE.write().unwrap(); + accept_index(&mut c, req); + } + assert!(wait_until_ready("docs", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + let coll = c.collections.get("docs").unwrap(); + assert_eq!(coll.vector_status, VectorStatus::Ready); + assert_eq!(coll.skipped, 2, "two blank segments skipped"); + assert_eq!(coll.embedded, 1, "one real segment embedded"); + } + + #[test] + fn stats_helpers_reflect_state() { + let _g = test_lock(); + index("a", "d1", &["one two three", "alpha beta"]); + index("b", "d2", &["four five"]); + assert!(wait_until_ready("a", Duration::from_secs(5))); + assert!(wait_until_ready("b", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + let (docs, segs) = totals(&c); + assert_eq!(docs, 2); + assert_eq!(segs, 3); + assert_eq!(current_dim(&c), MockEmbedder::DEFAULT_DIM); + } + + #[test] + fn shutdown_then_index_still_works() { + let _g = test_lock(); + index("docs", "d1", &["hello world"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + shutdown(); + // Worker is gone; a new ingest must spawn a fresh one and still finish. + index("docs", "d2", &["second document"]); + assert!( + wait_until_ready("docs", Duration::from_secs(5)), + "ingest after shutdown should respawn worker and complete" + ); + } + + #[test] + fn large_ingest_is_chunked_into_multiple_jobs_and_completes() { + // One ingest of 600 tiny segments must split into ceil(600/256) = 3 + // embed jobs (so a huge doc spreads across the worker pool) and still + // converge: every segment embedded, collection Ready. + let _g = test_lock(); + let n = 600usize; + { + let mut c = CORE.write().unwrap(); + let segments: Vec = + (0..n).map(|i| seg(&format!("tiny segment {i}"))).collect(); + accept_index( + &mut c, + IndexRequest { + collection: "big".to_string(), + doc_id: "d".to_string(), + name: "d".to_string(), + meta: json!({}), + segments, + description: None, + }, + ); + // Still holding the write lock, so no job can have applied yet: + // pending_jobs must equal the chunk count, not 1. + let coll = c.collections.get("big").unwrap(); + let want_chunks = ((n + EMBED_JOB_CHUNK - 1) / EMBED_JOB_CHUNK).max(1) as u32; + assert_eq!( + coll.pending_jobs, want_chunks, + "a {n}-segment ingest must enqueue {want_chunks} chunked jobs" + ); + } + assert!( + wait_until_ready("big", Duration::from_secs(10)), + "all chunked jobs must apply and flip the collection Ready" + ); + let c = CORE.read().unwrap(); + let coll = c.collections.get("big").unwrap(); + assert_eq!(coll.vector_status, VectorStatus::Ready); + assert_eq!(coll.embedded, n as u64, "every segment must be embedded"); + let doc = coll.docs.get("d").unwrap(); + assert!( + doc.segments.iter().all(|s| s.vector.is_some()), + "all segments across all chunks must carry vectors" + ); + } + + /// Minimal multi-session stand-in for the resize test: `bulk_concurrency` + /// is the tunable knob; vectors are valid (finite, non-zero) unit vectors + /// so `apply_job` counts them as embedded. + struct StubEmbedder(usize); + + impl Embedder for StubEmbedder { + fn dim(&self) -> usize { + 4 + } + fn embed_passages(&self, texts: &[String]) -> Vec> { + texts.iter().map(|_| vec![1.0, 0.0, 0.0, 0.0]).collect() + } + fn embed_query(&self, _text: &str) -> Vec { + vec![1.0, 0.0, 0.0, 0.0] + } + fn bulk_concurrency(&self) -> usize { + self.0 + } + } + + #[test] + fn worker_pool_resizes_when_bulk_concurrency_changes() { + let _g = test_lock(); + // Concurrency 2 → the first ingest lazily spawns a 2-thread pool. + { + let mut c = CORE.write().unwrap(); + c.embedder = Some(Arc::new(StubEmbedder(2))); + } + index("rz1", "d1", &["first doc body"]); + { + let c = CORE.read().unwrap(); + assert_eq!( + c.worker.as_ref().map(|w| w.handles.len()), + Some(2), + "pool must match the embedder's bulk concurrency" + ); + assert!(c.retired_workers.is_empty(), "nothing retired yet"); + } + // Swap in a concurrency-4 embedder (what a reconfigure does); the next + // ingest must retire the 2-thread pool and spawn a 4-thread one. + { + let mut c = CORE.write().unwrap(); + c.embedder = Some(Arc::new(StubEmbedder(4))); + } + index("rz2", "d2", &["second doc body"]); + { + let c = CORE.read().unwrap(); + assert_eq!( + c.worker.as_ref().map(|w| w.handles.len()), + Some(4), + "pool must be resized to the new bulk concurrency" + ); + assert_eq!( + c.retired_workers.len(), + 2, + "old threads must be stashed for shutdown to join" + ); + } + // shutdown must join current + retired threads without deadlocking and + // only after every queued job applied → both collections Ready. + shutdown(); + let c = CORE.read().unwrap(); + assert!(c.worker.is_none(), "shutdown takes the live pool"); + assert!(c.retired_workers.is_empty(), "shutdown joins retired threads"); + assert_eq!( + c.collections.get("rz1").unwrap().vector_status, + VectorStatus::Ready, + "pre-resize collection must finish" + ); + assert_eq!( + c.collections.get("rz2").unwrap().vector_status, + VectorStatus::Ready, + "post-resize collection must finish" + ); + } + + // =================================================================== + // index_raw: chunker + offset table (pure-function unit tests) + // =================================================================== + + /// Build a config with a known small target/cap/overlap for deterministic + /// chunking tests independent of the default 300-token budget. + fn cfg(target: usize, cap: usize, overlap: usize) -> ChunkConfig { + ChunkConfig::resolve(Some(target), Some(cap), Some(overlap), None) + } + + #[test] + fn estimate_tokens_uses_max_of_words_and_chars_over_4() { + // Five short words → 5 words; chars/4 is smaller, so words wins. + assert_eq!(estimate_tokens("a b c d e"), 5); + // One long no-space run: words=1 but chars/4 dominates so we don't + // wildly under-count a single huge token. + let long = "x".repeat(40); + assert_eq!(estimate_tokens(&long), 10); // 40 chars / 4 + assert_eq!(estimate_tokens(""), 0); + } + + #[test] + fn offset_table_maps_lines_to_byte_starts() { + // 3 lines, no trailing newline. + let text = "ab\ncde\nf"; + let off = build_line_offsets(text); + assert_eq!(off, vec![0, 3, 7]); // "ab\n"=0..3, "cde\n"=3..7, "f"=7 + // A trailing newline yields a final empty line with its own offset. + let text2 = "ab\ncd\n"; + let off2 = build_line_offsets(text2); + assert_eq!(off2, vec![0, 3, 6]); // last (empty) line starts at len() + } + + #[test] + fn chunker_snaps_to_lines_and_overlaps() { + // Six single-word lines; target 3 tokens means ~3 lines per chunk. + // overlap=1 → the last line of a chunk repeats as the first of the next. + let text = "l1\nl2\nl3\nl4\nl5\nl6"; + let chunks = chunk_text(text, &cfg(3, 100, 1)); + // Each chunk is at most 3 lines (the target), snapped to whole lines. + assert!(chunks.len() >= 2, "must split into multiple chunks"); + for ch in &chunks { + assert!(ch.line_end >= ch.line_start, "range well-formed"); + assert!(!ch.oversized); + // The chunk text is exactly the joined source lines. + let all_lines: Vec<&str> = text.split('\n').collect(); + let joined = + all_lines[(ch.line_start - 1) as usize..=(ch.line_end - 1) as usize].join("\n"); + assert_eq!(ch.text, joined); + } + // Overlap: consecutive chunks must share at least `overlap` line(s) — the + // next chunk starts no later than the previous chunk's last line. + for w in chunks.windows(2) { + assert!( + w[1].line_start <= w[0].line_end, + "consecutive chunks must overlap (got {} then start {})", + w[0].line_end, + w[1].line_start + ); + } + // Coverage: line 1 is in the first chunk, the last line in the last. + assert_eq!(chunks.first().unwrap().line_start, 1); + assert_eq!(chunks.last().unwrap().line_end, 6); + } + + #[test] + fn chunker_oversized_single_line_is_one_chunk() { + // A single line whose token estimate exceeds the hard cap (cap=3) must + // become exactly one oversized chunk covering just that line. + let huge = "tok ".repeat(50); // ~50 whitespace tokens, one line + let text = format!("short before\n{}\nshort after", huge.trim()); + let chunks = chunk_text(&text, &cfg(3, 5, 1)); + // Find the oversized chunk: it must be exactly one line, whole. + let oversized: Vec<&Chunk> = chunks.iter().filter(|c| c.oversized).collect(); + assert_eq!(oversized.len(), 1, "exactly one oversized chunk"); + let o = oversized[0]; + assert_eq!(o.line_start, o.line_end, "oversized chunk is a single line"); + assert!(o.text.starts_with("tok"), "oversized chunk holds the whole line"); + assert_eq!(o.text, huge.trim(), "stored oversized text is the full line"); + } + + #[test] + fn truncate_to_tokens_bounds_embed_text_on_char_boundary() { + let s = "a".repeat(100); + let t = truncate_to_tokens(&s, 5); // 5 tokens * 4 = 20 chars + assert_eq!(t.chars().count(), 20); + // Short text is returned unchanged. + assert_eq!(truncate_to_tokens("hi", 5), "hi"); + } + + // =================================================================== + // get_segment: O(1) slice, clamping, structural errors + // =================================================================== + + /// Accept a raw doc synchronously under the write lock and return its doc_id. + fn index_raw_doc(collection: &str, doc_id: Option<&str>, text: &str) -> String { + let mut c = CORE.write().unwrap(); + let res = accept_index_raw( + &mut c, + RawIndexRequest { + collection: collection.to_string(), + description: None, + doc_id: doc_id.map(String::from), + name: "raw doc".to_string(), + meta: json!({}), + text: text.to_string(), + target_tokens: Some(3), + max_tokens: Some(8), + overlap_lines: Some(1), + }, + ); + res.doc_id + } + + #[test] + fn get_segment_basic_slice_and_immediate_after_accept() { + let _g = test_lock(); + // CRLF on the wire → normalized to LF; get_segment works immediately, + // before any vectors are computed (we never wait_until_ready here). + let id = index_raw_doc("docs", Some("r1"), "line one\r\nline two\r\nline three"); + assert_eq!(id, "r1"); + let c = CORE.read().unwrap(); + let res = get_segment( + &c, + &GetSegmentRequest { + doc_id: "r1".to_string(), + line_start: 2, + line_end: 3, + max_lines: None, + }, + ) + .unwrap_or_else(|_| panic!("slice must succeed")); + // CRLF was normalized away; the slice is exactly lines 2..=3. + assert_eq!(res.text, "line two\nline three"); + assert_eq!(res.line_start, 2); + assert_eq!(res.line_end, 3); + assert_eq!(res.line_count, 3); + } + + #[test] + fn get_segment_out_of_range_clamps_and_returns_actual() { + let _g = test_lock(); + index_raw_doc("docs", Some("r2"), "a\nb\nc"); + let c = CORE.read().unwrap(); + // Request lines 0..=99 → clamp to 1..=3 and report the actual range. + let res = get_segment( + &c, + &GetSegmentRequest { + doc_id: "r2".to_string(), + line_start: 0, + line_end: 99, + max_lines: None, + }, + ) + .unwrap_or_else(|_| panic!("clamped slice must succeed")); + assert_eq!(res.line_start, 1); + assert_eq!(res.line_end, 3); + assert_eq!(res.text, "a\nb\nc"); + } + + #[test] + fn get_segment_respects_max_lines() { + let _g = test_lock(); + index_raw_doc("docs", Some("r3"), "a\nb\nc\nd\ne"); + let c = CORE.read().unwrap(); + let res = get_segment( + &c, + &GetSegmentRequest { + doc_id: "r3".to_string(), + line_start: 2, + line_end: 5, + max_lines: Some(2), + }, + ) + .unwrap_or_else(|_| panic!("max_lines slice must succeed")); + assert_eq!(res.line_start, 2); + assert_eq!(res.line_end, 3, "max_lines caps the returned range"); + assert_eq!(res.text, "b\nc"); + } + + #[test] + fn get_segment_on_atomic_record_is_no_line_index() { + let _g = test_lock(); + // An index_segments doc has no full text / offset table. + index("docs", "atomic1", &["just a segment, no document text"]); + let c = CORE.read().unwrap(); + let err = get_segment( + &c, + &GetSegmentRequest { + doc_id: "atomic1".to_string(), + line_start: 1, + line_end: 1, + max_lines: None, + }, + ); + assert!(matches!(err, Err(GetSegmentError::NoLineIndex))); + } + + #[test] + fn get_segment_unknown_doc_is_not_found() { + let _g = test_lock(); + let c = CORE.read().unwrap(); + let err = get_segment( + &c, + &GetSegmentRequest { + doc_id: "nope".to_string(), + line_start: 1, + line_end: 1, + max_lines: None, + }, + ); + assert!(matches!(err, Err(GetSegmentError::NotFound))); + } + + #[test] + fn index_raw_auto_assigns_doc_id() { + let _g = test_lock(); + // No doc_id supplied → a stable id is auto-assigned and returned. + let id = index_raw_doc("docs", None, "auto id document body"); + assert!(!id.is_empty(), "auto doc_id must be returned"); + let c = CORE.read().unwrap(); + assert!( + c.collections + .get("docs") + .unwrap() + .docs + .contains_key(&id), + "auto-assigned doc is addressable by the returned id" + ); + } + + #[test] + fn index_raw_doc_embeds_in_background() { + // `test_lock()` already drains the shared worker, so no stale job can + // flip our collection Ready before our own embed job applies. + let _g = test_lock(); + index_raw_doc("docs", Some("rb"), "alpha beta\ngamma delta\nepsilon zeta"); + // The worker fills vectors off-thread; collection reaches Ready. + assert!(wait_until_ready("docs", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + let doc = c.collections.get("docs").unwrap().docs.get("rb").unwrap(); + assert!( + doc.segments.iter().any(|s| s.vector.is_some()), + "raw-doc chunks must get vectors from the worker" + ); + // Full text + offset table are retained on the raw doc. + assert!(doc.full_text.is_some()); + assert!(doc.line_offsets.is_some()); + } + + // =================================================================== + // Stage 2: keyword tokenizer + score (pure-function unit tests) + // =================================================================== + + #[test] + fn keyword_tokenizer_splits_on_non_alphanumeric_and_keeps_ids_intact() { + // Lowercased; split on every non-alphanumeric char; empties dropped. + assert_eq!(keyword_tokens("Hello, World!"), vec!["hello", "world"]); + // A hyphenated SKU splits into its alphanumeric runs. + assert_eq!(keyword_tokens("ABC-123"), vec!["abc", "123"]); + // A glued identifier stays one token (the exact-id case keyword serves). + assert_eq!(keyword_tokens("ИНН7701234567"), vec!["инн7701234567"]); + assert!(keyword_tokens(" ,.; ").is_empty()); + } + + #[test] + fn keyword_score_ranks_by_distinct_coverage_then_density() { + let q = distinct(keyword_tokens("alpha beta")); + // Both distinct terms present → coverage 2 (+ density). + let both = keyword_score(&token_multiset("alpha beta gamma"), &q); + // One distinct term, repeated → coverage 1 (+ a little density). + let one_rep = keyword_score(&token_multiset("alpha alpha alpha"), &q); + // No query term → not a hit. + let none = keyword_score(&token_multiset("gamma delta"), &q); + assert!(both > one_rep, "more distinct terms must outrank repetition"); + assert!(one_rep > 0.0, "a present term is a hit"); + assert_eq!(none, 0.0, "no shared term → zero (non-hit)"); + // Whole-token match only: "alph" must not match inside "alpha". + let q2 = distinct(keyword_tokens("alph")); + assert_eq!( + keyword_score(&token_multiset("alpha beta"), &q2), + 0.0, + "no substring matching" + ); + } + + #[test] + fn token_multiset_counts_occurrences() { + let m = token_multiset("Alpha alpha BETA, beta! beta"); + assert_eq!(m.get("alpha"), Some(&2)); + assert_eq!(m.get("beta"), Some(&3)); + assert_eq!(m.get("gamma"), None); + } + + /// Build a `SearchRequest` with the given mode/min_score, scoped to a + /// collection, k large, no per-doc cap, no meta filter. + fn sreq(query: &str, collection: Option<&str>, mode: SearchMode, min_score: Option) -> SearchRequest { + SearchRequest { + query: query.to_string(), + collection: collection.map(String::from), + mode, + k: 50, + min_score, + max_per_doc: None, + include_text: true, + filter: MetaFilter::default(), + } + } + + #[test] + fn keyword_search_works_while_building_without_vectors() { + // Hold the WRITE lock so the worker can't run: text is installed at + // accept, but vectors stay None and the collection stays Building. + let _g = test_lock(); + let res = { + let mut c = CORE.write().unwrap(); + accept_index( + &mut c, + IndexRequest { + collection: "docs".to_string(), + doc_id: "k1".to_string(), + name: "n".to_string(), + meta: json!({}), + segments: vec![seg("the exact sku ABC-123 lives here"), seg("unrelated prose")], + description: None, + }, + ); + let coll = c.collections.get("docs").unwrap(); + assert_eq!(coll.vector_status, VectorStatus::Building); + assert!(coll.docs.get("k1").unwrap().segments.iter().all(|s| s.vector.is_none())); + // Keyword search reads only text → must find the sku segment now. + search(&c, sreq("ABC-123", Some("docs"), SearchMode::Keyword, None)) + }; + assert!(!res.partial, "keyword never reports partial (reads no vectors)"); + assert!(!res.hits.is_empty(), "keyword must work while Building"); + assert_eq!(res.hits[0].doc_id, "k1"); + assert!(res.hits[0].text.contains("ABC-123")); + } + + #[test] + fn hybrid_rrf_surfaces_both_keyword_only_and_dense_only_strong_docs() { + let _g = test_lock(); + // dwin: shares semantic tokens with the query (dense-strong) but NOT the + // exact id token. + // kwin: contains the exact identifier token (keyword-strong) but little + // semantic overlap with the rest of the query. + index("docs", "dwin", &["database connection pooling and tuning guide"]); + index("docs", "kwin", &["sku7701234567 inventory record"]); + index("docs", "noise", &["completely irrelevant banana smoothie"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + let c = CORE.read().unwrap(); + // Query blends a semantic phrase (favors dwin in dense) with the exact id + // (favors kwin in keyword). RRF must surface BOTH above the noise doc. + let res = search( + &c, + sreq("database connection sku7701234567", Some("docs"), SearchMode::Hybrid, None), + ); + let ids: Vec<&str> = res.hits.iter().map(|h| h.doc_id.as_str()).collect(); + assert!(ids.contains(&"dwin"), "dense-strong doc must surface via RRF"); + assert!(ids.contains(&"kwin"), "keyword-strong doc must surface via RRF"); + // The keyword-only-strong doc (kwin) must rank ABOVE the noise doc, which + // pure dense might otherwise have ordered ahead of it. + let pos = |id: &str| ids.iter().position(|x| *x == id); + if let (Some(k), Some(n)) = (pos("kwin"), pos("noise")) { + assert!(k < n, "exact-id doc must outrank noise under fusion"); + } + } + + // =================================================================== + // Stage 2: delete_document / delete_collection (store-level) + // =================================================================== + + #[test] + fn delete_document_removes_doc_and_drops_empty_collection() { + let _g = test_lock(); + index("docs", "d1", &["first segment", "second segment"]); + index("docs", "d2", &["other doc"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + + // Delete d1: removed, 2 segments gone, collection survives (d2 remains). + let res = { + let mut c = CORE.write().unwrap(); + delete_document(&mut c, "d1") + }; + assert!(res.deleted); + assert_eq!(res.collection.as_deref(), Some("docs")); + assert_eq!(res.removed_segments, 2); + assert!(!res.collection_dropped, "collection still has d2"); + { + let c = CORE.read().unwrap(); + let coll = c.collections.get("docs").unwrap(); + assert!(!coll.docs.contains_key("d1")); + assert!(coll.docs.contains_key("d2")); + } + + // Delete the last doc → collection becomes empty and is dropped. + let res = { + let mut c = CORE.write().unwrap(); + delete_document(&mut c, "d2") + }; + assert!(res.deleted); + assert!(res.collection_dropped, "empty collection must be dropped"); + let c = CORE.read().unwrap(); + assert!(!c.collections.contains_key("docs"), "empty collection gone"); + } + + #[test] + fn delete_unknown_document_is_structural_false() { + let _g = test_lock(); + let mut c = CORE.write().unwrap(); + let res = delete_document(&mut c, "ghost"); + assert!(!res.deleted, "unknown doc → deleted:false, not an error"); + assert!(res.collection.is_none()); + assert_eq!(res.removed_segments, 0); + assert!(!res.collection_dropped); + } + + #[test] + fn delete_collection_clears_all_docs_and_unknown_is_false() { + let _g = test_lock(); + index("docs", "d1", &["a", "b"]); + index("docs", "d2", &["c"]); + assert!(wait_until_ready("docs", Duration::from_secs(5))); + + let res = { + let mut c = CORE.write().unwrap(); + delete_collection(&mut c, "docs") + }; + assert!(res.deleted); + assert_eq!(res.removed_docs, 2); + assert_eq!(res.removed_segments, 3); + { + let c = CORE.read().unwrap(); + assert!(!c.collections.contains_key("docs")); + } + + // Unknown collection → structural false (no-op). + let mut c = CORE.write().unwrap(); + let res = delete_collection(&mut c, "missing"); + assert!(!res.deleted); + assert_eq!(res.removed_docs, 0); + assert_eq!(res.removed_segments, 0); + } +} diff --git a/rust-core/src/embed.rs b/rust-core/src/embed.rs new file mode 100644 index 0000000..f370a51 --- /dev/null +++ b/rust-core/src/embed.rs @@ -0,0 +1,256 @@ +//! Embedding seam. +//! +//! Stage 1 builds the entire store / ingest / search pipeline against an +//! [`Embedder`] *trait* rather than a concrete model. This is deliberate: the +//! real model integration (ONNX + tokenizer via fastembed/ort) is a separate +//! later card and drags in heavy native dependencies. By coding to the trait we +//! keep this whole slice pure-Rust and fully unit-testable now, and the real +//! impl slots in later by just implementing [`Embedder`]. +//! +//! Contract for every implementation: +//! * vectors are **L2-normalized**, so cosine similarity reduces to a plain +//! dot product (this is what `search` relies on); +//! * `embed_query` and `embed_passages` use the *same* scheme, so a query and +//! a passage that share tokens land close together in the vector space. + +/// A text → vector embedder. The store/search pipeline only ever sees this +/// trait, never a concrete model. +/// +/// `Send + Sync` is required because the embedder lives inside the process +/// singleton and is called from the background ingest worker thread. +pub trait Embedder: Send + Sync { + /// Dimensionality of the produced vectors. Fixed for the life of the + /// embedder and bound to the index (changing it invalidates all vectors). + fn dim(&self) -> usize; + + /// Embed a batch of passages (the documents being indexed). Returns one + /// L2-normalized vector per input, in the same order. + fn embed_passages(&self, texts: &[String]) -> Vec>; + + /// Embed a batch of passages on behalf of a *pinned* worker thread. + /// + /// `slot` is the stable worker-thread index (0-based, assigned at spawn and + /// fixed for the thread's lifetime). A multi-session implementation maps the + /// slot onto one of its model sessions so each worker thread always uses the + /// *same* session — eliminating the head-of-line blocking a shared/rotating + /// cursor can cause (a worker being assigned a session currently locked by a + /// slow sibling while other sessions sit idle). The default simply forwards + /// to [`Embedder::embed_passages`] (correct for any single-session embedder, + /// including the mock). + fn embed_passages_at(&self, _slot: usize, texts: &[String]) -> Vec> { + self.embed_passages(texts) + } + + /// Embed a single query string. Same scheme as [`Embedder::embed_passages`] + /// so cosine similarity between a query and a passage is meaningful. + fn embed_query(&self, text: &str) -> Vec; + + /// How many bulk embeds this embedder can run concurrently — i.e. how many + /// independent model sessions it holds. The ingest worker spawns this many + /// threads. Defaults to 1 (the mock and any single-session embedder). + fn bulk_concurrency(&self) -> usize { + 1 + } + + /// The most recent bulk/query embed failure, for diagnostics via `stats`. + /// + /// Inside the 1C host process there is no console, so an `eprintln!` on the + /// embed path goes nowhere; implementations that can fail (the real ONNX + /// embedder) record the last error string here instead so the operator can + /// see it in the `stats` payload. `None` means no failure has been observed + /// (the default, and always the answer for the infallible mock). + fn last_error(&self) -> Option { + None + } +} + +/// L2-normalize a vector in place. A zero (or non-finite) vector is left as an +/// all-zeros vector — callers treat such vectors as "no signal" and skip them. +fn l2_normalize(v: &mut [f32]) { + let norm_sq: f32 = v.iter().map(|x| x * x).sum(); + let norm = norm_sq.sqrt(); + if norm > 0.0 && norm.is_finite() { + for x in v.iter_mut() { + *x /= norm; + } + } +} + +/// Deterministic, dependency-free embedder used for the whole Stage 1 slice and +/// its tests. +/// +/// Scheme: lowercase the text, split on ASCII whitespace, hash each token into a +/// bucket in `[0, dim)`, accumulate a count per bucket, then L2-normalize. This +/// is a bag-of-hashed-tokens model: two texts that share tokens map onto +/// overlapping buckets and therefore have a higher dot product (== cosine, since +/// normalized). That property is exactly what makes search ranking testable. +pub struct MockEmbedder { + dim: usize, +} + +impl MockEmbedder { + /// Default dimensionality for the mock. Small on purpose — keeps test + /// vectors cheap while still leaving enough buckets to avoid pathological + /// collisions for short token sets. + pub const DEFAULT_DIM: usize = 64; + + /// Create a mock embedder with the default dimensionality. + pub fn new() -> Self { + Self::with_dim(Self::DEFAULT_DIM) + } + + /// Create a mock embedder with an explicit dimensionality (used by tests + /// that exercise reconfigure-with-different-dim). + pub fn with_dim(dim: usize) -> Self { + // A zero dim would make every vector empty and every score zero; clamp + // to at least 1 so the type stays well-behaved. + Self { dim: dim.max(1) } + } + + /// Hash a single token to a bucket in `[0, dim)` using a small, stable + /// FNV-1a hash. Stable across runs/platforms because it is pure arithmetic + /// over the token bytes (no `DefaultHasher`, whose seed can vary). + fn bucket(&self, token: &str) -> usize { + let mut hash: u64 = 0xcbf29ce484222325; // FNV offset basis + for b in token.bytes() { + hash ^= b as u64; + hash = hash.wrapping_mul(0x100000001b3); // FNV prime + } + (hash % self.dim as u64) as usize + } + + /// Core of the scheme: text → raw bucket counts → L2-normalized vector. + fn embed_one(&self, text: &str) -> Vec { + let mut v = vec![0.0f32; self.dim]; + for token in text.to_lowercase().split_whitespace() { + let idx = self.bucket(token); + v[idx] += 1.0; + } + l2_normalize(&mut v); + v + } +} + +impl Default for MockEmbedder { + fn default() -> Self { + Self::new() + } +} + +impl Embedder for MockEmbedder { + fn dim(&self) -> usize { + self.dim + } + + fn embed_passages(&self, texts: &[String]) -> Vec> { + texts.iter().map(|t| self.embed_one(t)).collect() + } + + fn embed_query(&self, text: &str) -> Vec { + self.embed_one(text) + } +} + +/// Dot product of two equal-length vectors. Because vectors are L2-normalized +/// this equals cosine similarity. Mismatched lengths (which should never happen +/// once `dim` is fixed) yield `0.0` rather than panicking. +pub fn dot(a: &[f32], b: &[f32]) -> f32 { + if a.len() != b.len() { + return 0.0; + } + // Manual 8-lane chunking gives LLVM a trivially auto-vectorizable shape: 8 + // independent f32 accumulators with no cross-lane dependency, so it can lower + // the inner loop to packed SIMD multiply-adds. The previous `.zip().map().sum()` + // form is a single serial f32 accumulator — one long dependency chain LLVM + // will not vectorize (reassociating f32 adds changes results, which the + // optimizer is not allowed to do on its own). Vectors are L2-normalized, so + // this still equals cosine; only the summation grouping changes (negligible, + // and ranking is robust to it). + const LANES: usize = 8; + let mut acc = [0.0f32; LANES]; + let mut ca = a.chunks_exact(LANES); + let mut cb = b.chunks_exact(LANES); + for (xa, xb) in ca.by_ref().zip(cb.by_ref()) { + for l in 0..LANES { + acc[l] += xa[l] * xb[l]; + } + } + let mut sum: f32 = acc.iter().sum(); + // Tail (len % 8) — at most 7 elements. + for (x, y) in ca.remainder().iter().zip(cb.remainder().iter()) { + sum += x * y; + } + sum +} + +#[cfg(test)] +mod tests { + use super::*; + + /// L2 norm of a vector, for asserting normalization. + fn norm(v: &[f32]) -> f32 { + v.iter().map(|x| x * x).sum::().sqrt() + } + + #[test] + fn embeddings_are_deterministic() { + let e = MockEmbedder::new(); + let a = e.embed_query("Hello World foo"); + let b = e.embed_query("Hello World foo"); + assert_eq!(a, b, "same input must produce identical vectors"); + } + + #[test] + fn vectors_are_l2_normalized() { + let e = MockEmbedder::new(); + let v = e.embed_query("alpha beta gamma"); + assert!((norm(&v) - 1.0).abs() < 1e-5, "expected unit norm, got {}", norm(&v)); + } + + #[test] + fn query_and_passage_use_same_scheme() { + let e = MockEmbedder::new(); + let q = e.embed_query("shared token text"); + let p = e.embed_passages(&["shared token text".to_string()]); + assert_eq!(q, p[0], "query and passage of same text must match"); + } + + #[test] + fn shared_tokens_score_higher() { + let e = MockEmbedder::new(); + let query = e.embed_query("database connection pool"); + // This passage shares 2/3 tokens with the query. + let close = e.embed_passages(&["database connection settings".to_string()]); + // This one shares nothing. + let far = e.embed_passages(&["banana orange apple".to_string()]); + + let s_close = dot(&query, &close[0]); + let s_far = dot(&query, &far[0]); + assert!( + s_close > s_far, + "shared-token passage ({s_close}) must outscore unrelated one ({s_far})" + ); + assert!(s_close > 0.0, "shared tokens must give positive similarity"); + } + + #[test] + fn case_insensitive() { + let e = MockEmbedder::new(); + assert_eq!(e.embed_query("Hello"), e.embed_query("hello")); + } + + #[test] + fn empty_text_is_zero_vector() { + let e = MockEmbedder::new(); + let v = e.embed_query(" "); + assert!(v.iter().all(|&x| x == 0.0), "blank text must give zero vector"); + assert_eq!(norm(&v), 0.0); + } + + #[test] + fn dim_is_respected() { + let e = MockEmbedder::with_dim(32); + assert_eq!(e.dim(), 32); + assert_eq!(e.embed_query("x").len(), 32); + } +} diff --git a/rust-core/src/fastembed_embedder.rs b/rust-core/src/fastembed_embedder.rs new file mode 100644 index 0000000..d5eff54 --- /dev/null +++ b/rust-core/src/fastembed_embedder.rs @@ -0,0 +1,665 @@ +//! Real, production [`Embedder`] backed by **fastembed** (ONNX Runtime + the +//! HuggingFace tokenizer). +//! +//! This module is compiled **only** under the `fastembed` cargo feature. Without +//! that feature the crate never pulls in `ort`/`tokenizers`, so the default +//! `cargo test` stays pure-Rust and fast (mock-only). The real model is selected +//! at runtime by `configure` (see `core::configure`) when the caller asks for it. +//! +//! ## Why two model instances (the deliberate architecture) +//! +//! fastembed's embedding call is **`TextEmbedding::embed(&mut self, …)`** — it +//! requires `&mut self` (the ONNX session is mutated per inference). Our +//! [`Embedder`] trait, by contrast, is `&self + Send + Sync`, because the +//! embedder is shared (`Arc`) and called concurrently from the +//! background ingest worker *and* from query threads. To bridge `&self` → +//! `&mut self` we must wrap each `TextEmbedding` in a `Mutex`. +//! +//! A single shared `Mutex` would mean a long **bulk reindex** +//! (embedding thousands of passages on the worker thread) holds the lock and +//! **blocks every incoming query** for the whole reindex — exactly the UI freeze +//! the async architecture exists to avoid. So we deliberately load **two +//! separate model instances**: +//! +//! * `bulk` — a pool of one or more sessions used by the ingest workers: +//! each worker thread is *pinned* to one session via +//! [`Embedder::embed_passages_at`] (slot → session), with +//! [`FastEmbedder::embed_passages`] round-robining as the fallback for +//! callers that have no stable worker slot; +//! * `query` — locked by [`FastEmbedder::embed_query`] (the latency path). +//! +//! Passage embedding and query embedding therefore never contend on the same +//! lock: a multi-minute reindex on `bulk` leaves query latency untouched. The +//! cost is a second copy of the model in memory; the e5-small ONNX file is small +//! (int8/quantized-friendly, dim 384), so the double load is an acceptable +//! trade for never blocking queries. +//! +//! ## e5 prefixes +//! +//! The multilingual-e5 family is trained with asymmetric prefixes: documents are +//! embedded as `"passage: …"` and queries as `"query: …"`. We prepend these here +//! (verified on this machine: a `"query: договор на поставку"` scores ru/en +//! contract passages ~0.86–0.90 and an unrelated sentence ~0.78). Omitting the +//! prefixes measurably degrades ranking, so they are not optional. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; + +use fastembed::{ + EmbeddingModel, ExecutionProviderDispatch, InitOptions, InitOptionsUserDefined, Pooling, + TextEmbedding, TokenizerFiles, UserDefinedEmbeddingModel, +}; + +use crate::embed::Embedder; + +/// e5 query prefix (prepended to every query before embedding). +const QUERY_PREFIX: &str = "query: "; +/// e5 passage prefix (prepended to every document before embedding). +const PASSAGE_PREFIX: &str = "passage: "; + +/// Whitelist of built-in models this core can load, as +/// `(canonical wire name, fastembed variant)` pairs — the source of truth for +/// [`resolve_model`]. The first entry is the default +/// ([`crate::core::DEFAULT_MODEL`]). +/// +/// The NAMES are duplicated in [`crate::core::SUPPORTED_MODEL_NAMES`] on +/// purpose: that list must exist without this feature compiled in (it powers +/// `list_models` and configure-time validation in the mock/lite build), while +/// the `EmbeddingModel` type only exists here. The +/// `supported_models_match_core_whitelist` test pins the two lists together so +/// they can never drift. All three multilingual-e5 variants share the +/// `query:`/`passage:` prefix convention and mean pooling, so the rest of this +/// module needs no per-model branches. +pub fn supported_models() -> &'static [(&'static str, EmbeddingModel)] { + &[ + ("multilingual-e5-small", EmbeddingModel::MultilingualE5Small), + ("multilingual-e5-base", EmbeddingModel::MultilingualE5Base), + ("multilingual-e5-large", EmbeddingModel::MultilingualE5Large), + ] +} + +/// Resolve a `configure`-time model name to its fastembed variant. +/// +/// * empty/whitespace → the default ([`crate::core::DEFAULT_MODEL`]); +/// * otherwise trimmed + ASCII-case-insensitive match against +/// [`supported_models`]; +/// * unknown → `Err` with the same `unknown model ''; supported: ` +/// message shape as [`crate::core::validate_model_name`] (the dispatcher +/// normally rejects unknown names with `bad_model` before this is reached). +pub fn resolve_model(name: &str) -> Result { + let trimmed = name.trim(); + if trimmed.is_empty() { + return resolve_model(crate::core::DEFAULT_MODEL); + } + let lower = trimmed.to_ascii_lowercase(); + supported_models() + .iter() + .find(|(n, _)| *n == lower) + .map(|(_, m)| m.clone()) + .ok_or_else(|| { + format!( + "unknown model '{trimmed}'; supported: {}", + supported_models() + .iter() + .map(|(n, _)| *n) + .collect::>() + .join(", ") + ) + }) +} + +/// Which onnxruntime execution provider to request for inference. +/// +/// ort registers EPs **best-effort**: CPU is the always-present default, so +/// requesting DirectML (`DirectML`/`Auto`) and finding no usable GPU/driver makes +/// ort **log and silently fall back to CPU** — no manual fallback code is needed. +/// `Auto` is the recommended default: try the GPU, transparently degrade to CPU. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Device { + /// Force CPU: register no GPU EP (empty list ⇒ ort's default CPU EP). + Cpu, + /// Request DirectML explicitly (still auto-falls-back to CPU if unavailable). + DirectML, + /// Try DirectML, transparently fall back to CPU. Same EP list as `DirectML`; + /// distinct variant so callers/echoes can tell an explicit `dml` from `auto`. + Auto, +} + +impl Device { + /// Build the ort execution-provider list this device selects. + /// + /// * `Cpu` → `vec![]` (empty ⇒ ort's default CPU EP only). + /// * `DirectML`/`Auto` → `vec![DirectML::default().build()]` — registered + /// best-effort; ort falls back to CPU automatically if DirectML can't + /// initialize (no GPU/driver). fastembed, seeing a DirectML EP, also sets + /// the DML-required session opts (memory_pattern + parallel_execution off). + fn execution_providers(self) -> Vec { + match self { + Device::Cpu => vec![], + Device::DirectML | Device::Auto => vec![ort::ep::DirectML::default().build()], + } + } +} + +/// Per-session ONNX tuning. The two model instances are tuned for *different* +/// workloads, not just split across locks: +/// * **bulk** (background reindex): the configured device (GPU/DirectML when +/// asked) and the bulk of the cores. +/// * **query** (search path): **CPU** + a small intra-op pool. A single short +/// query is latency-bound, and DirectML routinely loses to CPU on it because +/// of host↔device copy overhead; a small thread pool also avoids spin-up cost. +/// +/// Thread budgets are sized so `bulk + query ≤ ncpu` (no oversubscription, which +/// is what let reindex and queries fight over cores under the old all-cores-each +/// default). +#[derive(Clone, Copy)] +struct SessionTuning { + device: Device, + threads: usize, +} + +/// Resolve the bulk worker/session count from the `embed_workers` config knob: +/// `None` ⇒ 1 (single worker, the default); `Some(0)` ⇒ auto — `ncpu/2` clamped +/// to `1..=4`; `Some(n)` ⇒ exactly `n` (≥1). +/// +/// Why the auto cap at 4: measured scaling of multi-session CPU embedding +/// saturates around ~1.4× (the workload is memory-bandwidth bound, not +/// compute bound), so on a big machine an uncapped `ncpu/2` would load extra +/// model copies (one per session, RAM cost) for no additional throughput. An +/// explicit `Some(n)` still gets exactly what it asked for. +fn worker_count(embed_workers: Option) -> usize { + let ncpu = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + match embed_workers { + None => 1, + Some(0) => (ncpu / 2).clamp(1, 4), + Some(n) => (n as usize).max(1), + } +} + +/// Derive (per-bulk-session, query) tunings for `m_workers` bulk sessions. Query +/// gets `min(2, ncpu)` CPU threads. Each bulk session gets the explicit intra-op +/// override if given, else the remaining cores split across the `m_workers` +/// sessions (`(ncpu - query) / m`, ≥1) — so `m × per_session + query ≈ ncpu`, +/// no oversubscription whether you run one big session or several small ones. +/// +/// An explicit `intra_threads` is a *per-session* knob: with `m` sessions the +/// total thread demand is `m × t`, so an innocent-looking `intra_threads: 8` +/// under 4 workers would ask for 32 threads. To keep the no-oversubscription +/// invariant we clamp the explicit value to `(ncpu / m).max(1)` whenever +/// `m > 1`; a single session keeps the historical `min(t, ncpu)` behavior. +fn tunings( + device: Device, + intra_threads: Option, + m_workers: usize, +) -> (SessionTuning, SessionTuning) { + let ncpu = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + let query_threads = ncpu.min(2).max(1); + let m = m_workers.max(1); + let bulk_threads = match intra_threads { + // Explicit override: single session caps at ncpu; multi-session clamps + // to the per-session fair share so m sessions never oversubscribe. + Some(t) if t >= 1 && m <= 1 => (t as usize).min(ncpu), + Some(t) if t >= 1 => (t as usize).min((ncpu / m).max(1)), + // Single worker: one big session, leave the query threads free. Multiple + // workers: spread ALL cores across the `m` sessions (≈ ncpu/m each) so the + // task-parallel pool actually saturates the CPU. + _ if m <= 1 => ncpu.saturating_sub(query_threads).max(1), + _ => (ncpu / m).max(1), + }; + ( + SessionTuning { + device, + threads: bulk_threads, + }, + SessionTuning { + device: Device::Cpu, + threads: query_threads, + }, + ) +} + +/// Production embedder: two separately-loaded fastembed models behind their own +/// mutexes (see the module docs for the two-instance rationale). +pub struct FastEmbedder { + /// One or more model sessions for bulk passage embedding (the background- + /// worker path). A long reindex locks only these, leaving `query` free. With + /// >1 session the ingest worker runs that many threads, each **pinned** to + /// its own session via [`Embedder::embed_passages_at`] (worker slot → + /// session index), so jobs embed concurrently with no cross-session lock + /// contention (cost: one model copy in RAM per session). + bulk_pool: Vec>, + /// Round-robin cursor selecting a bulk session for the *fallback* path — + /// [`Embedder::embed_passages`] callers that have no stable worker slot + /// (e.g. ad-hoc/test callers). The pinned worker path never touches it. + bulk_cursor: AtomicUsize, + /// Model used for low-latency query embedding (the search path). + query: Mutex, + /// Most recent embed failure (poisoned session mutex or an ort inference + /// error), kept for diagnostics. `eprintln!` goes nowhere inside the 1C + /// host process, so `stats` surfaces this via [`Embedder::last_error`]. + last_error: Mutex>, + /// Output dimensionality, probed once at construction. Bound to the index. + dim: usize, + /// ONNX inference sub-batch size handed to fastembed. With `m` concurrent + /// bulk sessions, peak memory is `m × sub_batch × seq` — so we shrink it to + /// `≈256/m` (the fastembed default is 256) to keep total concurrent memory at + /// roughly the single-session level. Otherwise N sessions each padding a + /// 256-row tensor of long passages exhausts RAM and ort errors out. + embed_batch: usize, +} + +impl FastEmbedder { + /// Build a `FastEmbedder` for a built-in `model` (one of + /// [`supported_models`], resolved from its wire name by [`resolve_model`]; + /// the default is **MultilingualE5Small**). + /// + /// fastembed downloads (first run) and caches the ONNX + tokenizer files + /// from HuggingFace — into `cache_dir` when given (created on demand), else + /// into fastembed's default cache location; subsequent runs load from that + /// cache. This is the path the integration test verifies. Loading happens + /// once per session (bulk pool + query — see module docs); all share the + /// same on-disk cache, so only the first incurs a download. + /// + /// `device` selects the execution provider (CPU, or DirectML with automatic + /// CPU fallback — see [`Device`]). + /// + /// Returns a descriptive error string (never panics) so the FFI boundary can + /// surface a clean structural error if model load fails (e.g. offline with a + /// cold cache, or onnxruntime not loadable). + pub fn new_builtin( + model: EmbeddingModel, + cache_dir: Option<&str>, + device: Device, + intra_threads: Option, + embed_workers: Option, + ) -> Result { + // `m` independent bulk sessions + one query session (see module docs). + // The bulk sessions get the configured device + a per-session intra-op + // pool; query gets CPU + a few threads — see `tunings`. + let m = worker_count(embed_workers); + // Multi-worker is a CPU task-parallel strategy: DirectML is a single + // device and N concurrent sessions on it error out (and a GPU already + // parallelizes within ONE session). So force CPU for the bulk pool when + // running more than one worker; a single worker keeps the requested device. + let bulk_device = if m > 1 { Device::Cpu } else { device }; + let (bulk_t, query_t) = tunings(bulk_device, intra_threads, m); + let mut bulk_pool = Vec::with_capacity(m); + for _ in 0..m { + bulk_pool.push(Mutex::new( + Self::load_builtin(model.clone(), cache_dir, bulk_t) + .map_err(|e| format!("load bulk model: {e}"))?, + )); + } + let query = Self::load_builtin(model, cache_dir, query_t) + .map_err(|e| format!("load query model: {e}"))?; + + let mut me = FastEmbedder { + bulk_pool, + bulk_cursor: AtomicUsize::new(0), + query: Mutex::new(query), + last_error: Mutex::new(None), + // Provisional; replaced by the probe below. The built-in e5 family + // is 384/768/1024-dim (small/base/large), but we probe rather than + // hard-code so the value is always truthful for whatever loaded. + dim: 0, + embed_batch: (256 / m).max(8), + }; + me.dim = me.probe_dim()?; + Ok(me) + } + + /// Build a `FastEmbedder` from local ONNX + tokenizer files at `model_path` + /// (the **offline / air-gapped** path, §11.4). `model_path` is a directory + /// laid out like the fastembed HF cache snapshot: + /// + /// ```text + /// /onnx/model.onnx (or /model.onnx) + /// /tokenizer.json + /// /config.json + /// /special_tokens_map.json + /// /tokenizer_config.json + /// ``` + /// + /// Pooling is set to `Mean` (what multilingual-e5 uses); change this if you + /// point it at a CLS-pooled model. `device` selects the execution provider + /// (see [`Device`]). NOTE: this path is implemented but, unlike + /// `new_builtin`, is **not exercised by the integration test** (it needs a + /// pre-staged model directory). It is here so production can run offline. + pub fn new_local( + model_path: &str, + device: Device, + intra_threads: Option, + embed_workers: Option, + ) -> Result { + let m = worker_count(embed_workers); + // Multi-worker = CPU task-parallel strategy (see `new_builtin`): N + // concurrent DirectML sessions error out, so force CPU when m > 1. + let bulk_device = if m > 1 { Device::Cpu } else { device }; + let (bulk_t, query_t) = tunings(bulk_device, intra_threads, m); + let mut bulk_pool = Vec::with_capacity(m); + for _ in 0..m { + bulk_pool.push(Mutex::new( + Self::load_local(model_path, bulk_t).map_err(|e| format!("load bulk model: {e}"))?, + )); + } + let query = + Self::load_local(model_path, query_t).map_err(|e| format!("load query model: {e}"))?; + + let mut me = FastEmbedder { + bulk_pool, + bulk_cursor: AtomicUsize::new(0), + query: Mutex::new(query), + last_error: Mutex::new(None), + dim: 0, + embed_batch: (256 / m).max(8), + }; + me.dim = me.probe_dim()?; + Ok(me) + } + + /// Load one built-in `model` instance (download/cache via HF — rooted at + /// `cache_dir` when given, else fastembed's default cache), requesting + /// `device`'s execution providers (empty ⇒ CPU; DirectML otherwise, with + /// ort's automatic CPU fallback). + fn load_builtin( + model: EmbeddingModel, + cache_dir: Option<&str>, + tuning: SessionTuning, + ) -> Result { + let mut opts = InitOptions::new(model) + .with_show_download_progress(true) + .with_intra_threads(tuning.threads) + .with_execution_providers(tuning.device.execution_providers()); + if let Some(dir) = cache_dir { + // Caller-chosen download/cache root (e.g. next to the component so + // an admin can pre-stage or wipe it). fastembed/hf-hub creates the + // directory on demand. + opts = opts.with_cache_dir(std::path::PathBuf::from(dir)); + } + TextEmbedding::try_new(opts).map_err(|e| e.to_string()) + } + + /// Load one user-defined instance from on-disk ONNX + tokenizer files, + /// requesting `device`'s execution providers (same CPU/DirectML semantics as + /// [`Self::load_builtin`]). + fn load_local(model_path: &str, tuning: SessionTuning) -> Result { + use std::path::Path; + let dir = Path::new(model_path); + let read = |rel: &str| -> Result, String> { + std::fs::read(dir.join(rel)).map_err(|e| format!("read {rel}: {e}")) + }; + + // The ONNX file may live at /onnx/model.onnx (the HF snapshot layout) + // or directly at /model.onnx. Try the nested path first. + let onnx = match std::fs::read(dir.join("onnx").join("model.onnx")) { + Ok(bytes) => bytes, + Err(_) => read("model.onnx")?, + }; + + let tokenizer_files = TokenizerFiles { + tokenizer_file: read("tokenizer.json")?, + config_file: read("config.json")?, + special_tokens_map_file: read("special_tokens_map.json")?, + tokenizer_config_file: read("tokenizer_config.json")?, + }; + + let model = UserDefinedEmbeddingModel::new(onnx, tokenizer_files) + // multilingual-e5 is mean-pooled (matches the built-in path). + .with_pooling(Pooling::Mean); + + TextEmbedding::try_new_from_user_defined( + model, + InitOptionsUserDefined::new() + .with_intra_threads(tuning.threads) + .with_execution_providers(tuning.device.execution_providers()), + ) + .map_err(|e| e.to_string()) + } + + /// Probe the model's output dimensionality by embedding a single short + /// passage and measuring the vector length. Done once at construction so + /// `dim()` is O(1) and always truthful for whatever model was loaded. + fn probe_dim(&self) -> Result { + let mut m = self.bulk_pool[0] + .lock() + .map_err(|_| "embedder mutex poisoned".to_string())?; + let out = m + .embed(vec![format!("{PASSAGE_PREFIX}probe")], None) + .map_err(|e| format!("dim probe embed failed: {e}"))?; + out.first() + .map(|v| v.len()) + .ok_or_else(|| "dim probe returned no vector".to_string()) + } + + /// Record an embed failure so [`Embedder::last_error`] (and therefore the + /// `stats` payload) can surface it — `eprintln!` is invisible inside the 1C + /// host process. Last writer wins: we only keep the most recent failure. + fn note_error(last_error: &Mutex>, msg: String) { + if let Ok(mut g) = last_error.lock() { + *g = Some(msg); + } + } + + /// Run a batch through one of the model instances, prepending `prefix` to + /// each input. On any error (poisoned lock or an ort inference failure) we + /// return one all-zero vector per input: the ingest/search pipeline already + /// treats all-zero vectors as "no signal" (skipped, never indexed, never a + /// hit), so a transient embed failure degrades gracefully instead of + /// panicking across the FFI boundary. Failures are additionally recorded in + /// `last_error` (see [`Self::note_error`]) so they are visible via `stats`. + fn run( + model: &Mutex, + last_error: &Mutex>, + prefix: &str, + texts: &[String], + dim: usize, + batch: usize, + ) -> Vec> { + let prefixed: Vec = texts.iter().map(|t| format!("{prefix}{t}")).collect(); + let mut guard = match model.lock() { + Ok(g) => g, + Err(_) => { + eprintln!("rcore: embed skipped — model mutex poisoned"); + Self::note_error( + last_error, + "embed skipped — model mutex poisoned".to_string(), + ); + return vec![vec![0.0; dim]; texts.len()]; + } + }; + // Explicit sub-batch keeps concurrent-session peak memory bounded. + match guard.embed(prefixed, Some(batch)) { + Ok(vecs) => vecs, + Err(e) => { + // Surface the underlying ort/tokenizer error (was silently + // swallowed). All-zero vectors still signal "no signal" downstream. + let msg = format!("embed failed for {} texts: {e}", texts.len()); + eprintln!("rcore: {msg}"); + Self::note_error(last_error, msg); + vec![vec![0.0; dim]; texts.len()] + } + } + } +} + +impl Embedder for FastEmbedder { + fn dim(&self) -> usize { + self.dim + } + + fn embed_passages(&self, texts: &[String]) -> Vec> { + // Fallback bulk path for callers WITHOUT a stable worker slot (the + // pinned worker threads use `embed_passages_at` instead). Round-robin + // across the bulk session pool so even ad-hoc concurrent callers spread + // over distinct sessions, never touching the `query` session. + let n = self.bulk_pool.len(); + let idx = if n <= 1 { + 0 + } else { + self.bulk_cursor.fetch_add(1, Ordering::Relaxed) % n + }; + Self::run( + &self.bulk_pool[idx], + &self.last_error, + PASSAGE_PREFIX, + texts, + self.dim, + self.embed_batch, + ) + } + + fn embed_passages_at(&self, slot: usize, texts: &[String]) -> Vec> { + // Pinned worker path: worker thread `slot` always uses session + // `slot % n`, so a slow job on one session never blocks a sibling + // worker whose own session is idle (the head-of-line blocking the old + // shared round-robin cursor allowed). The `% n` guard matters: after a + // pool resize a *retired* worker thread with a high slot index may + // briefly run one last drained job against a smaller new pool — the + // modulo keeps that in bounds instead of panicking. + let n = self.bulk_pool.len().max(1); + Self::run( + &self.bulk_pool[slot % n], + &self.last_error, + PASSAGE_PREFIX, + texts, + self.dim, + self.embed_batch, + ) + } + + fn bulk_concurrency(&self) -> usize { + self.bulk_pool.len() + } + + fn embed_query(&self, text: &str) -> Vec { + // Query path: lock `query` only — never blocked by a bulk reindex. + let one = [text.to_string()]; + Self::run( + &self.query, + &self.last_error, + QUERY_PREFIX, + &one, + self.dim, + self.embed_batch, + ) + .into_iter() + .next() + .unwrap_or_else(|| vec![0.0; self.dim]) + } + + fn last_error(&self) -> Option { + // A poisoned slot only means a panicking writer; the stored string (if + // any) is still the best diagnostic we have, so recover it. + match self.last_error.lock() { + Ok(g) => g.clone(), + Err(p) => p.into_inner().clone(), + } + } +} + +#[cfg(test)] +mod model_tests { + use super::*; + + /// Empty/whitespace resolves to the default; matching is trimmed and + /// case-insensitive. (Fast: name resolution only, no model load.) + #[test] + fn resolve_model_default_and_case_insensitive() { + assert_eq!( + resolve_model("").unwrap(), + EmbeddingModel::MultilingualE5Small + ); + assert_eq!( + resolve_model(" ").unwrap(), + EmbeddingModel::MultilingualE5Small + ); + assert_eq!( + resolve_model("Multilingual-E5-BASE").unwrap(), + EmbeddingModel::MultilingualE5Base + ); + assert_eq!( + resolve_model(" multilingual-e5-large ").unwrap(), + EmbeddingModel::MultilingualE5Large + ); + } + + /// Unknown names error with a message that names the offender and lists + /// every supported model (what the dispatcher's `bad_model` error carries). + #[test] + fn resolve_model_unknown_lists_supported() { + let err = resolve_model("bge-zzz").unwrap_err(); + assert!(err.contains("unknown model 'bge-zzz'"), "got: {err}"); + for (name, _) in supported_models() { + assert!(err.contains(name), "error must list '{name}': {err}"); + } + } + + /// Pin this module's name→variant table to the feature-independent name + /// list in `core` (which powers `list_models` / mock-build validation): + /// same names, same order, and the core default resolves to the default + /// variant. This is what lets the two lists exist without drifting. + #[test] + fn supported_models_match_core_whitelist() { + let names: Vec<&str> = supported_models().iter().map(|(n, _)| *n).collect(); + assert_eq!(names, crate::core::SUPPORTED_MODEL_NAMES); + assert_eq!( + resolve_model(crate::core::DEFAULT_MODEL).unwrap(), + EmbeddingModel::MultilingualE5Small + ); + for name in crate::core::SUPPORTED_MODEL_NAMES { + assert!(crate::core::validate_model_name(name).is_ok()); + assert!(resolve_model(name).is_ok(), "core name '{name}' must resolve"); + } + } +} + +#[cfg(test)] +mod conc_tests { + use super::*; + use std::sync::Arc; + + /// Reproduce concurrent multi-session embedding to surface any error (the 1C + /// run failed 7500/7561 under 6 workers). Forces CPU to isolate from DirectML. + /// Needs the offline model staged at target/offline-model-mini (skips if + /// absent). Run: + /// cargo test --release --features fastembed concurrent_multi_session -- --ignored --nocapture + #[test] + #[ignore] + fn concurrent_multi_session_embed() { + let path = "target/offline-model-mini"; + if !std::path::Path::new(path).exists() { + eprintln!("model not staged at {path}; skipping"); + return; + } + let workers = 6usize; + let emb = Arc::new( + FastEmbedder::new_local(path, Device::Cpu, None, Some(workers as u64)) + .expect("load model pool"), + ); + eprintln!("bulk_concurrency = {}", emb.bulk_concurrency()); + let mut handles = vec![]; + for t in 0..workers { + let e = Arc::clone(&emb); + handles.push(std::thread::spawn(move || { + // Long passages (~like real Gherkin scenarios) so this exercises + // the concurrent-session PEAK-MEMORY path, not just short strings. + let long = "Когда я открываю список документов и проверяю фильтр по компании \ + Тогда в списке только документы выбранной компании и колонки заполнены " + .repeat(8); + let texts: Vec = (0..500) + .map(|i| format!("scenario {t} segment {i} unique{i} {long}")) + .collect(); + let v = e.embed_passages(&texts); + let zeros = v.iter().filter(|x| x.iter().all(|&f| f == 0.0)).count(); + eprintln!("thread {t}: {} vecs, {} all-zero (failed)", v.len(), zeros); + zeros + })); + } + let total_zeros: usize = handles.into_iter().map(|h| h.join().unwrap()).sum(); + eprintln!("TOTAL all-zero (failed) = {total_zeros} / {}", workers * 500); + } +} diff --git a/rust-core/src/filter.rs b/rust-core/src/filter.rs new file mode 100644 index 0000000..6c8d23b --- /dev/null +++ b/rust-core/src/filter.rs @@ -0,0 +1,303 @@ +//! Combinable, domain-agnostic meta filters shared by `search` and `grep`. +//! +//! The core stores only free-form `meta` (a JSON object) on documents and on +//! segments — there are **no** domain-specific fields (no `sku`, `inn`, +//! `scenario_name`, …) baked into the schema. Filters therefore operate purely +//! over arbitrary key/value pairs. +//! +//! ## Effective meta +//! +//! A hit is filtered against its *effective* meta: the document-level `meta` +//! merged with the segment-level `meta`, where **segment keys override document +//! keys on collision**. Build it with [`effective_meta`]. (Rationale: a segment +//! is a more specific scope than its document, so the narrower annotation wins.) +//! +//! ## Filter shape (parsed from the request payload's `filter` object) +//! +//! ```json +//! "filter": { +//! "all": { "kind": "manual", "lang": "en" }, // AND over key==value +//! "any": { "status": ["draft", "review"] }, // OR over key==value +//! "tags_all": ["security", "db"], // every tag present +//! "tags_any": ["draft", "wip"] // at least one tag +//! } +//! ``` +//! +//! All four sub-filters are optional and **combinable**; a hit passes only if it +//! satisfies *every* sub-filter that is present (the sub-filters are ANDed with +//! each other, while `any`/`tags_any` are internally ORed). An absent / empty +//! filter matches everything. +//! +//! ### Value matching +//! For a single `key: value` constraint: +//! * if the constraint `value` is an **array**, the constraint is satisfied +//! when the effective meta's value for `key` equals *any* element (so you can +//! express "key is one of …" inline); +//! * otherwise the constraint is satisfied when the effective meta's value for +//! `key` equals the constraint value, OR — when the meta value is itself an +//! array — when the constraint value is a member of it. +//! +//! ### Tags +//! Tags are modelled as a conventional `"tags"` key in meta holding an array of +//! strings (any non-array/absent value is treated as "no tags"). `tags_all` +//! requires every listed tag to be present; `tags_any` requires at least one. + +use serde_json::Value; + +/// A parsed, ready-to-apply set of meta filters. An all-empty `MetaFilter` +/// matches every hit (the common no-filter case is essentially free). +#[derive(Debug, Default, Clone)] +pub struct MetaFilter { + /// AND: every `(key, value)` must match the effective meta. + all: Vec<(String, Value)>, + /// OR: at least one `(key, value)` must match. Empty ⇒ this clause is unused. + any: Vec<(String, Value)>, + /// Every tag in this list must be present in the effective `tags` array. + tags_all: Vec, + /// At least one tag in this list must be present. Empty ⇒ clause unused. + tags_any: Vec, +} + +impl MetaFilter { + /// True when no sub-filter is set, so the filter matches everything. Lets + /// callers cheaply skip building effective meta on the hot path. + pub fn is_empty(&self) -> bool { + self.all.is_empty() + && self.any.is_empty() + && self.tags_all.is_empty() + && self.tags_any.is_empty() + } + + /// Parse a `filter` object from a request payload. A missing/non-object + /// `filter` (the common case) yields an empty filter that matches all. + /// + /// Unknown keys inside `filter` are ignored (forward-compatible). The `all` + /// and `any` clauses must be JSON objects; anything else is ignored. + pub fn parse(payload: &Value) -> MetaFilter { + let filter = match payload.get("filter") { + Some(f) if f.is_object() => f, + _ => return MetaFilter::default(), + }; + + MetaFilter { + all: parse_kv_object(filter.get("all")), + any: parse_kv_object(filter.get("any")), + tags_all: parse_string_array(filter.get("tags_all")), + tags_any: parse_string_array(filter.get("tags_any")), + } + } + + /// Evaluate the filter against an already-computed effective meta object. + /// Returns `true` if the hit passes every present sub-filter. + pub fn matches(&self, effective: &Value) -> bool { + if self.is_empty() { + return true; + } + + // all: every constraint must hold (AND). + for (k, v) in &self.all { + if !constraint_matches(effective, k, v) { + return false; + } + } + + // any: at least one constraint must hold (OR). Only enforced if present. + if !self.any.is_empty() && !self.any.iter().any(|(k, v)| constraint_matches(effective, k, v)) + { + return false; + } + + // tags: derived from the conventional `tags` string array. + if !self.tags_all.is_empty() || !self.tags_any.is_empty() { + let tags = effective_tags(effective); + if !self.tags_all.iter().all(|t| tags.iter().any(|x| x == t)) { + return false; + } + if !self.tags_any.is_empty() + && !self.tags_any.iter().any(|t| tags.iter().any(|x| x == t)) + { + return false; + } + } + + true + } + + /// Convenience: build the effective meta from a `(doc_meta, seg_meta)` pair + /// and test it. Avoids allocating the merged object when the filter is empty. + pub fn matches_doc_seg(&self, doc_meta: &Value, seg_meta: &Value) -> bool { + if self.is_empty() { + return true; + } + let eff = effective_meta(doc_meta, seg_meta); + self.matches(&eff) + } +} + +/// Merge document-level meta with segment-level meta into the *effective* meta: +/// start from the document's keys, then overlay the segment's keys so a segment +/// value wins on any key collision. Non-object inputs contribute nothing. +pub fn effective_meta(doc_meta: &Value, seg_meta: &Value) -> Value { + let mut out = serde_json::Map::new(); + if let Some(obj) = doc_meta.as_object() { + for (k, v) in obj { + out.insert(k.clone(), v.clone()); + } + } + if let Some(obj) = seg_meta.as_object() { + for (k, v) in obj { + out.insert(k.clone(), v.clone()); // segment overrides document + } + } + Value::Object(out) +} + +/// Pull the effective `tags` as a list of string slices. A missing or non-array +/// `tags` value yields an empty list; non-string array elements are skipped. +fn effective_tags(effective: &Value) -> Vec<&str> { + match effective.get("tags").and_then(|t| t.as_array()) { + Some(arr) => arr.iter().filter_map(|x| x.as_str()).collect(), + None => Vec::new(), + } +} + +/// Does the effective meta satisfy a single `key == value` constraint? +/// +/// See the module docs for the array semantics: an array `want` is an inline +/// "one of"; an array meta value is membership-tested against a scalar `want`. +fn constraint_matches(effective: &Value, key: &str, want: &Value) -> bool { + let have = match effective.get(key) { + Some(v) => v, + None => return false, // key absent ⇒ cannot match + }; + + match want { + // Constraint value is an array ⇒ "have equals any element of want". + Value::Array(opts) => opts.iter().any(|opt| value_eq_or_member(have, opt)), + // Scalar constraint ⇒ equality, or membership if `have` is an array. + _ => value_eq_or_member(have, want), + } +} + +/// Equality between a meta value and a wanted scalar, with array-membership: if +/// `have` is itself an array, the constraint matches when `want` is one of its +/// elements (so a multi-valued meta key like `["a","b"]` matches `want = "a"`). +fn value_eq_or_member(have: &Value, want: &Value) -> bool { + if have == want { + return true; + } + if let Some(arr) = have.as_array() { + return arr.iter().any(|el| el == want); + } + false +} + +/// Parse a JSON object into a `Vec<(key, value)>`, preserving every key. A +/// missing or non-object input yields an empty vec. +fn parse_kv_object(v: Option<&Value>) -> Vec<(String, Value)> { + match v.and_then(|x| x.as_object()) { + Some(obj) => obj.iter().map(|(k, val)| (k.clone(), val.clone())).collect(), + None => Vec::new(), + } +} + +/// Parse a JSON array of strings into `Vec`. Non-string elements are +/// skipped; a missing or non-array input yields an empty vec. +fn parse_string_array(v: Option<&Value>) -> Vec { + match v.and_then(|x| x.as_array()) { + Some(arr) => arr.iter().filter_map(|x| x.as_str().map(String::from)).collect(), + None => Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn empty_filter_matches_everything() { + let f = MetaFilter::parse(&json!({})); + assert!(f.is_empty()); + assert!(f.matches(&json!({"anything": "goes"}))); + } + + #[test] + fn effective_meta_segment_overrides_document() { + let doc = json!({"kind": "manual", "lang": "en"}); + let seg = json!({"lang": "ru", "section": "intro"}); + let eff = effective_meta(&doc, &seg); + assert_eq!(eff["kind"], json!("manual")); // from doc + assert_eq!(eff["lang"], json!("ru")); // segment overrides doc + assert_eq!(eff["section"], json!("intro")); // from segment + } + + #[test] + fn all_is_and() { + let f = MetaFilter::parse(&json!({"filter": {"all": {"kind": "manual", "lang": "en"}}})); + assert!(f.matches(&json!({"kind": "manual", "lang": "en"}))); + // Missing one key ⇒ fails the AND. + assert!(!f.matches(&json!({"kind": "manual"}))); + assert!(!f.matches(&json!({"kind": "manual", "lang": "ru"}))); + } + + #[test] + fn any_is_or() { + let f = MetaFilter::parse(&json!({"filter": {"any": {"status": "draft", "kind": "note"}}})); + assert!(f.matches(&json!({"status": "draft"}))); + assert!(f.matches(&json!({"kind": "note"}))); + assert!(!f.matches(&json!({"status": "published", "kind": "manual"}))); + } + + #[test] + fn combined_all_and_any() { + let f = MetaFilter::parse(&json!({"filter": { + "all": {"lang": "en"}, + "any": {"status": ["draft", "review"]} + }})); + // Passes: lang matches AND status is one of the OR options. + assert!(f.matches(&json!({"lang": "en", "status": "review"}))); + // Fails the AND (wrong lang) even though status matches. + assert!(!f.matches(&json!({"lang": "ru", "status": "draft"}))); + // Fails the OR (status not in the set) even though lang matches. + assert!(!f.matches(&json!({"lang": "en", "status": "published"}))); + } + + #[test] + fn array_constraint_is_one_of() { + let f = MetaFilter::parse(&json!({"filter": {"all": {"kind": ["manual", "guide"]}}})); + assert!(f.matches(&json!({"kind": "guide"}))); + assert!(!f.matches(&json!({"kind": "note"}))); + } + + #[test] + fn array_meta_value_membership() { + // A multi-valued meta key matches a scalar constraint by membership. + let f = MetaFilter::parse(&json!({"filter": {"all": {"role": "admin"}}})); + assert!(f.matches(&json!({"role": ["user", "admin"]}))); + assert!(!f.matches(&json!({"role": ["user", "guest"]}))); + } + + #[test] + fn tags_all_and_any() { + let f = MetaFilter::parse(&json!({"filter": { + "tags_all": ["db", "security"], + "tags_any": ["draft", "wip"] + }})); + assert!(f.matches(&json!({"tags": ["db", "security", "wip", "extra"]}))); + // Missing a required tag. + assert!(!f.matches(&json!({"tags": ["db", "wip"]}))); + // Has both required, but none of the any tags. + assert!(!f.matches(&json!({"tags": ["db", "security"]}))); + // No tags at all. + assert!(!f.matches(&json!({}))); + } + + #[test] + fn matches_doc_seg_uses_effective() { + let f = MetaFilter::parse(&json!({"filter": {"all": {"lang": "ru"}}})); + // doc says en, segment overrides to ru ⇒ effective lang is ru ⇒ matches. + assert!(f.matches_doc_seg(&json!({"lang": "en"}), &json!({"lang": "ru"}))); + assert!(!f.matches_doc_seg(&json!({"lang": "en"}), &json!({}))); + } +} diff --git a/rust-core/src/grep.rs b/rust-core/src/grep.rs new file mode 100644 index 0000000..a1bf4df --- /dev/null +++ b/rust-core/src/grep.rs @@ -0,0 +1,309 @@ +//! `grep` — regex search over the *text* of stored segments. +//! +//! Unlike dense `search`, grep needs **no vectors**: it scans the segment text +//! that `accept_index` installs synchronously (Task 1), so it works the instant +//! a doc is accepted — even while the collection's `vector_status` is still +//! `Building`. It is a purely textual, line-oriented operation. +//! +//! ## Engine +//! Matching uses the [`regex`] crate (RE2-style finite automata): linear-time, +//! no backreferences or lookaround, so a user-supplied pattern can never trigger +//! catastrophic backtracking (ReDoS). `RegexBuilder` carries the `ignore_case` +//! and `multiline` flags. A pattern that fails to compile is reported as a +//! *structural* error ([`GrepError::BadPattern`]) — never a panic. +//! +//! ## Line semantics +//! Each segment's text is normalized **CRLF → LF** before scanning. Line numbers +//! are **1-based and inclusive**, counted by `\n` after normalization, relative +//! to each segment's text (segment-local, not document-global). `context_lines` +//! attaches up to N preceding / following lines from the *same* segment. +//! +//! ## Limits +//! `max_matches` caps the total number of returned hits; hitting it sets the +//! top-level `truncated` flag (and `total_found` may then be a lower bound — it +//! counts what we produced up to the cap, see below). `max_per_doc` caps hits +//! per document. Scanning stops early once the global cap is reached. + +use std::borrow::Cow; + +use regex::RegexBuilder; +use serde_json::Value; + +use crate::core::Core; +use crate::filter::MetaFilter; + +/// A parsed `grep` request. +pub struct GrepRequest { + /// The regular expression to search for (RE2 syntax). + pub pattern: String, + /// Optional collection scope; `None` ⇒ scan all collections. + pub collection: Option, + /// Combinable meta filters over each hit's effective meta (doc ∪ segment). + pub filter: MetaFilter, + /// Case-insensitive matching (`RegexBuilder::case_insensitive`). + pub ignore_case: bool, + /// Multi-line mode: `^`/`$` match at line boundaries + /// (`RegexBuilder::multi_line`). Defaults to `true` so anchors behave the way + /// a line-oriented grep user expects. + pub multiline: bool, + /// Lines of context to include before/after each matching line (per side). + pub context_lines: usize, + /// Maximum number of hits to return overall. Reaching it sets `truncated`. + pub max_matches: usize, + /// Optional cap on hits per document. + pub max_per_doc: Option, +} + +/// One grep hit: a single matching line plus optional context, located by +/// document + segment and a segment-local 1-based line number. +pub struct GrepHit { + pub doc_id: String, + pub name: String, + pub collection: String, + pub segment_id: u64, + /// 1-based, inclusive line number within the segment's (LF-normalized) text. + pub line_number: usize, + pub line_text: String, + /// Up to `context_lines` lines immediately before the match (in order). + pub context_before: Vec, + /// Up to `context_lines` lines immediately after the match (in order). + pub context_after: Vec, +} + +/// Result of a grep: hits plus paging/limit metadata. +pub struct GrepResult { + pub hits: Vec, + /// True when `max_matches` was reached and scanning stopped early. + pub truncated: bool, + /// Number of matching lines produced (equals `hits.len()`; when `truncated` + /// it is the capped count, i.e. a lower bound on the true total). + pub total_found: usize, +} + +/// Why a grep could not run. The only failure mode is an uncompilable pattern, +/// surfaced to the dispatcher as a structural `bad_pattern` error. +#[derive(Debug)] +pub enum GrepError { + /// The supplied pattern failed to compile; carries the engine's message. + BadPattern(String), +} + +/// Run a grep over the store. Compiles the pattern first (a compile failure is a +/// structural [`GrepError::BadPattern`], not a panic), then scans the text of +/// every in-scope segment line by line. +/// +/// Borrows `&Core` read-only — the dispatcher holds the read lock. Scanning is +/// linear in total text size and stops as soon as `max_matches` is reached. +pub fn grep(core: &Core, req: &GrepRequest) -> Result { + // Compile the pattern up front. RE2 → no catastrophic backtracking. + let re = RegexBuilder::new(&req.pattern) + .case_insensitive(req.ignore_case) + .multi_line(req.multiline) + .build() + .map_err(|e| GrepError::BadPattern(e.to_string()))?; + + let mut hits: Vec = Vec::new(); + let mut truncated = false; + + // Deterministic-ish ordering: iterate collections, then docs, then segments. + // (HashMap order is unspecified, but each doc's segments keep ingest order.) + 'outer: for (cname, coll) in core.collections.iter() { + if let Some(want) = req.collection.as_ref() { + if want != cname { + continue; + } + } + + for doc in coll.docs.values() { + let mut per_doc = 0usize; + + for seg in &doc.segments { + // grep is a text operation: it needs no vector, so it ignores + // `seg.vector` entirely and works while vectors are Building. + + // Meta filter over the hit's effective meta (segment overrides + // doc). Cheap no-op when no filter was supplied. + if !req.filter.matches_doc_seg(&doc.meta, &seg.meta) { + continue; + } + + // Normalize CRLF → LF, then split into lines for 1-based, + // \n-counted line numbers. `split('\n')` yields exactly + // (count of '\n') + 1 pieces, which is the line model we want. + let normalized = normalize_newlines(&seg.text); + let lines: Vec<&str> = normalized.split('\n').collect(); + + for (idx, line) in lines.iter().enumerate() { + if !re.is_match(line) { + continue; + } + + // Build context windows from the same segment. + let before_start = idx.saturating_sub(req.context_lines); + let context_before: Vec = lines[before_start..idx] + .iter() + .map(|s| s.to_string()) + .collect(); + let after_end = (idx + 1 + req.context_lines).min(lines.len()); + let context_after: Vec = lines[idx + 1..after_end] + .iter() + .map(|s| s.to_string()) + .collect(); + + hits.push(GrepHit { + doc_id: doc.doc_id.clone(), + name: doc.name.clone(), + collection: cname.clone(), + segment_id: seg.segment_id, + line_number: idx + 1, // 1-based, inclusive + line_text: line.to_string(), + context_before, + context_after, + }); + + per_doc += 1; + + // Global cap → stop everything and flag truncation. + if hits.len() >= req.max_matches { + truncated = true; + break 'outer; + } + // Per-doc cap → move on to the next document. + if let Some(max) = req.max_per_doc { + if per_doc >= max { + break; + } + } + } + + // If the per-doc cap was hit mid-segment, skip this doc's + // remaining segments too. + if let Some(max) = req.max_per_doc { + if per_doc >= max { + break; + } + } + } + } + } + + let total_found = hits.len(); + Ok(GrepResult { + hits, + truncated, + total_found, + }) +} + +/// Normalize line endings to `\n`: turn CRLF and bare CR into LF so line numbers +/// (counted by `\n`) are stable regardless of the source's newline convention. +/// +/// Fast path: a single pass first checks for any `\r`. The common case — text +/// that is already LF-only (every `index_raw` segment, plus any LF source) — has +/// no CR, so we **borrow** the input with zero allocation instead of running two +/// `replace` passes that would each allocate a fresh `String`. Only CR-bearing +/// text takes the owned, two-replace path. +fn normalize_newlines(s: &str) -> Cow<'_, str> { + if !s.as_bytes().contains(&b'\r') { + return Cow::Borrowed(s); + } + // Replace CRLF first, then any remaining lone CR, so "\r\n" doesn't become + // two line breaks. + Cow::Owned(s.replace("\r\n", "\n").replace('\r', "\n")) +} + +/// Serialize one grep hit to JSON. `context_before` / `context_after` are only +/// included when non-empty, keeping the common (no-context) payload compact. +pub fn hit_to_json(h: &GrepHit) -> Value { + let mut obj = serde_json::json!({ + "doc_id": h.doc_id, + "name": h.name, + "collection": h.collection, + "segment_id": h.segment_id, + "line_number": h.line_number, + "line_text": h.line_text, + }); + if !h.context_before.is_empty() { + obj["context_before"] = serde_json::json!(h.context_before); + } + if !h.context_after.is_empty() { + obj["context_after"] = serde_json::json!(h.context_after); + } + obj +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{accept_index, IndexRequest, SegmentInput, VectorStatus, CORE, TEST_LOCK}; + use serde_json::json; + + #[test] + fn normalize_handles_crlf_and_cr() { + // CR-bearing input → owned, fully normalized. + assert_eq!(normalize_newlines("a\r\nb\rc\nd").as_ref(), "a\nb\nc\nd"); + assert!(matches!(normalize_newlines("a\r\nb"), Cow::Owned(_))); + // LF-only input → borrowed with zero allocation. + assert!(matches!(normalize_newlines("a\nb\nc"), Cow::Borrowed(_))); + } + + /// Build a default `GrepRequest` for `pattern`, scoped to `collection`. + fn req(pattern: &str, collection: Option<&str>) -> GrepRequest { + GrepRequest { + pattern: pattern.to_string(), + collection: collection.map(String::from), + filter: MetaFilter::default(), + ignore_case: false, + multiline: true, + context_lines: 0, + max_matches: 200, + max_per_doc: None, + } + } + + #[test] + fn grep_finds_text_while_collection_is_building() { + // Ties Task 1 + Task 3 together, deterministically: hold the WRITE lock so + // the background worker cannot run, install text via `accept_index`, then + // grep the text while `vector_status` is still `Building` and vectors are + // still `None`. No `wait_until_ready`, no race. + let _g = TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + // The mutating section runs under the write lock; we drop it before grep + // (grep takes a read lock) but assert Building/None *while still holding + // it*, which the blocked worker cannot have changed. + { + let mut c = CORE.write().unwrap(); + c.reset(); + accept_index( + &mut c, + IndexRequest { + collection: "docs".to_string(), + doc_id: "b1".to_string(), + name: "n".to_string(), + meta: json!({}), + segments: vec![SegmentInput { + text: "needle in the haystack".to_string(), + embed_text: None, + line_start: None, + line_end: None, + meta: json!({}), + }], + description: None, + }, + ); + let coll = c.collections.get("docs").unwrap(); + assert_eq!(coll.vector_status, VectorStatus::Building); + let doc = coll.docs.get("b1").unwrap(); + assert!(doc.segments.iter().all(|s| s.vector.is_none())); + } + // Now grep under a read lock. Because the prior write section never let + // the worker apply (it was blocked on the write lock), the collection may + // *or may not* have flipped Ready by now — either way grep over text works + // and must find the needle. + let c = CORE.read().unwrap(); + let res = grep(&c, &req("needle", Some("docs"))).unwrap(); + assert_eq!(res.hits.len(), 1, "grep must find text regardless of vectors"); + assert_eq!(res.hits[0].doc_id, "b1"); + assert_eq!(res.hits[0].line_number, 1); + } +} diff --git a/rust-core/src/lib.rs b/rust-core/src/lib.rs new file mode 100644 index 0000000..9c96faf --- /dev/null +++ b/rust-core/src/lib.rs @@ -0,0 +1,2141 @@ +//! `rcore` — the Rust search core for the 1C MCP component. +//! +//! =========================================================================== +//! FFI MEMORY-OWNERSHIP CONVENTION (read before touching this module) +//! =========================================================================== +//! Strings cross the C ABI boundary as JSON. The ownership rules are strict and +//! enforced here so the C++ side cannot leak, double-free, or free across CRTs: +//! +//! 1. Every `*const c_char` / `*mut c_char` returned to C is allocated by +//! Rust via `CString::into_raw`. Rust hands ownership to the caller. +//! +//! 2. The caller MUST return that pointer to `rcore_free_string` exactly once +//! to release it. `rcore_free_string` reclaims it with `CString::from_raw` +//! and drops it — freeing on Rust's allocator/CRT, the same one that +//! allocated it. The C++ side must NEVER call `free`/`delete` on these +//! pointers (that would be a cross-CRT free → heap corruption). +//! +//! 3. Input pointers (`method`, `payload_json`) are *borrowed* from C. Rust +//! copies what it needs and never frees them. +//! +//! 4. Null and double-free are guarded: `rcore_free_string(NULL)` is a no-op; +//! passing a non-null pointer twice is a use-after-free and is the +//! caller's contract to avoid (the C++ `RustString` RAII wrapper in +//! RustCore.h enforces single-free on that side). +//! +//! The boundary is also panic-free: every `extern "C"` function wraps its body +//! in `catch_unwind`, because unwinding across the FFI boundary is undefined +//! behaviour. On an unexpected panic we return a structural JSON error. +//! =========================================================================== + +mod core; +mod embed; +// The real fastembed-backed embedder is compiled only under the `fastembed` +// feature so the default (mock-only) build never pulls in ort/tokenizers. +#[cfg(feature = "fastembed")] +mod fastembed_embedder; +mod filter; +mod grep; +mod protocol; + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; +use std::panic::{self, AssertUnwindSafe}; + +use serde_json::{json, Value}; + +use crate::core::{ + self as store, Config, GetSegmentRequest, IndexRequest, RawIndexRequest, SearchMode, + SearchRequest, SegmentInput, CORE, +}; +use crate::filter::MetaFilter; +use crate::grep::GrepRequest; +use crate::protocol::{codes, Envelope}; + +/// ABI revision of this boundary. Bump when the FFI surface changes shape so the +/// C++ side can detect a mismatch at runtime (via `rcore_version`). +const ABI_VERSION: u32 = 1; + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +/// Move an owned `String` across the boundary as a heap `char*` owned by C. +/// +/// Returns a pointer that MUST be freed by `rcore_free_string`. If `s` contains +/// an interior NUL byte (which valid JSON never does) we substitute a constant +/// error JSON so we still return a freeable, well-formed C string. +fn into_c_string(s: String) -> *mut c_char { + match CString::new(s) { + Ok(cs) => cs.into_raw(), + Err(_) => CString::new( + r#"{"ok":false,"error":{"code":"internal","message":"interior NUL in response"}}"#, + ) + .expect("static error string is NUL-free") + .into_raw(), + } +} + +/// Borrow a C string as a Rust `&str` without taking ownership. Returns `None` +/// for a null pointer or non-UTF-8 input. +/// +/// # Safety +/// `ptr` must be null or point to a valid NUL-terminated C string that stays +/// alive for the duration of the borrow. +unsafe fn borrow_c_str<'a>(ptr: *const c_char) -> Option<&'a str> { + if ptr.is_null() { + return None; + } + CStr::from_ptr(ptr).to_str().ok() +} + +/// Run `f`, converting any panic into a structural JSON error envelope. This is +/// the single place that keeps panics from crossing the C ABI. +fn guard String>(f: F) -> String { + match panic::catch_unwind(AssertUnwindSafe(f)) { + Ok(s) => s, + Err(_) => Envelope::err(codes::INTERNAL, "panic in core").to_json_string(), + } +} + +/// Lock-free process-global counter of dispatched calls, surfaced via `stats`. +/// Kept OUT of the `RwLock`-guarded store so a `search`/`grep` never has to take +/// the exclusive write lock merely to bump a counter (which would serialize +/// otherwise-concurrent readers). +static CALLS_HANDLED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +// --------------------------------------------------------------------------- +// Method dispatch (pure Rust, no FFI types — easy to unit-test) +// --------------------------------------------------------------------------- + +/// Route a `(method, payload)` pair to its handler and return the JSON response +/// string. Unknown methods yield a structural `unknown_method` error rather +/// than panicking. Stage 1 adds `configure` / `index_segments` / `search` arms. +fn dispatch(method: &str, payload_json: &str) -> String { + // Parse the payload up front. An empty payload is treated as JSON null so + // stub methods that ignore their input still work with `""`/`"null"`. + let payload: Value = if payload_json.trim().is_empty() { + Value::Null + } else { + match serde_json::from_str(payload_json) { + Ok(v) => v, + Err(e) => { + return Envelope::err(codes::BAD_PAYLOAD, format!("invalid JSON payload: {e}")) + .to_json_string(); + } + } + }; + + // Count every dispatched call (surfaced via `stats`) WITHOUT taking the + // index write lock — a lock-free atomic, so concurrent search/grep calls + // don't serialize on the exclusive lock just to bump a counter. + CALLS_HANDLED.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + let envelope = match method { + // Liveness probe. Echoes the payload back so the C++ side can verify a + // full JSON round-trip through the boundary. + "ping" => Envelope::ok(json!({ "pong": true, "echo": payload })), + + // Report process-singleton state: the cold-start progress surfaced to + // the 1C form. Per-collection two-axis state + counters, plus totals. + "stats" => { + let core = match CORE.read() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned").to_json_string() + } + }; + + // Per-collection two-axis state machine + progress counters. + let mut collections = serde_json::Map::new(); + for (name, coll) in core.collections.iter() { + let mut obj = json!({ + "text_ready": coll.text_ready, + "vector_status": coll.vector_status.as_str(), + "embedded": coll.embedded, + "failed": coll.failed, + "skipped": coll.skipped, + "n_docs": coll.docs.len(), + "n_segments": coll.n_segments(), + }); + // `error` only present when a fatal fault occurred. + if let Some(err) = &coll.error { + obj["error"] = json!(err); + } + // `description` only present when the collection was labelled. + if let Some(desc) = &coll.description { + obj["description"] = json!(desc); + } + collections.insert(name.clone(), obj); + } + + let (total_docs, total_segments) = store::totals(&core); + Envelope::ok(json!({ + "configured": core.configured, + "dim": store::current_dim(&core), + "n_docs": total_docs, + "n_segments": total_segments, + "collections": collections, + "callsHandled": CALLS_HANDLED.load(std::sync::atomic::Ordering::Relaxed), + // Most recent bulk/query embed failure, surfaced because + // eprintln! is invisible inside the 1C host process. Null when + // none has occurred (or for the infallible mock embedder). + "last_embed_error": core.embedder.as_ref().and_then(|e| e.last_error()), + })) + } + + // AI-facing discovery: a flat array of the collections that exist, each + // with its description + counts + state, so a client can choose which + // collection(s) to search (then pass them as `collections` to `search`). + "list_collections" => { + let core = match CORE.read() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned").to_json_string() + } + }; + let mut list: Vec = core + .collections + .iter() + .map(|(name, coll)| { + json!({ + "name": name, + "description": coll.description, + "n_docs": coll.docs.len(), + "n_segments": coll.n_segments(), + "vector_status": coll.vector_status.as_str(), + "text_ready": coll.text_ready, + }) + }) + .collect(); + // Stable order (HashMap iteration is otherwise nondeterministic). + list.sort_by(|a, b| { + a["name"] + .as_str() + .unwrap_or("") + .cmp(b["name"].as_str().unwrap_or("")) + }); + Envelope::ok(json!({ "collections": list })) + } + + // Discovery of the BUILT-IN embedding models `configure` accepts in its + // `model` field, so a client can render a dropdown. Static data (no + // lock, no state): works before `configure` and identically with or + // without the `fastembed` feature — a lite/mock build still knows the + // names even though it cannot load the models. + "list_models" => Envelope::ok(json!({ + "default": store::DEFAULT_MODEL, + "models": store::SUPPORTED_MODEL_NAMES, + })), + + // Clear mutable state back to defaults. Idempotent. + "reset" => { + match CORE.write() { + Ok(mut core) => core.reset(), + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned").to_json_string() + } + } + Envelope::ok(json!({ "reset": true })) + } + + // Load/select the embedder and fix `dim`. Idempotent; reconfiguring + // with a different dim while data is indexed resets the index. + "configure" => match parse_config(&payload) { + // An unknown built-in `model` name is a structural `bad_model` + // error returned BEFORE the store is touched — never a silent + // fallback. Only the NAME is checked up front; a known model that + // then fails to LOAD keeps the existing degrade-to-mock semantics + // inside `build_embedder` (non-fatal, dim reverts to the mock's). + Ok(config) => match validate_builtin_model(&config) { + Err(msg) => Envelope::err(codes::BAD_MODEL, msg), + Ok(()) => { + let mut core = match CORE.write() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + let res = store::configure(&mut core, config); + Envelope::ok(json!({ + "configured": true, + "dim": res.dim, + "reset": res.reset_due_to_dim_change, + "model": core.config.model, + "model_path": core.config.model_path, + "cache_dir": core.config.cache_dir, + "normalize": core.config.normalize, + "max_seq_len": core.config.max_seq_len, + "device": core.config.device, + "intra_threads": core.config.intra_threads, + "embed_workers": core.config.embed_workers, + })) + } + }, + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + // Asynchronous ingest: the CPU-heavy prepare (token multisets over all + // segment text) runs BEFORE the write lock so concurrent searches are + // never blocked behind it; only the short commit (id allocation, doc + // install, job enqueue) holds the lock. The background worker embeds + // off-thread. Returns immediately. + "index_segments" => match parse_index_request(&payload) { + Ok(req) => { + let prepared = store::prepare_index(req); + let mut core = match CORE.write() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + let res = store::commit_index(&mut core, prepared); + Envelope::ok(json!({ + "accepted": true, + "collection": res.collection, + "segment_count": res.segment_count, + })) + } + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + // Asynchronous raw-document ingest: normalization, the line offset + // table, and chunking — the O(text) work — run in prepare BEFORE the + // write lock; only the short commit (id allocation, full-text + chunk + // install, job enqueue) holds it. The worker embeds off-thread. Returns + // immediately, with the (possibly auto-assigned) doc_id so the caller + // can upsert/delete. + "index_raw" => match parse_raw_index_request(&payload) { + Ok(req) => { + // Read max_seq_len under a brief read lock, then drop it for the + // heavy prepare. The TOCTOU vs a concurrent reconfigure is + // accepted (worst case: chunk sizing off for this one doc). + let max_seq_len = match CORE.read() { + Ok(c) => c.config.max_seq_len, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + let prepared = store::prepare_index_raw(req, max_seq_len); + let mut core = match CORE.write() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + let res = store::commit_index_raw(&mut core, prepared); + Envelope::ok(json!({ + "accepted": true, + "collection": res.collection, + "doc_id": res.doc_id, + "segment_count": res.segment_count, + })) + } + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + // O(1) line-range slice of a raw document's stored full text via the + // offset table. Works the instant after accept (text+offsets are + // synchronous). Out-of-range clamps and returns the ACTUAL range used. + // An atomic index_segments record (no offset table) → no_line_index. + "get_segment" => match parse_get_segment_request(&payload) { + Ok(req) => { + let core = match CORE.read() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + match store::get_segment(&core, &req) { + Ok(res) => Envelope::ok(json!({ + "doc_id": res.doc_id, + "line_start": res.line_start, + "line_end": res.line_end, + "line_count": res.line_count, + "text": res.text, + })), + Err(store::GetSegmentError::NotFound) => Envelope::err( + codes::NOT_FOUND, + format!("no document with doc_id '{}'", req.doc_id), + ), + Err(store::GetSegmentError::NoLineIndex) => Envelope::err( + codes::NO_LINE_INDEX, + format!( + "doc_id '{}' has no line index (not an index_raw document)", + req.doc_id + ), + ), + } + } + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + // Dense top-k search over normalized vectors (dot product). + "search" => match parse_search_request(&payload) { + Ok(req) => { + let include_text = req.include_text; + let core = match CORE.read() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + let res = store::search(&core, req); + let hits: Vec = res + .hits + .iter() + .map(|h| hit_to_json(h, include_text)) + .collect(); + Envelope::ok(json!({ + "hits": hits, + "partial": res.partial, + })) + } + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + // Regex search over stored segment TEXT. Needs no vectors, so it works + // the instant a doc is accepted (even while vectors are still Building). + // A broken pattern is a structural `bad_pattern` error, never a panic. + "grep" => match parse_grep_request(&payload) { + Ok(req) => { + let core = match CORE.read() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + match grep::grep(&core, &req) { + Ok(res) => { + let hits: Vec = res.hits.iter().map(grep::hit_to_json).collect(); + Envelope::ok(json!({ + "hits": hits, + "truncated": res.truncated, + "total_found": res.total_found, + })) + } + Err(grep::GrepError::BadPattern(msg)) => { + Envelope::err(codes::BAD_PATTERN, format!("invalid grep pattern: {msg}")) + } + } + } + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + // Incremental delete of a single document (and all its segments) by + // `doc_id`, atomically under one short write lock. An unknown doc_id is + // a structural no-op (`deleted:false`), never an error — a benign + // idempotent retry. Echoes which collection it lived in, how many + // segments went away, and whether the (now-empty) collection was dropped. + "delete_document" => match parse_delete_document_request(&payload) { + Ok(doc_id) => { + let mut core = match CORE.write() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + let res = store::delete_document(&mut core, &doc_id); + Envelope::ok(json!({ + "deleted": res.deleted, + "doc_id": doc_id, + "collection": res.collection, + "removed_segments": res.removed_segments, + "collection_dropped": res.collection_dropped, + })) + } + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + // Incremental delete of an entire collection (all docs + segments), + // atomically. An unknown collection is a structural no-op + // (`deleted:false`), consistent with `delete_document`. + "delete_collection" => match parse_delete_collection_request(&payload) { + Ok(collection) => { + let mut core = match CORE.write() { + Ok(c) => c, + Err(_) => { + return Envelope::err(codes::INTERNAL, "core lock poisoned") + .to_json_string() + } + }; + let res = store::delete_collection(&mut core, &collection); + Envelope::ok(json!({ + "deleted": res.deleted, + "collection": collection, + "removed_docs": res.removed_docs, + "removed_segments": res.removed_segments, + })) + } + Err(e) => Envelope::err(codes::BAD_PAYLOAD, e), + }, + + other => Envelope::err( + codes::UNKNOWN_METHOD, + format!("no such method: '{other}'"), + ), + }; + + envelope.to_json_string() +} + +// --------------------------------------------------------------------------- +// Payload parsing (JSON -> typed requests). Kept here so the store module stays +// free of JSON shape concerns. Each returns Err(message) on a bad payload. +// --------------------------------------------------------------------------- + +/// Pull an optional string field. +fn opt_str(v: &Value, key: &str) -> Option { + v.get(key).and_then(|x| x.as_str()).map(|s| s.to_string()) +} + +/// Pull an optional u64 field. +fn opt_u64(v: &Value, key: &str) -> Option { + v.get(key).and_then(|x| x.as_u64()) +} + +/// Pull an optional bool field with a default. +fn bool_or(v: &Value, key: &str, default: bool) -> bool { + v.get(key).and_then(|x| x.as_bool()).unwrap_or(default) +} + +/// Pull a meta object, defaulting to `{}` if absent. Non-object values are +/// coerced to `{}` so hits always echo a JSON object. +fn meta_or_empty(v: &Value, key: &str) -> Value { + match v.get(key) { + Some(m) if m.is_object() => m.clone(), + _ => json!({}), + } +} + +/// Parse a `configure` payload. All fields optional; a null payload is fine. +/// +/// `model` selects a built-in real model by whitelisted name (see +/// `list_models` / `core::SUPPORTED_MODEL_NAMES`) and `model_path` points at +/// offline ONNX + tokenizer files; either one (when the `fastembed` feature is +/// compiled in) selects the real embedder, otherwise the mock is used — see +/// `core::configure`. `cache_dir` overrides where the built-in model is +/// downloaded/cached (fastembed's default cache location otherwise). +/// +/// `device` is `"cpu" | "dml" | "auto"` (default `"auto"` = DirectML with +/// automatic CPU fallback). It only affects the real embedder; the mock build +/// parses and echoes it but otherwise ignores it. +fn parse_config(payload: &Value) -> Result { + Ok(Config { + model: opt_str(payload, "model"), + model_path: opt_str(payload, "model_path"), + cache_dir: opt_str(payload, "cache_dir"), + normalize: bool_or(payload, "normalize", true), + max_seq_len: opt_u64(payload, "max_seq_len"), + // Default `"auto"` = DirectML with ort's automatic CPU fallback (the + // GPU-acceleration default). Explicit `"cpu"`/`"dml"` force the EP. Only + // meaningful under the `fastembed` feature; ignored by the mock build. + device: opt_str(payload, "device").unwrap_or_else(|| "auto".to_string()), + intra_threads: opt_u64(payload, "intra_threads"), + // Concurrent bulk-embedding workers/sessions. Absent ⇒ single worker + // (back-compat); 0 ⇒ auto (≈ ncpu/2); n ⇒ exactly n. (fastembed only.) + embed_workers: opt_u64(payload, "embed_workers"), + }) +} + +/// Pre-validate a `configure` request's built-in `model` name against the +/// whitelist (`core::SUPPORTED_MODEL_NAMES`), BEFORE the store is touched. +/// +/// The check applies only when the BUILTIN path would be taken — i.e. there is +/// no non-empty `model_path` (local files are loaded by path; the name +/// whitelist does not apply to them). Empty/absent `model` is always fine (it +/// means "the default" / "the mock"). The whitelist lives in `core`, NOT +/// behind the `fastembed` feature, so a mock/lite build rejects exactly the +/// same names a full build would. +fn validate_builtin_model(config: &Config) -> Result<(), String> { + let has_path = config + .model_path + .as_deref() + .map(|p| !p.trim().is_empty()) + .unwrap_or(false); + if has_path { + return Ok(()); + } + store::validate_model_name(config.model.as_deref().unwrap_or("")) +} + +/// Parse an `index_segments` payload. `doc_id` is required and non-empty. +fn parse_index_request(payload: &Value) -> Result { + let collection = opt_str(payload, "collection") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| "missing or empty 'collection'".to_string())?; + let doc_id = opt_str(payload, "doc_id") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| "missing or empty 'doc_id' (required for upsert/delete)".to_string())?; + let name = opt_str(payload, "name").unwrap_or_default(); + let meta = meta_or_empty(payload, "meta"); + + let segments_val = payload + .get("segments") + .and_then(|s| s.as_array()) + .ok_or_else(|| "missing 'segments' array".to_string())?; + + let mut segments = Vec::with_capacity(segments_val.len()); + for (i, s) in segments_val.iter().enumerate() { + let text = opt_str(s, "text") + .ok_or_else(|| format!("segment[{i}] missing 'text'"))?; + segments.push(SegmentInput { + text, + embed_text: opt_str(s, "embed_text"), + line_start: opt_u64(s, "line_start"), + line_end: opt_u64(s, "line_end"), + meta: meta_or_empty(s, "meta"), + }); + } + + Ok(IndexRequest { + collection, + doc_id, + name, + meta, + segments, + description: opt_str(payload, "collection_description"), + }) +} + +/// Parse an `index_raw` payload. `collection`, `name`, and `text` are required; +/// `doc_id` is optional (auto-assigned when absent). `chunk_cfg` is an optional +/// object with `target_tokens` / `max_tokens` / `overlap_lines` overrides. +fn parse_raw_index_request(payload: &Value) -> Result { + let collection = opt_str(payload, "collection") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| "missing or empty 'collection'".to_string())?; + // doc_id is optional; a present-but-blank value is treated as absent so the + // caller still gets an auto-assigned id rather than an unusable "" key. + let doc_id = opt_str(payload, "doc_id").filter(|s| !s.trim().is_empty()); + let name = opt_str(payload, "name").unwrap_or_default(); + let meta = meta_or_empty(payload, "meta"); + // `text` is required (an empty string is allowed → a single empty chunk). + let text = payload + .get("text") + .and_then(|x| x.as_str()) + .map(|s| s.to_string()) + .ok_or_else(|| "missing 'text'".to_string())?; + + // Optional chunker overrides under `chunk_cfg`. + let cfg = payload.get("chunk_cfg"); + let target_tokens = cfg + .and_then(|c| opt_u64(c, "target_tokens")) + .map(|v| v as usize); + let max_tokens = cfg + .and_then(|c| opt_u64(c, "max_tokens")) + .map(|v| v as usize); + let overlap_lines = cfg + .and_then(|c| opt_u64(c, "overlap_lines")) + .map(|v| v as usize); + + Ok(RawIndexRequest { + collection, + description: opt_str(payload, "collection_description"), + doc_id, + name, + meta, + text, + target_tokens, + max_tokens, + overlap_lines, + }) +} + +/// Parse a `get_segment` payload. `doc_id` is required and non-empty; +/// `line_start` / `line_end` default to 1 (the store clamps out-of-range values). +fn parse_get_segment_request(payload: &Value) -> Result { + let doc_id = opt_str(payload, "doc_id") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| "missing or empty 'doc_id'".to_string())?; + // Missing line bounds default to 1; the store clamps to the actual range. + let line_start = opt_u64(payload, "line_start").unwrap_or(1); + let line_end = opt_u64(payload, "line_end").unwrap_or(line_start); + let max_lines = opt_u64(payload, "max_lines").map(|v| v as usize); + Ok(GetSegmentRequest { + doc_id, + line_start, + line_end, + max_lines, + }) +} + +/// Parse a `delete_document` payload. `doc_id` is required and non-empty. +fn parse_delete_document_request(payload: &Value) -> Result { + opt_str(payload, "doc_id") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| "missing or empty 'doc_id'".to_string()) +} + +/// Parse a `delete_collection` payload. `collection` is required and non-empty. +fn parse_delete_collection_request(payload: &Value) -> Result { + opt_str(payload, "collection") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| "missing or empty 'collection'".to_string()) +} + +/// Parse a `search` payload. `query` required; `k` defaults to 10; `mode` +/// defaults to `dense` (an unknown mode also falls back to `dense`). +fn parse_search_request(payload: &Value) -> Result { + let query = opt_str(payload, "query") + .ok_or_else(|| "missing 'query'".to_string())?; + let k = opt_u64(payload, "k").map(|v| v as usize).unwrap_or(10).max(1); + let min_score = payload + .get("min_score") + .and_then(|x| x.as_f64()) + .map(|x| x as f32); + let max_per_doc = opt_u64(payload, "max_per_doc").map(|v| v as usize); + // `mode`: dense | keyword | hybrid. Absent/unknown → dense (back-compat). + let mode = SearchMode::parse(payload.get("mode").and_then(|x| x.as_str())); + Ok(SearchRequest { + query, + // Scope: a `collections` array (search a subset) takes precedence over a + // single `collection` string; absent ⇒ all collections. The array is + // joined into the comma-separated form `in_scope` matches against. + collection: parse_collection_scope(payload), + mode, + k, + min_score, + max_per_doc, + include_text: bool_or(payload, "include_text", true), + filter: MetaFilter::parse(payload), + }) +} + +/// Resolve the collection scope from a `collections` array (preferred) or a +/// single `collection` string, into the comma-joined form the store matches on. +/// Returns `None` (= all collections) when neither is present/non-empty. +fn parse_collection_scope(payload: &Value) -> Option { + if let Some(arr) = payload.get("collections").and_then(|x| x.as_array()) { + let names: Vec = arr + .iter() + .filter_map(|v| v.as_str()) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + if !names.is_empty() { + return Some(names.join(",")); + } + } + opt_str(payload, "collection").filter(|s| !s.trim().is_empty()) +} + +/// Parse a `grep` payload. `pattern` is required and non-empty. `max_matches` +/// defaults to 200 (and is clamped to at least 1); `multiline` defaults to true; +/// `context_lines` defaults to 0. +fn parse_grep_request(payload: &Value) -> Result { + let pattern = opt_str(payload, "pattern") + .filter(|s| !s.is_empty()) + .ok_or_else(|| "missing or empty 'pattern'".to_string())?; + let max_matches = opt_u64(payload, "max_matches") + .map(|v| v as usize) + .unwrap_or(200) + .max(1); + let context_lines = opt_u64(payload, "context_lines").map(|v| v as usize).unwrap_or(0); + let max_per_doc = opt_u64(payload, "max_per_doc").map(|v| v as usize); + Ok(GrepRequest { + pattern, + collection: opt_str(payload, "collection"), + filter: MetaFilter::parse(payload), + ignore_case: bool_or(payload, "ignore_case", false), + multiline: bool_or(payload, "multiline", true), + context_lines, + max_matches, + max_per_doc, + }) +} + +/// Build the JSON object for one hit. When `include_text` is false the full text +/// is replaced by a short `preview` so the wire payload stays small. +fn hit_to_json(h: &store::Hit, include_text: bool) -> Value { + let mut obj = json!({ + "doc_id": h.doc_id, + "name": h.name, + "collection": h.collection, + "meta": h.meta, + "segment_id": h.segment_id, + "score": h.score, + }); + if let Some(ls) = h.line_start { + obj["line_start"] = json!(ls); + } + if let Some(le) = h.line_end { + obj["line_end"] = json!(le); + } + if include_text { + obj["text"] = json!(h.text); + } else { + // Preview: first ~120 chars, on a char boundary. + let preview: String = h.text.chars().take(120).collect(); + obj["preview"] = json!(preview); + } + obj +} + +// --------------------------------------------------------------------------- +// C ABI surface +// --------------------------------------------------------------------------- + +/// Return a JSON object describing this core: `{"name", "version", "abi"}`. +/// +/// The returned pointer is Rust-owned and MUST be freed with +/// `rcore_free_string`. +#[no_mangle] +pub extern "C" fn rcore_version() -> *mut c_char { + let s = guard(|| { + json!({ + "name": env!("CARGO_PKG_NAME"), + "version": env!("CARGO_PKG_VERSION"), + "abi": ABI_VERSION, + }) + .to_string() + }); + into_c_string(s) +} + +/// The single generic JSON-in / JSON-out entry point. +/// +/// * `method` — borrowed C string naming the operation (e.g. "ping"). +/// * `payload_json` — borrowed C string with the method's JSON arguments; +/// may be null or empty (treated as JSON null). +/// +/// Returns a Rust-owned JSON string that MUST be freed with `rcore_free_string`. +/// Never returns null and never panics across the boundary. +/// +/// # Safety +/// `method` and `payload_json` must each be null or a valid NUL-terminated +/// C string that remains valid for the duration of the call. +#[no_mangle] +pub unsafe extern "C" fn rcore_dispatch( + method: *const c_char, + payload_json: *const c_char, +) -> *mut c_char { + let s = guard(|| { + let method = match borrow_c_str(method) { + Some(m) => m, + None => { + return Envelope::err(codes::BAD_PAYLOAD, "method is null or not UTF-8") + .to_json_string() + } + }; + // A null payload pointer is allowed and means "no arguments". + let payload = borrow_c_str(payload_json).unwrap_or(""); + dispatch(method, payload) + }); + into_c_string(s) +} + +/// Free a string previously returned by any `rcore_*` function. +/// +/// Calling with a null pointer is a safe no-op. Calling more than once on the +/// same non-null pointer is undefined behaviour (double-free) and is the +/// caller's responsibility to avoid. +/// +/// # Safety +/// `s` must be either null or a pointer obtained from an `rcore_*` function and +/// not yet freed. +#[no_mangle] +pub unsafe extern "C" fn rcore_free_string(s: *mut c_char) { + if s.is_null() { + return; + } + // Reclaim ownership and drop, freeing on Rust's allocator (the same one + // that allocated it in `into_c_string`). catch_unwind guards the (extremely + // unlikely) drop panic so it never crosses the boundary. + let _ = panic::catch_unwind(AssertUnwindSafe(|| { + drop(CString::from_raw(s)); + })); +} + +/// Best-effort teardown hook. Intended for `doStopListen` / form-close on the +/// C++ side (NOT `~HttpServerComponent`, since the singleton outlives any one +/// component). Stage 1 will cancel + join background workers here so the DLL can +/// unload cleanly without crashing inside a still-running native thread. +/// +/// For Stage 0 there are no workers, so this is an idempotent, safe no-op. It is +/// explicitly safe to call from a C++ destructor path and safe to call multiple +/// times. +#[no_mangle] +pub extern "C" fn rcore_shutdown() { + let _ = panic::catch_unwind(|| { + // Signal the background ingest worker to stop and join it, so the DLL + // can unload without crashing inside a still-running native thread. + // Idempotent: if no worker is running this is a no-op, and a later + // ingest will lazily respawn one. + store::shutdown(); + }); +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + /// Helper: take a Rust-owned `*mut c_char`, copy it to a `String`, then free + /// it through the real `rcore_free_string` path (exercising the free path). + unsafe fn take_and_free(ptr: *mut c_char) -> String { + assert!(!ptr.is_null(), "core returned a null pointer"); + let owned = CStr::from_ptr(ptr).to_str().unwrap().to_owned(); + rcore_free_string(ptr); + owned + } + + #[test] + fn ping_round_trips_and_frees() { + let method = CString::new("ping").unwrap(); + let payload = CString::new(r#"{"hello":"world"}"#).unwrap(); + + let out = unsafe { rcore_dispatch(method.as_ptr(), payload.as_ptr()) }; + let s = unsafe { take_and_free(out) }; + + let v: Value = serde_json::from_str(&s).unwrap(); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["pong"], json!(true)); + assert_eq!(v["result"]["echo"]["hello"], json!("world")); + } + + #[test] + fn unknown_method_is_structural_error_not_panic() { + let method = CString::new("does_not_exist").unwrap(); + let out = unsafe { rcore_dispatch(method.as_ptr(), std::ptr::null()) }; + let s = unsafe { take_and_free(out) }; + + let v: Value = serde_json::from_str(&s).unwrap(); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::UNKNOWN_METHOD)); + } + + #[test] + fn bad_payload_is_structural_error() { + let method = CString::new("ping").unwrap(); + let payload = CString::new("{not json").unwrap(); + let out = unsafe { rcore_dispatch(method.as_ptr(), payload.as_ptr()) }; + let s = unsafe { take_and_free(out) }; + + let v: Value = serde_json::from_str(&s).unwrap(); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::BAD_PAYLOAD)); + } + + #[test] + fn reset_and_stats_share_singleton_state() { + // Hold the shared singleton lock so a concurrent `configure`/ingest in + // another test can't flip `configured` between our reset and stats. + let _g = crate::core::TEST_LOCK + .lock() + .unwrap_or_else(|p| p.into_inner()); + let reset = CString::new("reset").unwrap(); + let stats = CString::new("stats").unwrap(); + + let r = unsafe { rcore_dispatch(reset.as_ptr(), std::ptr::null()) }; + let _ = unsafe { take_and_free(r) }; + + let out = unsafe { rcore_dispatch(stats.as_ptr(), std::ptr::null()) }; + let s = unsafe { take_and_free(out) }; + let v: Value = serde_json::from_str(&s).unwrap(); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["configured"], json!(false)); + // callsHandled is process-global and monotonic; it must be > 0 because + // these very calls were counted. + assert!(v["result"]["callsHandled"].as_u64().unwrap() >= 1); + } + + #[test] + fn version_reports_name_and_abi() { + let out = rcore_version(); + let s = unsafe { take_and_free(out) }; + let v: Value = serde_json::from_str(&s).unwrap(); + assert_eq!(v["name"], json!("rcore")); + assert_eq!(v["abi"], json!(ABI_VERSION)); + } + + #[test] + fn free_null_is_noop() { + // Must not crash. + unsafe { rcore_free_string(std::ptr::null_mut()) }; + } + + #[test] + fn shutdown_is_idempotent() { + rcore_shutdown(); + rcore_shutdown(); + } + + // -- Stage 1 end-to-end dispatch tests (full JSON round-trip via FFI) ----- + + /// Serialize tests that mutate the shared `CORE` singleton, and reset it to + /// a clean slate at the start of each. Uses a process-global mutex so these + /// don't race each other or the store-level tests. + fn e2e_guard() -> std::sync::MutexGuard<'static, ()> { + // Share the SAME lock as the store-level tests so the two modules don't + // race on the global singleton. + let g = crate::core::TEST_LOCK + .lock() + .unwrap_or_else(|p| p.into_inner()); + // Drain the shared background worker first so a job enqueued by a prior + // test can't apply to this test's same-named collection and flip it + // `Ready` prematurely (the worker respawns lazily on the next ingest). + store::shutdown(); + let reset = CString::new("reset").unwrap(); + let out = unsafe { rcore_dispatch(reset.as_ptr(), std::ptr::null()) }; + let _ = unsafe { take_and_free(out) }; + g + } + + /// Call a method by name with a JSON payload and parse the response. + fn call(method: &str, payload: &str) -> Value { + let m = CString::new(method).unwrap(); + let p = CString::new(payload).unwrap(); + let out = unsafe { rcore_dispatch(m.as_ptr(), p.as_ptr()) }; + let s = unsafe { take_and_free(out) }; + serde_json::from_str(&s).unwrap() + } + + #[test] + fn collections_carry_descriptions_and_scope_search() { + let _g = e2e_guard(); + call("configure", "{}"); + // Two labelled collections (keyword path needs no embedding wait). + call( + "index_segments", + r#"{"collection":"alpha","doc_id":"a","collection_description":"Alpha corpus","segments":[{"text":"shared token alpha-only"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"beta","doc_id":"b","collection_description":"Beta corpus","segments":[{"text":"shared token beta-only"}]}"#, + ); + + // list_collections exposes name + description for AI discovery. + let lc = call("list_collections", ""); + let arr = lc["result"]["collections"].as_array().unwrap(); + assert_eq!(arr.len(), 2); + let by_name: std::collections::HashMap<&str, &Value> = + arr.iter().map(|c| (c["name"].as_str().unwrap(), c)).collect(); + assert_eq!(by_name["alpha"]["description"], json!("Alpha corpus")); + assert_eq!(by_name["beta"]["description"], json!("Beta corpus")); + + // stats also carries the description. + let st = call("stats", ""); + assert_eq!( + st["result"]["collections"]["alpha"]["description"], + json!("Alpha corpus") + ); + + // Multi-collection scope: searching "shared" restricted to ["alpha"] + // returns only alpha hits; omitting the filter searches every collection. + let scoped = call( + "search", + r#"{"query":"shared","mode":"keyword","k":10,"collections":["alpha"]}"#, + ); + let hits = scoped["result"]["hits"].as_array().unwrap(); + assert!(!hits.is_empty()); + assert!(hits.iter().all(|h| h["collection"] == json!("alpha"))); + + let all = call("search", r#"{"query":"shared","mode":"keyword","k":10}"#); + let colls: std::collections::HashSet<&str> = all["result"]["hits"] + .as_array() + .unwrap() + .iter() + .map(|h| h["collection"].as_str().unwrap()) + .collect(); + assert!(colls.contains("alpha") && colls.contains("beta")); + } + + #[test] + fn configure_echoes_dim_and_config() { + let _g = e2e_guard(); + let v = call( + "configure", + r#"{"model_path":"/models/bge","normalize":true,"device":"cpu","max_seq_len":512}"#, + ); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["configured"], json!(true)); + assert!(v["result"]["dim"].as_u64().unwrap() > 0); + assert_eq!(v["result"]["model_path"], json!("/models/bge")); + assert_eq!(v["result"]["device"], json!("cpu")); + assert_eq!(v["result"]["max_seq_len"], json!(512)); + } + + #[test] + fn list_models_lists_whitelist_before_configure() { + // Static data: needs no `configure`, takes no lock, and returns the + // same list with or without the `fastembed` feature (a client renders + // its model dropdown from this even in the lite/mock build). + let v = call("list_models", ""); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["default"], json!("multilingual-e5-small")); + let models: Vec<&str> = v["result"]["models"] + .as_array() + .unwrap() + .iter() + .map(|m| m.as_str().unwrap()) + .collect(); + assert_eq!( + models, + vec![ + "multilingual-e5-small", + "multilingual-e5-base", + "multilingual-e5-large" + ] + ); + // The advertised default is always one of the listed models. + assert!(models.contains(&v["result"]["default"].as_str().unwrap())); + } + + #[test] + fn configure_unknown_model_is_bad_model_error() { + let _g = e2e_guard(); + // Unknown name → structural `bad_model` BEFORE the store is touched + // (works in the default mock build: the whitelist is feature-free). + let v = call("configure", r#"{"model":"nope-9000"}"#); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::BAD_MODEL)); + let msg = v["error"]["message"].as_str().unwrap(); + assert!(msg.contains("unknown model 'nope-9000'"), "got: {msg}"); + for name in store::SUPPORTED_MODEL_NAMES { + assert!(msg.contains(name), "message must list '{name}': {msg}"); + } + // The store really was untouched: still unconfigured. + let s = call("stats", ""); + assert_eq!(s["result"]["configured"], json!(false)); + } + + #[test] + fn model_name_validation_default_case_insensitive_and_path_bypass() { + // Direct unit checks (no embedder load, fast in every build): + // empty/whitespace = "the default" = ok; matching is trimmed + + // case-insensitive; unknown errors list the whole whitelist. + assert!(store::validate_model_name("").is_ok()); + assert!(store::validate_model_name(" ").is_ok()); + assert!(store::validate_model_name("multilingual-e5-small").is_ok()); + assert!(store::validate_model_name(" Multilingual-E5-LARGE ").is_ok()); + let err = store::validate_model_name("bge-zzz").unwrap_err(); + assert!(err.contains("unknown model 'bge-zzz'"), "got: {err}"); + for name in store::SUPPORTED_MODEL_NAMES { + assert!(err.contains(name), "error must list '{name}': {err}"); + } + + // A non-empty `model_path` takes the LOCAL path, so the builtin-name + // whitelist deliberately does not apply (local files have no name). + let cfg = parse_config(&json!({"model":"not-a-model","model_path":"/m/x"})).unwrap(); + assert!(validate_builtin_model(&cfg).is_ok()); + // …but with a blank path the same unknown name is rejected. + let cfg = parse_config(&json!({"model":"not-a-model","model_path":" "})).unwrap(); + assert!(validate_builtin_model(&cfg).is_err()); + } + + #[test] + fn configure_parses_and_echoes_cache_dir() { + let _g = e2e_guard(); + // parse_config picks up cache_dir… + let cfg = parse_config(&json!({"cache_dir": "D:/hf-cache"})).unwrap(); + assert_eq!(cfg.cache_dir.as_deref(), Some("D:/hf-cache")); + // …and `configure` echoes it back alongside model/model_path. No + // `model`/`model_path` here, so this stays on the fast mock path in + // every build (cache_dir alone never selects the real embedder). + let v = call("configure", r#"{"cache_dir":"D:/hf-cache"}"#); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["cache_dir"], json!("D:/hf-cache")); + assert_eq!(v["result"]["model"], json!(null)); + } + + #[test] + fn index_segments_requires_doc_id() { + let _g = e2e_guard(); + let v = call( + "index_segments", + r#"{"collection":"c","segments":[{"text":"x"}]}"#, + ); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::BAD_PAYLOAD)); + } + + #[test] + fn full_pipeline_index_search_stats() { + let _g = e2e_guard(); + call("configure", "{}"); + + // Accept is immediate. + let v = call( + "index_segments", + r#"{"collection":"docs","doc_id":"d1","name":"DB guide", + "meta":{"kind":"manual"}, + "segments":[ + {"text":"database connection pooling and tuning","line_start":1,"line_end":3}, + {"text":"banana orange apple smoothie"} + ]}"#, + ); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["accepted"], json!(true)); + assert_eq!(v["result"]["collection"], json!("docs")); + assert_eq!(v["result"]["segment_count"], json!(2)); + + // Wait for the worker (deterministic condvar helper). + assert!(store::wait_until_ready( + "docs", + std::time::Duration::from_secs(5) + )); + + // Search returns the DB segment as top hit with full metadata. + let v = call( + "search", + r#"{"query":"database connection","collection":"docs","k":5,"include_text":true}"#, + ); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["partial"], json!(false)); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(!hits.is_empty()); + assert_eq!(hits[0]["doc_id"], json!("d1")); + assert_eq!(hits[0]["name"], json!("DB guide")); + assert_eq!(hits[0]["collection"], json!("docs")); + assert_eq!(hits[0]["meta"]["kind"], json!("manual")); + assert!(hits[0]["text"].as_str().unwrap().contains("database")); + assert_eq!(hits[0]["line_start"], json!(1)); + + // include_text:false yields a preview instead of text. + let v = call( + "search", + r#"{"query":"database connection","collection":"docs","k":1,"include_text":false}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(hits[0].get("text").is_none()); + assert!(hits[0].get("preview").is_some()); + + // stats reflects two-axis state + counters + totals. + let v = call("stats", ""); + let r = &v["result"]; + assert_eq!(r["configured"], json!(true)); + assert!(r["dim"].as_u64().unwrap() > 0); + assert_eq!(r["n_docs"], json!(1)); + assert_eq!(r["n_segments"], json!(2)); + let coll = &r["collections"]["docs"]; + assert_eq!(coll["text_ready"], json!(true)); + assert_eq!(coll["vector_status"], json!("ready")); + assert_eq!(coll["embedded"], json!(2)); + assert_eq!(coll["failed"], json!(0)); + assert_eq!(coll["skipped"], json!(0)); + assert_eq!(coll["n_docs"], json!(1)); + assert_eq!(coll["n_segments"], json!(2)); + } + + // -- Stage 2: meta filters on `search` (Task 2) --------------------------- + + /// Index two docs with distinguishing meta, wait for vectors, then return. + /// Both share the query tokens so ranking can't accidentally hide a doc; the + /// meta filter is what selects between them. + fn seed_meta_docs() { + call("configure", "{}"); + call( + "index_segments", + r#"{"collection":"c","doc_id":"en1","name":"english", + "meta":{"lang":"en","status":"draft"}, + "segments":[{"text":"shared token alpha beta"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"ru1","name":"russian", + "meta":{"lang":"ru","status":"published"}, + "segments":[{"text":"shared token alpha beta"}]}"#, + ); + assert!(store::wait_until_ready("c", std::time::Duration::from_secs(5))); + } + + #[test] + fn search_meta_filter_all_is_and() { + let _g = e2e_guard(); + seed_meta_docs(); + // all: lang==en AND status==draft → only en1. + let v = call( + "search", + r#"{"query":"shared token","collection":"c","k":10, + "filter":{"all":{"lang":"en","status":"draft"}}}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1, "AND filter must select exactly one doc"); + assert_eq!(hits[0]["doc_id"], json!("en1")); + } + + #[test] + fn search_hits_echo_effective_meta() { + let _g = e2e_guard(); + call("configure", "{}"); + // Doc-level meta overlaid by segment-level meta (segment wins) — the + // hit must echo the same effective view the filters match against. + call( + "index_segments", + r#"{"collection":"em","doc_id":"d1","meta":{"lang":"en","src":"doc"}, + "segments":[{"text":"unique effmeta token","meta":{"type":"step","lang":"ru"}}]}"#, + ); + let v = call( + "search", + r#"{"query":"effmeta","collection":"em","mode":"keyword","k":3}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1); + let meta = &hits[0]["meta"]; + assert_eq!(meta["type"], json!("step"), "segment meta must be visible"); + assert_eq!(meta["src"], json!("doc"), "doc meta must be preserved"); + assert_eq!(meta["lang"], json!("ru"), "segment value wins on collision"); + } + + #[test] + fn search_meta_filter_any_is_or() { + let _g = e2e_guard(); + seed_meta_docs(); + // any: lang==ru OR status==draft → both docs (en1 via status, ru1 via lang). + let v = call( + "search", + r#"{"query":"shared token","collection":"c","k":10, + "filter":{"any":{"lang":"ru","status":"draft"}}}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + let ids: Vec<&str> = hits.iter().map(|h| h["doc_id"].as_str().unwrap()).collect(); + assert_eq!(hits.len(), 2, "OR filter should match both"); + assert!(ids.contains(&"en1") && ids.contains(&"ru1")); + } + + #[test] + fn search_meta_filter_combined() { + let _g = e2e_guard(); + seed_meta_docs(); + // all (lang==en) AND any (status in [draft, review]) → only en1. + let v = call( + "search", + r#"{"query":"shared token","collection":"c","k":10, + "filter":{"all":{"lang":"en"},"any":{"status":["draft","review"]}}}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0]["doc_id"], json!("en1")); + + // Same all-clause but an any-clause that excludes en1's status → empty. + let v = call( + "search", + r#"{"query":"shared token","collection":"c","k":10, + "filter":{"all":{"lang":"en"},"any":{"status":["review"]}}}"#, + ); + assert!(v["result"]["hits"].as_array().unwrap().is_empty()); + } + + // -- Stage 2: grep (Task 3) ---------------------------------------------- + + /// Index a multi-line doc (text installed synchronously) and wait for ready. + /// The segment text uses JSON `\r\n` escapes (two-char sequences in the wire + /// JSON) so the stored text really contains CRLF, exercising normalization. + fn seed_grep_doc() { + call("configure", "{}"); + // A 4-line segment with CRLF endings to exercise normalization. The + // raw-string keeps the backslash-r-backslash-n as JSON escapes, not + // literal control bytes (which would be invalid JSON). + call( + "index_segments", + r#"{"collection":"docs","doc_id":"g1","name":"log", + "meta":{"kind":"manual"}, + "segments":[{"text":"first line ERROR here\r\nsecond plain line\r\nthird Error again\r\nfourth done"}]}"#, + ); + assert!(store::wait_until_ready("docs", std::time::Duration::from_secs(5))); + } + + #[test] + fn grep_basic_match() { + let _g = e2e_guard(); + seed_grep_doc(); + let v = call("grep", r#"{"pattern":"ERROR","collection":"docs"}"#); + assert_eq!(v["ok"], json!(true)); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1, "only the exact-case ERROR line matches"); + assert_eq!(hits[0]["doc_id"], json!("g1")); + assert_eq!(hits[0]["line_number"], json!(1)); + assert_eq!(hits[0]["line_text"], json!("first line ERROR here")); + assert_eq!(v["result"]["truncated"], json!(false)); + assert_eq!(v["result"]["total_found"], json!(1)); + } + + #[test] + fn grep_ignore_case() { + let _g = e2e_guard(); + seed_grep_doc(); + let v = call( + "grep", + r#"{"pattern":"error","collection":"docs","ignore_case":true}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 2, "case-insensitive matches both ERROR and Error"); + // Line numbers are 1-based after CRLF normalization. + assert_eq!(hits[0]["line_number"], json!(1)); + assert_eq!(hits[1]["line_number"], json!(3)); + } + + #[test] + fn grep_context_lines() { + let _g = e2e_guard(); + seed_grep_doc(); + let v = call( + "grep", + r#"{"pattern":"third","collection":"docs","context_lines":1}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1); + let h = &hits[0]; + assert_eq!(h["line_number"], json!(3)); + assert_eq!(h["context_before"], json!(["second plain line"])); + assert_eq!(h["context_after"], json!(["fourth done"])); + } + + #[test] + fn grep_max_matches_truncates() { + let _g = e2e_guard(); + seed_grep_doc(); + // "line" appears on lines 1 and 2; cap at 1 → truncated. + let v = call( + "grep", + r#"{"pattern":"line","collection":"docs","max_matches":1}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1, "max_matches caps hits"); + assert_eq!(v["result"]["truncated"], json!(true)); + } + + #[test] + fn grep_broken_pattern_is_structural_error() { + let _g = e2e_guard(); + seed_grep_doc(); + // An unclosed group is an invalid regex → structural bad_pattern error. + let v = call("grep", r#"{"pattern":"(unclosed","collection":"docs"}"#); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::BAD_PATTERN)); + } + + #[test] + fn grep_meta_filtered() { + let _g = e2e_guard(); + call("configure", "{}"); + call( + "index_segments", + r#"{"collection":"docs","doc_id":"m1","name":"a","meta":{"kind":"manual"}, + "segments":[{"text":"keyword target one"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"docs","doc_id":"n1","name":"b","meta":{"kind":"note"}, + "segments":[{"text":"keyword target two"}]}"#, + ); + assert!(store::wait_until_ready("docs", std::time::Duration::from_secs(5))); + // Filter to kind==manual → only m1 should match the same pattern. + let v = call( + "grep", + r#"{"pattern":"keyword","collection":"docs","filter":{"all":{"kind":"manual"}}}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 1, "meta filter must scope grep"); + assert_eq!(hits[0]["doc_id"], json!("m1")); + } + + // -- Stage 2: index_raw + get_segment (this card) ------------------------ + + #[test] + fn index_raw_returns_doc_id_and_segment_count() { + let _g = e2e_guard(); + call("configure", "{}"); + // No doc_id → auto-assigned and returned in the ack so the caller can + // upsert/delete later. + let v = call( + "index_raw", + r#"{"collection":"raw","name":"manual","text":"line a\nline b\nline c"}"#, + ); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["accepted"], json!(true)); + assert_eq!(v["result"]["collection"], json!("raw")); + let doc_id = v["result"]["doc_id"].as_str().unwrap(); + assert!(!doc_id.is_empty(), "auto doc_id must be returned in the ack"); + assert!(v["result"]["segment_count"].as_u64().unwrap() >= 1); + } + + #[test] + fn index_raw_sets_collection_description() { + let _g = e2e_guard(); + call("configure", "{}"); + // Same `collection_description` semantics as index_segments: last + // non-empty wins, blank/absent leaves the label untouched. + let v = call( + "index_raw", + r#"{"collection":"rawdesc","collection_description":"Feature files","text":"line a\nline b"}"#, + ); + assert_eq!(v["ok"], json!(true)); + let st = call("stats", ""); + assert_eq!( + st["result"]["collections"]["rawdesc"]["description"], + json!("Feature files") + ); + // A second doc without a description must not clear the label. + call("index_raw", r#"{"collection":"rawdesc","text":"more text"}"#); + let st = call("stats", ""); + assert_eq!( + st["result"]["collections"]["rawdesc"]["description"], + json!("Feature files") + ); + } + + #[test] + fn index_raw_get_segment_works_right_after_accept() { + let _g = e2e_guard(); + call("configure", "{}"); + // CRLF on the wire is normalized to LF on accept; line numbers are + // 1-based inclusive. We do NOT wait for vectors — text+offsets are + // synchronous, so get_segment must work immediately. + let v = call( + "index_raw", + r#"{"collection":"raw","doc_id":"d1","name":"manual", + "text":"first line\r\nsecond line\r\nthird line\r\nfourth line"}"#, + ); + assert_eq!(v["result"]["doc_id"], json!("d1")); + + let v = call( + "get_segment", + r#"{"doc_id":"d1","line_start":2,"line_end":3}"#, + ); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["line_start"], json!(2)); + assert_eq!(v["result"]["line_end"], json!(3)); + assert_eq!(v["result"]["line_count"], json!(4)); + // CRLF normalized away; exactly lines 2..=3 returned. + assert_eq!(v["result"]["text"], json!("second line\nthird line")); + } + + #[test] + fn get_segment_clamps_out_of_range_and_returns_actual() { + let _g = e2e_guard(); + call("configure", "{}"); + call( + "index_raw", + r#"{"collection":"raw","doc_id":"d2","name":"n","text":"a\nb\nc"}"#, + ); + // Request well beyond the end → clamp to [1,3] and report actual range. + let v = call( + "get_segment", + r#"{"doc_id":"d2","line_start":2,"line_end":900}"#, + ); + assert_eq!(v["result"]["line_start"], json!(2)); + assert_eq!(v["result"]["line_end"], json!(3)); + assert_eq!(v["result"]["text"], json!("b\nc")); + + // max_lines caps the returned line count. + let v = call( + "get_segment", + r#"{"doc_id":"d2","line_start":1,"line_end":3,"max_lines":1}"#, + ); + assert_eq!(v["result"]["line_start"], json!(1)); + assert_eq!(v["result"]["line_end"], json!(1)); + assert_eq!(v["result"]["text"], json!("a")); + } + + #[test] + fn get_segment_on_atomic_record_is_structural_error() { + let _g = e2e_guard(); + call("configure", "{}"); + // An atomic index_segments doc has no document-wide text/offset table. + call( + "index_segments", + r#"{"collection":"docs","doc_id":"atomic","name":"a", + "segments":[{"text":"a segment with no document line index"}]}"#, + ); + let v = call("get_segment", r#"{"doc_id":"atomic","line_start":1,"line_end":1}"#); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::NO_LINE_INDEX)); + } + + #[test] + fn get_segment_unknown_doc_is_not_found() { + let _g = e2e_guard(); + call("configure", "{}"); + let v = call("get_segment", r#"{"doc_id":"ghost","line_start":1,"line_end":1}"#); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::NOT_FOUND)); + } + + #[test] + fn index_raw_doc_is_greppable_immediately_and_searchable_when_ready() { + let _g = e2e_guard(); + call("configure", "{}"); + // A multi-line raw doc; chunked synchronously on accept. + call( + "index_raw", + r#"{"collection":"raw","doc_id":"big","name":"guide", + "text":"intro paragraph\ndatabase connection pooling and tuning\nunrelated banana smoothie\nclosing notes"}"#, + ); + // Greppable immediately (no vectors needed): the text is in the store. + let v = call("grep", r#"{"pattern":"database","collection":"raw"}"#); + assert_eq!(v["ok"], json!(true)); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(!hits.is_empty(), "raw-doc text must be greppable right after accept"); + assert_eq!(hits[0]["doc_id"], json!("big")); + + // Dense-searchable once the worker finishes embedding the chunks. + assert!(store::wait_until_ready("raw", std::time::Duration::from_secs(5))); + let v = call( + "search", + r#"{"query":"database connection","collection":"raw","k":5}"#, + ); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["partial"], json!(false)); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(!hits.is_empty(), "raw-doc chunks must be dense-searchable when ready"); + assert_eq!(hits[0]["doc_id"], json!("big")); + // The hit carries the chunk's 1-based inclusive line range. + assert!(hits[0]["line_start"].as_u64().is_some()); + assert!(hits[0]["line_end"].as_u64().is_some()); + } + + #[test] + fn index_raw_oversized_line_truncates_embed_but_get_segment_returns_whole() { + let _g = e2e_guard(); + // Small max_seq_len so a long line is "oversized". The line stays whole + // in the store (get_segment), but its embed text is truncated to the cap. + call("configure", r#"{"max_seq_len":5}"#); + // Build a single very long line (many tokens) surrounded by short ones. + let long_line = "word ".repeat(60); // ~60 tokens, well over the cap of 5 + let payload = serde_json::json!({ + "collection": "raw", + "doc_id": "ov", + "name": "oversized", + "text": format!("short head\n{}\nshort tail", long_line.trim()), + }); + let v = call("index_raw", &payload.to_string()); + assert_eq!(v["ok"], json!(true)); + + // get_segment must return the oversized line WHOLE (line 2). + let v = call("get_segment", r#"{"doc_id":"ov","line_start":2,"line_end":2}"#); + let text = v["result"]["text"].as_str().unwrap(); + assert_eq!( + text, + long_line.trim(), + "get_segment returns the oversized line in full, untruncated" + ); + + // It still embeds (truncated) in the background and reaches Ready. + assert!(store::wait_until_ready("raw", std::time::Duration::from_secs(5))); + } + + // -- Stage 2: hybrid search (mode: dense | keyword | hybrid) ------------- + + /// Seed three docs whose contents separate the two retrieval channels: + /// * `dwin` — semantically close to "database connection" (dense-strong) + /// but has none of the exact-id token, so keyword wouldn't surface it. + /// * `kwin` — carries the exact identifier `sku7701234567` (keyword-strong) + /// but little semantic overlap, so dense alone may bury it under noise. + /// * `noise` — unrelated, to give RRF something to outrank. + fn seed_hybrid_docs() { + call("configure", "{}"); + call( + "index_segments", + r#"{"collection":"c","doc_id":"dwin","name":"db", + "segments":[{"text":"database connection pooling and tuning guide"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"kwin","name":"sku", + "segments":[{"text":"sku7701234567 inventory record"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"noise","name":"x", + "segments":[{"text":"completely irrelevant banana smoothie"}]}"#, + ); + assert!(store::wait_until_ready("c", std::time::Duration::from_secs(5))); + } + + #[test] + fn keyword_mode_ranks_by_term_match() { + let _g = e2e_guard(); + call("configure", "{}"); + // doc m has two distinct query terms; doc o has one; doc z has none. + call( + "index_segments", + r#"{"collection":"c","doc_id":"m","name":"m", + "segments":[{"text":"alpha beta gamma"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"o","name":"o", + "segments":[{"text":"alpha delta epsilon"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"z","name":"z", + "segments":[{"text":"nothing relevant here"}]}"#, + ); + assert!(store::wait_until_ready("c", std::time::Duration::from_secs(5))); + + let v = call( + "search", + r#"{"query":"alpha beta","collection":"c","mode":"keyword","k":10}"#, + ); + assert_eq!(v["ok"], json!(true)); + // keyword reads only text → never partial. + assert_eq!(v["result"]["partial"], json!(false)); + let hits = v["result"]["hits"].as_array().unwrap(); + let ids: Vec<&str> = hits.iter().map(|h| h["doc_id"].as_str().unwrap()).collect(); + assert_eq!(ids.len(), 2, "only docs containing a query term match"); + assert_eq!(ids[0], "m", "two-term doc outranks one-term doc"); + assert_eq!(ids[1], "o"); + assert!(!ids.contains(&"z"), "non-matching doc is excluded"); + } + + #[test] + fn keyword_mode_works_while_building() { + let _g = e2e_guard(); + call("configure", "{}"); + // Index but do NOT wait for vectors — keyword must work immediately. + call( + "index_segments", + r#"{"collection":"c","doc_id":"b1","name":"b", + "segments":[{"text":"exact token magicword present"}]}"#, + ); + let v = call( + "search", + r#"{"query":"magicword","collection":"c","mode":"keyword","k":5}"#, + ); + assert_eq!(v["ok"], json!(true)); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(!hits.is_empty(), "keyword works before vectors are ready"); + assert_eq!(hits[0]["doc_id"], json!("b1")); + } + + #[test] + fn hybrid_mode_rrf_surfaces_both_channels() { + let _g = e2e_guard(); + seed_hybrid_docs(); + // A query blending a semantic phrase (favors dwin via dense) with the + // exact id token (favors kwin via keyword). RRF must surface BOTH. + let v = call( + "search", + r#"{"query":"database connection sku7701234567","collection":"c","mode":"hybrid","k":10}"#, + ); + assert_eq!(v["ok"], json!(true)); + let hits = v["result"]["hits"].as_array().unwrap(); + let ids: Vec<&str> = hits.iter().map(|h| h["doc_id"].as_str().unwrap()).collect(); + assert!(ids.contains(&"dwin"), "dense-strong doc must surface"); + assert!(ids.contains(&"kwin"), "keyword-only-strong doc must surface"); + } + + #[test] + fn keyword_and_hybrid_honor_filter_k_and_max_per_doc() { + let _g = e2e_guard(); + call("configure", "{}"); + // Two docs share the query term; meta distinguishes them. Each doc has + // two matching segments so max_per_doc is observable. + call( + "index_segments", + r#"{"collection":"c","doc_id":"en1","name":"en","meta":{"lang":"en"}, + "segments":[{"text":"shared keyword one"},{"text":"shared keyword two"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"ru1","name":"ru","meta":{"lang":"ru"}, + "segments":[{"text":"shared keyword three"},{"text":"shared keyword four"}]}"#, + ); + assert!(store::wait_until_ready("c", std::time::Duration::from_secs(5))); + + // keyword + meta filter (lang==en) → only en1's segments. + let v = call( + "search", + r#"{"query":"shared keyword","collection":"c","mode":"keyword","k":10, + "filter":{"all":{"lang":"en"}}}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(!hits.is_empty()); + assert!( + hits.iter().all(|h| h["doc_id"] == json!("en1")), + "keyword must respect the meta filter" + ); + + // keyword + k caps total hits. + let v = call( + "search", + r#"{"query":"shared keyword","collection":"c","mode":"keyword","k":1}"#, + ); + assert_eq!(v["result"]["hits"].as_array().unwrap().len(), 1, "k caps keyword"); + + // hybrid + max_per_doc:1 → at most one hit per doc. + let v = call( + "search", + r#"{"query":"shared keyword","collection":"c","mode":"hybrid","k":10,"max_per_doc":1}"#, + ); + let hits = v["result"]["hits"].as_array().unwrap(); + let mut per_doc = std::collections::HashMap::new(); + for h in hits { + *per_doc.entry(h["doc_id"].as_str().unwrap()).or_insert(0) += 1; + } + assert!(per_doc.values().all(|&n| n <= 1), "hybrid must respect max_per_doc"); + } + + // -- Stage 2: incremental delete ---------------------------------------- + + #[test] + fn delete_document_removes_from_search_grep_and_get_segment() { + let _g = e2e_guard(); + call("configure", "{}"); + // A raw doc → searchable + greppable + get_segment-able. + call( + "index_raw", + r#"{"collection":"c","doc_id":"d1","name":"guide", + "text":"database connection tuning\nsecond informative line"}"#, + ); + // A second doc so the collection survives the first delete. + call( + "index_segments", + r#"{"collection":"c","doc_id":"d2","name":"keep","segments":[{"text":"keep me around"}]}"#, + ); + assert!(store::wait_until_ready("c", std::time::Duration::from_secs(5))); + + // Present in all three surfaces before the delete. + assert!(!call("grep", r#"{"pattern":"database","collection":"c"}"#)["result"]["hits"] + .as_array() + .unwrap() + .is_empty()); + assert!(!call("search", r#"{"query":"database connection","collection":"c","mode":"keyword","k":10}"#)["result"]["hits"] + .as_array() + .unwrap() + .is_empty()); + assert_eq!( + call("get_segment", r#"{"doc_id":"d1","line_start":1,"line_end":1}"#)["ok"], + json!(true) + ); + + // Delete d1. + let v = call("delete_document", r#"{"doc_id":"d1"}"#); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["deleted"], json!(true)); + assert_eq!(v["result"]["collection"], json!("c")); + assert!(v["result"]["removed_segments"].as_u64().unwrap() >= 1); + assert_eq!(v["result"]["collection_dropped"], json!(false), "d2 keeps it alive"); + + // Gone from grep, keyword search, and get_segment (now not_found). + assert!(call("grep", r#"{"pattern":"database","collection":"c"}"#)["result"]["hits"] + .as_array() + .unwrap() + .is_empty(), "grep no longer finds the deleted doc"); + assert!(call("search", r#"{"query":"database connection","collection":"c","mode":"keyword","k":10}"#)["result"]["hits"] + .as_array() + .unwrap() + .is_empty(), "keyword search no longer finds the deleted doc"); + let g = call("get_segment", r#"{"doc_id":"d1","line_start":1,"line_end":1}"#); + assert_eq!(g["ok"], json!(false)); + assert_eq!(g["error"]["code"], json!(codes::NOT_FOUND)); + + // d2 is untouched. + assert!(!call("grep", r#"{"pattern":"keep","collection":"c"}"#)["result"]["hits"] + .as_array() + .unwrap() + .is_empty()); + } + + #[test] + fn delete_document_drops_empty_collection() { + let _g = e2e_guard(); + call("configure", "{}"); + call( + "index_segments", + r#"{"collection":"solo","doc_id":"only","name":"n","segments":[{"text":"lonely doc"}]}"#, + ); + assert!(store::wait_until_ready("solo", std::time::Duration::from_secs(5))); + + let v = call("delete_document", r#"{"doc_id":"only"}"#); + assert_eq!(v["result"]["deleted"], json!(true)); + assert_eq!(v["result"]["collection_dropped"], json!(true), "last doc → drop collection"); + + // The empty collection is no longer reported by stats. + let s = call("stats", ""); + assert!(s["result"]["collections"].get("solo").is_none(), "empty collection gone from stats"); + } + + #[test] + fn delete_unknown_document_is_structural_false() { + let _g = e2e_guard(); + call("configure", "{}"); + let v = call("delete_document", r#"{"doc_id":"ghost"}"#); + // A no-op result, not an error envelope. + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["deleted"], json!(false)); + assert_eq!(v["result"]["removed_segments"], json!(0)); + } + + #[test] + fn delete_collection_clears_it_and_unknown_is_false() { + let _g = e2e_guard(); + call("configure", "{}"); + call( + "index_segments", + r#"{"collection":"c","doc_id":"d1","name":"a","segments":[{"text":"one"},{"text":"two"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"d2","name":"b","segments":[{"text":"three"}]}"#, + ); + assert!(store::wait_until_ready("c", std::time::Duration::from_secs(5))); + + let v = call("delete_collection", r#"{"collection":"c"}"#); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["deleted"], json!(true)); + assert_eq!(v["result"]["removed_docs"], json!(2)); + assert_eq!(v["result"]["removed_segments"], json!(3)); + + // The collection is gone: search and grep find nothing in it. + assert!(call("search", r#"{"query":"one two three","collection":"c","mode":"keyword","k":10}"#)["result"]["hits"] + .as_array() + .unwrap() + .is_empty()); + let s = call("stats", ""); + assert!(s["result"]["collections"].get("c").is_none()); + + // Unknown collection → structural false (no-op), not an error. + let v = call("delete_collection", r#"{"collection":"nope"}"#); + assert_eq!(v["ok"], json!(true)); + assert_eq!(v["result"]["deleted"], json!(false)); + assert_eq!(v["result"]["removed_docs"], json!(0)); + } + + #[test] + fn delete_document_requires_doc_id() { + let _g = e2e_guard(); + let v = call("delete_document", r#"{}"#); + assert_eq!(v["ok"], json!(false)); + assert_eq!(v["error"]["code"], json!(codes::BAD_PAYLOAD)); + } + + // -- Real fastembed integration test (gated) ----------------------------- + // + // Compiled and run ONLY under `--features fastembed`. It mirrors the probe: + // configure the real multilingual-e5-small model, index ru/en/uk docs (two + // about a contract, one unrelated — a cat), wait for the background worker + // to embed, then dense-search a contract query and assert the contract docs + // outrank the unrelated one and that the index dim is 384. + // + // This downloads the model from HuggingFace on a cold cache (slow) and loads + // onnxruntime, so it lives behind the feature flag and is not part of the + // fast default suite. + #[cfg(feature = "fastembed")] + #[test] + fn fastembed_real_model_ranks_contracts_above_cat() { + let _g = e2e_guard(); + + // Select the real model with `device:"auto"` → ort registers the + // DirectML EP (best-effort). On a machine with no usable GPU/driver ort + // logs and falls back to CPU automatically, so this MUST NOT crash either + // way. With the feature on, e5-small loads at dim 384. The mock test + // suite never passes `model`, so it stays at dim 64. + let v = call("configure", r#"{"model":"multilingual-e5-small","device":"auto"}"#); + assert_eq!(v["ok"], json!(true), "configure failed: {v}"); + // The chosen device is echoed back verbatim (the EP selection — and any + // DirectML→CPU fallback — happens inside ort and is transparent here). + assert_eq!(v["result"]["device"], json!("auto")); + assert_eq!( + v["result"]["dim"].as_u64().unwrap(), + 384, + "real e5-small must report dim 384 (got {}). If this is 64 the real \ + model failed to load and we fell back to the mock — inspect the \ + onnxruntime/model-download path.", + v["result"]["dim"] + ); + + // Index two contract docs (ru + en) and one unrelated doc (uk: a cat on + // a windowsill). These mirror the probe's verified passages. + call( + "index_segments", + r#"{"collection":"c","doc_id":"ru","name":"ru-contract", + "segments":[{"text":"Договор поставки товара №123 от 5 июня"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"en","name":"en-contract", + "segments":[{"text":"Contract for the supply of goods"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"c","doc_id":"cat","name":"uk-cat", + "segments":[{"text":"Кіт сидить на вікні"}]}"#, + ); + + // Real embedding is much slower than the mock; give the worker time. + assert!( + store::wait_until_ready("c", std::time::Duration::from_secs(120)), + "collection did not reach Ready (worker may have failed to embed)" + ); + + // Dense search for a contract query (e5 query prefix is applied inside + // the embedder). Both contract docs must outrank the cat. + let v = call( + "search", + r#"{"query":"договор на поставку товаров","collection":"c","mode":"dense","k":10,"include_text":true}"#, + ); + assert_eq!(v["ok"], json!(true), "search failed: {v}"); + assert_eq!(v["result"]["partial"], json!(false)); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(hits.len() >= 3, "expected all three docs scored, got {}", hits.len()); + + // Map doc_id -> score and assert both contracts beat the cat. + let mut score_of = std::collections::HashMap::new(); + for h in hits { + score_of.insert( + h["doc_id"].as_str().unwrap().to_string(), + h["score"].as_f64().unwrap(), + ); + } + let ru = score_of["ru"]; + let en = score_of["en"]; + let cat = score_of["cat"]; + // Surfaced with --nocapture so the verified cosine ranking is visible. + eprintln!("e5-small cosine scores: ru={ru:.4} en={en:.4} cat={cat:.4}"); + assert!( + ru > cat && en > cat, + "contract docs must outrank the cat: ru={ru:.4} en={en:.4} cat={cat:.4}" + ); + + // The top hit is one of the two contracts (not the cat). + let top = hits[0]["doc_id"].as_str().unwrap(); + assert!(top == "ru" || top == "en", "top hit must be a contract, got '{top}'"); + + // dim is surfaced as 384 in stats too. + let s = call("stats", ""); + assert_eq!(s["result"]["dim"].as_u64().unwrap(), 384); + } + + // -- Built-in download into a caller-specified cache_dir (gated) ---------- + // + // Mirrors the gating of `fastembed_real_model_ranks_contracts_above_cat`: + // compiled and run ONLY under `--features fastembed`, never part of the + // fast default suite. Deliberately slow — it downloads e5-small from + // HuggingFace into a FRESH temp directory every run, bypassing any warm + // default cache, because an empty-then-populated directory is the only + // direct proof the `with_cache_dir` plumbing was actually applied. + #[cfg(feature = "fastembed")] + #[test] + fn fastembed_builtin_downloads_into_cache_dir() { + let _g = e2e_guard(); + + let cache = + std::env::temp_dir().join(format!("rcore-cache-test-{}", std::process::id())); + // Hermetic: start from an empty directory so "files appeared" below is + // meaningful even if a previous run with this PID left residue behind. + let _ = std::fs::remove_dir_all(&cache); + std::fs::create_dir_all(&cache).expect("create temp cache dir"); + let cache_str = cache.to_string_lossy().to_string(); + + let payload = format!( + r#"{{"model":"multilingual-e5-small","device":"cpu","cache_dir":{}}}"#, + json!(cache_str) + ); + let v = call("configure", &payload); + assert_eq!(v["ok"], json!(true), "configure failed: {v}"); + assert_eq!(v["result"]["cache_dir"], json!(cache_str)); + assert_eq!( + v["result"]["dim"].as_u64().unwrap(), + 384, + "e5-small from the custom cache dir must report dim 384 (got {}); a 64 \ + means the load failed and we silently fell back to the mock", + v["result"]["dim"] + ); + + // The model files were materialized under OUR directory (hf-hub lays + // out a models--intfloat--multilingual-e5-small snapshot inside it), + // not under fastembed's default cache. + let entries: Vec<_> = std::fs::read_dir(&cache) + .expect("cache dir must exist after configure") + .filter_map(|e| e.ok()) + .collect(); + assert!( + !entries.is_empty(), + "cache_dir {} stayed empty — with_cache_dir was not applied to the \ + builtin download path", + cache.display() + ); + + // Drop the model sessions (they hold no files open after load, but the + // embedder is replaced anyway) and reclaim the ~100 MB of model files. + call("reset", ""); + let _ = std::fs::remove_dir_all(&cache); + } + + // -- Offline / air-gapped local-model init (gated; needs a staged model dir) - + // + // Empirically closes §11.4: `configure` with a local `model_path` routes to + // FastEmbedder::new_local (try_new_from_user_defined over on-disk bytes — no + // hf-hub). Runs ONLY when RCORE_TEST_MODEL_DIR points at a pre-staged model + // directory (onnx/model.onnx + the four tokenizer/config files); skipped + // otherwise, so the normal suite and CI stay unaffected. The runner blocks + // network (a dead HTTP(S) proxy + HF_HUB_OFFLINE=1), so a PASS proves the + // local path makes no implicit egress on first `configure` or first embed. + #[cfg(feature = "fastembed")] + #[test] + fn fastembed_offline_local_init() { + let dir = match std::env::var("RCORE_TEST_MODEL_DIR") { + Ok(d) if !d.trim().is_empty() => d, + _ => { + eprintln!( + "RCORE_TEST_MODEL_DIR not set — skipping offline local-init test \ + (set it to a staged model dir to run)" + ); + return; + } + }; + let _g = e2e_guard(); + + // Local path → new_local. dim 384 proves the staged e5-small loaded; a 64 + // would mean new_local failed and we silently fell back to the mock. + let payload = format!(r#"{{"model_path":{},"device":"cpu"}}"#, json!(dir)); + let v = call("configure", &payload); + assert_eq!(v["ok"], json!(true), "offline configure failed: {v}"); + assert_eq!( + v["result"]["dim"].as_u64().unwrap(), + 384, + "offline local e5-small must report dim 384 (got {}); a 64 means new_local \ + failed to load the staged files and we fell back to the mock", + v["result"]["dim"] + ); + + // Exercise the real embed path offline (where any hidden egress in + // try_new_from_user_defined / inference would surface as a failure). + call( + "index_segments", + r#"{"collection":"off","doc_id":"ru","name":"ru-contract", + "segments":[{"text":"Договор поставки товара №123 от 5 июня"}]}"#, + ); + call( + "index_segments", + r#"{"collection":"off","doc_id":"cat","name":"uk-cat", + "segments":[{"text":"Кіт сидить на вікні"}]}"#, + ); + assert!( + store::wait_until_ready("off", std::time::Duration::from_secs(180)), + "offline collection did not reach Ready — the worker failed to embed \ + (an unexpected network dependency would surface here)" + ); + + let v = call( + "search", + r#"{"query":"договор на поставку товаров","collection":"off","mode":"dense","k":10,"include_text":true}"#, + ); + assert_eq!(v["ok"], json!(true), "offline search failed: {v}"); + let hits = v["result"]["hits"].as_array().unwrap(); + assert!(!hits.is_empty(), "offline search returned no hits: {v}"); + assert_eq!( + hits[0]["doc_id"], + json!("ru"), + "offline dense search must rank the contract above the cat: {v}" + ); + eprintln!("offline local-init OK: dim=384, top hit=ru, network blocked"); + } +} diff --git a/rust-core/src/protocol.rs b/rust-core/src/protocol.rs new file mode 100644 index 0000000..4e92395 --- /dev/null +++ b/rust-core/src/protocol.rs @@ -0,0 +1,75 @@ +//! JSON envelopes that cross the FFI boundary. +//! +//! Every call to `rcore_dispatch` returns a JSON object with a top-level `ok` +//! field. On success: `{"ok": true, "result": }`. On failure: +//! `{"ok": false, "error": {"code": "...", "message": "..."}}`. +//! +//! Errors are *structural* — they are normal return values, never panics. This +//! keeps the C ABI boundary panic-free: a malformed payload or unknown method +//! produces a well-formed JSON error object that the C++ side can inspect, +//! rather than unwinding across the FFI boundary (which is undefined behaviour). + +use serde::Serialize; +use serde_json::Value; + +/// Stable error codes returned in the `error.code` field. Kept as `&'static str` +/// constants so call sites (and future Stage 1 code) share one source of truth. +pub mod codes { + pub const UNKNOWN_METHOD: &str = "unknown_method"; + pub const BAD_PAYLOAD: &str = "bad_payload"; + pub const INTERNAL: &str = "internal"; + /// A `grep` pattern failed to compile (invalid regex). Structural, not a panic. + pub const BAD_PATTERN: &str = "bad_pattern"; + /// `configure` named a built-in `model` that is not on the whitelist + /// (see `core::SUPPORTED_MODEL_NAMES` / the `list_models` method). The + /// message lists the supported names. Returned BEFORE the store is touched. + pub const BAD_MODEL: &str = "bad_model"; + /// `get_segment` could not find the requested `doc_id` in any collection. + pub const NOT_FOUND: &str = "not_found"; + /// `get_segment` targeted a document that has no full-text/offset table + /// (an atomic `index_segments` record). Line slicing is unsupported there. + pub const NO_LINE_INDEX: &str = "no_line_index"; +} + +/// A structural error object: `{"code": ..., "message": ...}`. +#[derive(Serialize)] +pub struct ErrorBody { + pub code: &'static str, + pub message: String, +} + +/// The success/failure envelope serialized back to C as a JSON string. +#[derive(Serialize)] +#[serde(untagged)] +pub enum Envelope { + Ok { ok: bool, result: Value }, + Err { ok: bool, error: ErrorBody }, +} + +impl Envelope { + /// Build a success envelope: `{"ok": true, "result": }`. + pub fn ok(result: Value) -> Self { + Envelope::Ok { ok: true, result } + } + + /// Build an error envelope: `{"ok": false, "error": {"code", "message"}}`. + pub fn err(code: &'static str, message: impl Into) -> Self { + Envelope::Err { + ok: false, + error: ErrorBody { + code, + message: message.into(), + }, + } + } + + /// Serialize to a JSON string. Serialization of these envelopes cannot + /// realistically fail (only owned, serializable types), but if it ever + /// does we fall back to a hand-written constant error string so the + /// boundary never returns malformed JSON. + pub fn to_json_string(&self) -> String { + serde_json::to_string(self).unwrap_or_else(|_| { + String::from(r#"{"ok":false,"error":{"code":"internal","message":"serialization failed"}}"#) + }) + } +} diff --git a/vanessa-plugin/MCPRagSearch/MCPRagSearch.xml b/vanessa-plugin/MCPRagSearch/MCPRagSearch.xml new file mode 100644 index 0000000..7da5625 --- /dev/null +++ b/vanessa-plugin/MCPRagSearch/MCPRagSearch.xml @@ -0,0 +1,35 @@ + + + + + + c3831ec8-d8d5-4f93-8a22-f9bfae07327f + 252a528f-575c-4397-9447-34bd75ae406c + + + f5d953c1-c423-4ef7-8eab-f4f5745a1899 + 9a6a9d9a-c148-47de-bf6b-28421ca6232f + + + + MCPRagSearch + + + ru + MCP сервер + семантический поиск (http1c) + + + en + MCP server + semantic search (http1c) + + + + ExternalDataProcessor.MCPRagSearch.Form.Форма + + + +
Форма
+ + + + \ No newline at end of file diff --git a/vanessa-plugin/MCPRagSearch/MCPRagSearch/Ext/ObjectModule.bsl b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Ext/ObjectModule.bsl new file mode 100644 index 0000000..b29a433 --- /dev/null +++ b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Ext/ObjectModule.bsl @@ -0,0 +1,20 @@ +#Область ОписаниеПеременных + +Перем КонтекстЯдра; + +#КонецОбласти + +// { Plugin interface +Функция ОписаниеПлагина(ВозможныеТипыПлагинов) Экспорт + Результат = Новый Структура; + Результат.Вставить("Тип", ВозможныеТипыПлагинов.Утилита); + Результат.Вставить("Идентификатор", Метаданные().Имя); + Результат.Вставить("Представление", "MCP сервер + семантический поиск (http1c)"); + + Возврат Новый ФиксированнаяСтруктура(Результат); +КонецФункции + +Процедура Инициализация(КонтекстЯдраПараметр) Экспорт + КонтекстЯдра = КонтекстЯдраПараметр; +КонецПроцедуры +// } Plugin interface diff --git "a/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260.xml" "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260.xml" new file mode 100644 index 0000000..6e728ff --- /dev/null +++ "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260.xml" @@ -0,0 +1,27 @@ + + +
+ + Форма + + + ru + Форма + + + en + Form + + + + Managed + false + + PlatformApplication + MobilePlatformApplication + + Any + + +
+ \ No newline at end of file diff --git "a/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260/Ext/Form.xml" "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260/Ext/Form.xml" new file mode 100644 index 0000000..cbfde84 --- /dev/null +++ "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260/Ext/Form.xml" @@ -0,0 +1,936 @@ + +
+ + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>MCP сервер + семантический поиск (http1c)</v8:content> + </v8:item> + <v8:item> + <v8:lang>en</v8:lang> + <v8:content>MCP server + semantic search (http1c)</v8:content> + </v8:item> + + Use + + + ПриОткрытии + ПередЗакрытием + ВнешнееСобытие + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Управление</v8:content> + </v8:item> + + Vertical + Usual + NormalSeparation + true + + + + РежимРаботы + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Режим работы</v8:content> + </v8:item> + + + + ru + MCP — компонента обслуживает инструменты Vanessa Automation. MCP+RAG — дополнительно семантический поиск по шагам (нужны rcore.dll и модель эмбеддинга). + + + Tumbler + HorizontalIfPossible + + + + 0 + + + + ru + MCP + + + MCP + + + + + 0 + + + + ru + MCP+RAG + + + MCP+RAG + + + + + + + РежимРаботыПриИзменении + + + + AlwaysHorizontal + Usual + None + + + + + + + + + + СтатусСервера + true + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Сервер</v8:content> + </v8:item> + + true + + + + + СтатусИндексации + true + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Индексация</v8:content> + </v8:item> + + true + + + + + СчетчикИнструментов + true + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Инструментов</v8:content> + </v8:item> + + 10 + false + + + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Настройки сервера</v8:content> + </v8:item> + + Vertical + Usual + NormalSeparation + true + + + + Порт + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Порт</v8:content> + </v8:item> + + + + ru + TCP-порт MCP сервера. ИИ-агент подключается к http://localhost:<порт>/mcp + + + 8 + false + + + ru + 8765 + + + + + + + КаталогКомпоненты + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Каталог компоненты</v8:content> + </v8:item> + + + + ru + Распакованный бандл http1c-addin-full.zip. Компонента подключается прямо из этой папки (in-place), чтобы rcore.dll был рядом. + + + true + true + + + ru + Папка с libhttp1cWin.dll, rcore.dll, DirectML.dll + + + + + + КаталогКомпонентыНачалоВыбора + + + + AlwaysHorizontal + Usual + None + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Скачать компоненту (последний релиз)</v8:content> + </v8:item> + + + + ru + Скачивает http1c-addin-full.zip (libhttp1cWin.dll + rcore.dll + DirectML.dll) из последнего релиза. Если прямая ссылка не работает — откройте страницу релизов: https://github.com/DitriXNew/MCP-DB-Client/releases + + + true + + + + СсылкаСкачатьКомпонентуНажатие + + + + + + + ТокенАвторизации + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Токен авторизации</v8:content> + </v8:item> + + + + ru + Если задан, клиент обязан передавать заголовок Authorization: Bearer <токен>. + + + true + + + ru + (необязательно) + + + + + + + ВестиЛог + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Вести лог</v8:content> + </v8:item> + + + + ru + Писать лог компоненты (укажите файл ниже). + + + + + + + ФайлЛога + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Файл лога</v8:content> + </v8:item> + + + + ru + Полный путь к файлу лога компоненты. + + + true + true + + + ru + C:\temp\http1c.log + + + + + + ФайлЛогаНачалоВыбора + + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Семантический поиск (RAG)</v8:content> + </v8:item> + + Vertical + Usual + NormalSeparation + true + + + + Модель + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Модель эмбеддинга</v8:content> + </v8:item> + + + + ru + Выберите модель из списка — она скачается автоматически при первом запуске (в Каталог моделей). Либо укажите путь к каталогу офлайн-модели (onnx/model.onnx + tokenizer.json и т.д.) — тогда сеть не нужна. + + + true + + + ru + имя из списка или путь к каталогу офлайн-модели + + + + + + МодельПриИзменении + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Страница модели</v8:content> + </v8:item> + + true + + + + СсылкаНаМодельНажатие + + + + КаталогМоделей + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Каталог моделей</v8:content> + </v8:item> + + + + ru + Каталог загрузки/кэша модели при первом запуске. Пусто — кэш fastembed по умолчанию (рядом с процессом 1С). + + + true + true + + + ru + куда скачать/кэшировать модель + + + + + + КаталогМоделейНачалоВыбора + + + + Устройство + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Устройство</v8:content> + </v8:item> + + + + ru + auto — GPU (DirectML) с автоматическим откатом на CPU; dml — только GPU; cpu — только процессор. + + + 10 + false + + + + + 4 + ТаблицаКоллекций + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Коллекции файлов</v8:content> + </v8:item> + + + + ru + Дополнительные коллекции для семантического поиска: файлы из указанных папок (например, фича-файлы проекта) индексируются целиком — работают search, grep и get_segment по строкам файлов. + + + + + + + + ТаблицаКоллекций + SearchStringRepresentation + + + + + + + ТаблицаКоллекций + ViewStatusRepresentation + + + + + + + ТаблицаКоллекций + SearchControl + + + + + + + ТаблицаКоллекций.Коллекция + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Коллекция</v8:content> + </v8:item> + + + + ru + Имя коллекции для поиска (желательно латиница, цифры и подчёркивание, например project_features). + + + EnterOnInput + 15 + + + + + ТаблицаКоллекций.Описание + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Описание</v8:content> + </v8:item> + + + + ru + Описание коллекции для ИИ-агента (передаётся как collection_description). + + + EnterOnInput + 25 + + + + + ТаблицаКоллекций.Каталог + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Каталог</v8:content> + </v8:item> + + + + ru + Путь к папке с файлами (поиск рекурсивный, включая подпапки). + + + EnterOnInput + 30 + true + + + + ТаблицаКоллекцийКаталогНачалоВыбора + + + + ТаблицаКоллекций.Расширения + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Расширения</v8:content> + </v8:item> + + + + ru + Фильтр расширений: «feature» или «feature;md;txt» (допускается и «*.feature»). Пусто — feature. + + + EnterOnInput + 12 + + + ru + feature + + + + + + +
+ + + + + + + cfg:ExternalDataProcessorObject.MCPRagSearch + + true + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Режим работы</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + true + + РежимРаботы + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Порт</v8:content> + </v8:item> + + + xs:decimal + + 5 + 0 + Nonnegative + + + true + + Порт + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Каталог компоненты</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + true + + КаталогКомпоненты + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Токен авторизации</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + true + + ТокенАвторизации + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Вести лог</v8:content> + </v8:item> + + + xs:boolean + + true + + ВестиЛог + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Файл лога</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + true + + ФайлЛога + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Модель</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + true + + Модель + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Каталог моделей</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + true + + КаталогМоделей + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Устройство</v8:content> + </v8:item> + + + xs:string + + 10 + Variable + + + true + + Устройство + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Статус сервера</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Статус индексации</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Инструментов MCP</v8:content> + </v8:item> + + + xs:decimal + + 10 + 0 + Nonnegative + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Коллекции файлов</v8:content> + </v8:item> + + + v8:ValueTable + + true + + ТаблицаКоллекций + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Коллекция</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Описание</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Каталог</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Расширения</v8:content> + </v8:item> + + + xs:string + + 0 + Variable + + + + + + + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Запустить</v8:content> + </v8:item> + + + + ru + Подключить компоненту, собрать инструменты MCPVA и запустить MCP сервер + + + Запустить + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Остановить</v8:content> + </v8:item> + + + + ru + Остановить прослушивание и отключить фасад МСП + + + Остановить + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Переиндексировать</v8:content> + </v8:item> + + + + ru + Пересоздать коллекцию vanessa_steps и коллекции файлов, проиндексировать всё заново + + + Переиндексировать + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Обновить статус</v8:content> + </v8:item> + + + + ru + Опросить состояние транспорта и индексации + + + ОбновитьСтатус + + + + <v8:item> + <v8:lang>ru</v8:lang> + <v8:content>Установить из архива…</v8:content> + </v8:item> + + + + ru + Выбрать скачанный http1c-addin-full.zip, распаковать в каталог и подставить его в поле «Каталог компоненты» + + + УстановитьИзАрхива + + + \ No newline at end of file diff --git "a/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260/Ext/Form/Module.bsl" "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260/Ext/Form/Module.bsl" new file mode 100644 index 0000000..ae8b72c --- /dev/null +++ "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Forms/\320\244\320\276\321\200\320\274\320\260/Ext/Form/Module.bsl" @@ -0,0 +1,1796 @@ +// ============================================================================ +// MCPRagSearch — плагин Vanessa Automation: MCP сервер + семантический поиск. +// +// Транспорт — внешняя компонента http1c (libhttp1cWin.dll, AddIn.VAHttp1c.HttpServer): +// - инструменты MCP собираются из MCPVA (ВызватьРегистрациюИнструментовMCP) +// через фасад этой формы и передаются в компоненту (ПрименитьНастройки); +// - вызовы инструментов приходят событием ВнешнееСобытие/ToolCall и +// диспетчеризуются обратно в обработчики MCPVA; +// - ответ — конверт {"id":..., "body":""}: body — СТРОКА с уже +// сериализованным результатом MCP (двойная сериализация, протокол http1c); +// - промпт search-guide (prompts_json) — инструкция для ИИ-агента из макета +// СкилПоиска, отдаётся по событию PromptGet тем же конвертом {id, body}; +// дубль — инструмент get_search_guide (для клиентов, прячущих промпты). +// +// RAG — Rust-ядро rcore.dll за методом компоненты RagВыполнить(метод, json): +// configure → delete_collection → index_segments (батчи) → база знаний +// (коллекция "vanessa_kb") → коллекции файлов из таблицы формы (на каждую: +// delete_collection → index_raw по одному файлу) → опрос stats до +// vector_status = "ready" по всем коллекциям. +// Коллекция "vanessa_steps" — все шаги VA. +// ============================================================================ + +#Область ОписаниеПеременных + +&НаКлиенте +Перем Ванесса; + +&НаКлиенте +Перем МодульMCPVA; + +&НаКлиенте +Перем Компонента; + +&НаКлиенте +Перем ТранспортЗапущен; + +&НаКлиенте +Перем ОжидающиеЗапросы; // Соответствие: идентификатор запроса -> Истина + +&НаКлиенте +Перем РежимДиспетчеризации; + +&НаКлиенте +Перем РежимСбораИнструментов; + +&НаКлиенте +Перем СписокИнструментовСбор; // Массив структур {name, description, inputSchema} + +&НаКлиенте +Перем КартаОбработчиков; // Соответствие: имя инструмента -> имя процедуры МодульMCPVA + +&НаКлиенте +Перем ОчередьИндексации; // Массив подготовленных пакетов index_segments + +&НаКлиенте +Перем ОчередьКоллекцийФайлов; // Массив структур {Имя, Описание, Каталог, Расширения} из таблицы формы + +&НаКлиенте +Перем ТекущаяКоллекцияФайлов; // Структура обрабатываемой коллекции: + Файлы (массив путей), Индекс, ТекущийФайл + +&НаКлиенте +Перем RagНастроен; + +&НаКлиенте +Перем СсылкаНаМодельURL; // страница выбранной модели на Hugging Face (для декорации СсылкаНаМодель) + +&НаКлиенте +Перем ЯзыкИндексации; // язык генератора Gherkin на момент индексации шагов (meta.lang) + +&НаКлиенте +Перем КоличествоШаговПриИндексации; // размер библиотеки шагов на момент индексации + +#КонецОбласти + +#Область ИнтерфейсПлагина + +&НаКлиенте +Процедура УстановитьСлужебныеПеременные(КонтекстФреймворка) Экспорт + Ванесса = КонтекстФреймворка; +КонецПроцедуры + +&НаКлиенте +Процедура Инициализация(КонтекстЯдраПараметр) Экспорт + Ванесса = КонтекстЯдраПараметр; +КонецПроцедуры + +// Делает отключение модуля +&НаКлиенте +Функция ОтключениеМодуля() Экспорт + + ОстановитьТранспорт(); + Ванесса = Неопределено; + +КонецФункции + +&НаКлиенте +Функция ОписаниеПлагина(ВозможныеТипыПлагинов) Экспорт + Возврат ОписаниеПлагинаНаСервере(ВозможныеТипыПлагинов); +КонецФункции + +&НаСервере +Функция ОписаниеПлагинаНаСервере(ВозможныеТипыПлагинов) + Возврат РеквизитФормыВЗначение("Объект").ОписаниеПлагина(ВозможныеТипыПлагинов); +КонецФункции + +#КонецОбласти + +#Область ОбработчикиСобытийФормы + +&НаКлиенте +Процедура ПриОткрытии(Отказ) +#Если НЕ ВебКлиент Тогда + ИнициализироватьСостояние(); + + Если Порт = 0 Тогда + Порт = 8765; + КонецЕсли; + Если НЕ ЗначениеЗаполнено(РежимРаботы) Тогда + РежимРаботы = "MCP"; + КонецЕсли; + Если НЕ ЗначениеЗаполнено(Модель) Тогда + Модель = "multilingual-e5-small"; + КонецЕсли; + Если НЕ ЗначениеЗаполнено(Устройство) Тогда + Устройство = "auto"; + КонецЕсли; + + Элементы.Модель.КнопкаВыпадающегоСписка = Истина; + ЗаполнитьСписокВыбораМоделей(СписокМоделейПоУмолчанию()); + + Элементы.Устройство.РежимВыбораИзСписка = Истина; + Элементы.Устройство.СписокВыбора.Очистить(); + Элементы.Устройство.СписокВыбора.Добавить("auto"); + Элементы.Устройство.СписокВыбора.Добавить("dml"); + Элементы.Устройство.СписокВыбора.Добавить("cpu"); + + СтатусСервера = "Остановлен"; + СтатусИндексации = ""; + СчетчикИнструментов = 0; + + ОбновитьСсылкуНаМодель(); + ОбновитьДоступностьЭлементов(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ПередЗакрытием(Отказ, ЗавершениеРаботы, ТекстПредупреждения, СтандартнаяОбработка) +#Если НЕ ВебКлиент Тогда + // Без вопросов: просто останавливаем транспорт. + Попытка + ОстановитьТранспорт(); + Исключение + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ВнешнееСобытие(Источник, Событие, Данные) +#Если НЕ ВебКлиент Тогда + Если Источник <> "HttpServer" Тогда + Возврат; + КонецЕсли; + Если Компонента = Неопределено Тогда + Возврат; + КонецЕсли; + + ИнициализироватьСостояние(); + + Если Событие = "ToolCall" Тогда + ОбработатьВызовИнструмента(Данные); + ИначеЕсли Событие = "PromptGet" Тогда + ОбработатьЗапросПромпта(Данные); + ИначеЕсли Событие = "ResourceRead" Тогда + Сообщение = ПрочитатьJSONИзСтроки(Данные); + Если ТипЗнч(Сообщение) = Тип("Соответствие") Тогда + Идентификатор = Сообщение["id"]; + ОжидающиеЗапросы.Вставить(Идентификатор, Истина); + ОтправитьРезультатИнструмента(Идентификатор, Событие + " is not supported", Истина); + КонецЕсли; + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура РежимРаботыПриИзменении(Элемент) + ОбновитьДоступностьЭлементов(); + Если РежимРаботы = "MCP+RAG" И ТранспортЗапущен = Истина И RagНастроен <> Истина Тогда + СтатусИндексации = "Нажмите «Переиндексировать» для запуска семантического поиска"; + КонецЕсли; +КонецПроцедуры + +#КонецОбласти + +#Область ОбработчикиСобытийЭлементовФормы + +&НаКлиенте +Процедура МодельПриИзменении(Элемент) + ОбновитьСсылкуНаМодель(); +КонецПроцедуры + +&НаКлиенте +Процедура СсылкаНаМодельНажатие(Элемент, СтандартнаяОбработка) +#Если НЕ ВебКлиент Тогда + СтандартнаяОбработка = Ложь; + Если НЕ ЗначениеЗаполнено(СсылкаНаМодельURL) Тогда + Возврат; + КонецЕсли; + Попытка + НачатьЗапускПриложения( + Новый ОписаниеОповещения("ЗапускБраузераЗавершение", ЭтотОбъект), СсылкаНаМодельURL); + Исключение + // не удалось открыть браузер — молча игнорируем + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ЗапускБраузераЗавершение(КодВозврата, ДополнительныеПараметры) Экспорт + // fire-and-forget: код возврата браузера не важен +КонецПроцедуры + +&НаКлиенте +Процедура КаталогКомпонентыНачалоВыбора(Элемент, ДанныеВыбора, СтандартнаяОбработка) +#Если НЕ ВебКлиент Тогда + СтандартнаяОбработка = Ложь; + НачатьВыборКаталога("КаталогКомпонентыВыборЗавершение", КаталогКомпоненты, + "Каталог компоненты (libhttp1cWin.dll, rcore.dll, DirectML.dll)"); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура КаталогКомпонентыВыборЗавершение(ВыбранныеФайлы, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Если ТипЗнч(ВыбранныеФайлы) = Тип("Массив") И ВыбранныеФайлы.Количество() > 0 Тогда + КаталогКомпоненты = ВыбранныеФайлы[0]; + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура СсылкаСкачатьКомпонентуНажатие(Элемент, СтандартнаяОбработка) +#Если НЕ ВебКлиент Тогда + СтандартнаяОбработка = Ложь; + Попытка + НачатьЗапускПриложения( + Новый ОписаниеОповещения("ЗапускБраузераЗавершение", ЭтотОбъект), + "https://github.com/DitriXNew/MCP-DB-Client/releases/latest/download/http1c-addin-full.zip"); + Исключение + // не удалось открыть браузер — молча игнорируем + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура КаталогМоделейНачалоВыбора(Элемент, ДанныеВыбора, СтандартнаяОбработка) +#Если НЕ ВебКлиент Тогда + СтандартнаяОбработка = Ложь; + НачатьВыборКаталога("КаталогМоделейВыборЗавершение", КаталогМоделей, + "Каталог загрузки/кэша моделей"); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура КаталогМоделейВыборЗавершение(ВыбранныеФайлы, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Если ТипЗнч(ВыбранныеФайлы) = Тип("Массив") И ВыбранныеФайлы.Количество() > 0 Тогда + КаталогМоделей = ВыбранныеФайлы[0]; + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ФайлЛогаНачалоВыбора(Элемент, ДанныеВыбора, СтандартнаяОбработка) +#Если НЕ ВебКлиент Тогда + СтандартнаяОбработка = Ложь; + Диалог = Новый ДиалогВыбораФайла(РежимДиалогаВыбораФайла.Сохранение); + Диалог.Заголовок = "Файл лога компоненты"; + Диалог.Фильтр = "Лог (*.log;*.txt)|*.log;*.txt|Все файлы|*.*"; + Если ЗначениеЗаполнено(ФайлЛога) Тогда + Диалог.ПолноеИмяФайла = ФайлЛога; + КонецЕсли; + Диалог.Показать(Новый ОписаниеОповещения("ФайлЛогаВыборЗавершение", ЭтотОбъект)); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ФайлЛогаВыборЗавершение(ВыбранныеФайлы, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Если ТипЗнч(ВыбранныеФайлы) = Тип("Массив") И ВыбранныеФайлы.Количество() > 0 Тогда + ФайлЛога = ВыбранныеФайлы[0]; + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ТаблицаКоллекцийКаталогНачалоВыбора(Элемент, ДанныеВыбора, СтандартнаяОбработка) +#Если НЕ ВебКлиент Тогда + СтандартнаяОбработка = Ложь; + ДанныеСтроки = Элементы.ТаблицаКоллекций.ТекущиеДанные; + Если ДанныеСтроки = Неопределено Тогда + Возврат; + КонецЕсли; + // строка таблицы уходит в колбэк через ДополнительныеПараметры + ПараметрыВыбора = Новый Структура("ДанныеСтроки", ДанныеСтроки); + НачатьВыборКаталога("ТаблицаКоллекцийКаталогВыборЗавершение", ДанныеСтроки.Каталог, + "Каталог файлов коллекции", ПараметрыВыбора); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ТаблицаКоллекцийКаталогВыборЗавершение(ВыбранныеФайлы, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Если ТипЗнч(ВыбранныеФайлы) = Тип("Массив") И ВыбранныеФайлы.Количество() > 0 Тогда + ДополнительныеПараметры.ДанныеСтроки.Каталог = ВыбранныеФайлы[0]; + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура НачатьВыборКаталога(ИмяОбработчика, ТекущийКаталог, Заголовок, ДополнительныеПараметры = Неопределено) +#Если НЕ ВебКлиент Тогда + Диалог = Новый ДиалогВыбораФайла(РежимДиалогаВыбораФайла.ВыборКаталога); + Диалог.Заголовок = Заголовок; + Если ЗначениеЗаполнено(ТекущийКаталог) Тогда + Диалог.Каталог = ТекущийКаталог; + КонецЕсли; + Диалог.Показать(Новый ОписаниеОповещения(ИмяОбработчика, ЭтотОбъект, ДополнительныеПараметры)); +#КонецЕсли +КонецПроцедуры + +#КонецОбласти + +#Область ОбработчикиКомандФормы + +&НаКлиенте +Процедура Запустить(Команда) +#Если НЕ ВебКлиент Тогда + НачатьЗапускТранспорта(); +#КонецЕсли +КонецПроцедуры + +// Программный запуск (сервисная фича, e2e, автоматизация). Настройки - Структура +// с любыми из полей: Порт, КаталогКомпоненты, РежимРаботы, Модель, КаталогМоделей, +// Устройство, ТокенАвторизации, ВестиЛог, ФайлЛога. +&НаКлиенте +Процедура ЗапуститьСНастройками(Настройки) Экспорт +#Если НЕ ВебКлиент Тогда + Если ТипЗнч(Настройки) = Тип("Структура") Тогда + Для Каждого КлючИЗначение Из Настройки Цикл + Попытка + ЭтотОбъект[КлючИЗначение.Ключ] = КлючИЗначение.Значение; + Исключение + // нет такого реквизита — пропускаем + КонецПопытки; + КонецЦикла; + КонецЕсли; + ОбновитьСсылкуНаМодель(); + ОбновитьДоступностьЭлементов(); + НачатьЗапускТранспорта(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура НачатьЗапускТранспорта() +#Если НЕ ВебКлиент Тогда + ИнициализироватьСостояние(); + + Если ТранспортЗапущен = Истина Тогда + Сообщить("MCP сервер уже запущен на порту " + Формат(Порт, "ЧГ=0") + "."); + Возврат; + КонецЕсли; + + СистемнаяИнформация = Новый СистемнаяИнформация; + Если СистемнаяИнформация.ТипПлатформы <> ТипПлатформы.Windows_x86_64 Тогда + Сообщить("Компонента http1c доступна только для Windows x86-64."); + Возврат; + КонецЕсли; + + Если Ванесса = Неопределено Тогда + НайтиКонтекстВанессы(); + КонецЕсли; + Если Ванесса = Неопределено Тогда + Сообщить("Откройте Vanessa Automation."); + Возврат; + КонецЕсли; + + МодульMCPVA = Ванесса.МодульMCPVA(); + + Если НЕ ЗначениеЗаполнено(КаталогКомпоненты) Тогда + Сообщить("Укажите каталог компоненты (libhttp1cWin.dll, rcore.dll, DirectML.dll)."); + Возврат; + КонецЕсли; + + ПутьККомпоненте = КаталогКомпоненты; + Если Прав(ПутьККомпоненте, 1) <> "\" И Прав(ПутьККомпоненте, 1) <> "/" Тогда + ПутьККомпоненте = ПутьККомпоненте + "\"; + КонецЕсли; + ПутьККомпоненте = ПутьККомпоненте + "libhttp1cWin.dll"; + + СтатусСервера = "Подключение компоненты..."; + Оповещение = Новый ОписаниеОповещения("ПодключениеКомпонентыЗавершение", ЭтотОбъект); + // НеИзолированно поддерживается не всеми версиями платформы — вызываем через + // Выполнить с откатом на 4-арговый вариант (паттерн VA, УправляемаяФорма). + Попытка + Выполнить("НачатьПодключениеВнешнейКомпоненты(Оповещение, ПутьККомпоненте, ""VAHttp1c"", ТипВнешнейКомпоненты.Native, ТипПодключенияВнешнейКомпоненты.НеИзолированно)"); + Исключение + НачатьПодключениеВнешнейКомпоненты(Оповещение, ПутьККомпоненте, "VAHttp1c", ТипВнешнейКомпоненты.Native); + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура Остановить(Команда) +#Если НЕ ВебКлиент Тогда + Если ТранспортЗапущен <> Истина Тогда + Сообщить("MCP сервер не запущен."); + Возврат; + КонецЕсли; + ОстановитьТранспорт(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура Переиндексировать(Команда) +#Если НЕ ВебКлиент Тогда + Если ТранспортЗапущен <> Истина Тогда + Сообщить("Сначала запустите MCP сервер."); + Возврат; + КонецЕсли; + Если РежимРаботы <> "MCP+RAG" Тогда + Сообщить("Переиндексация доступна только в режиме MCP+RAG."); + Возврат; + КонецЕсли; + + Если RagНастроен <> Истина Тогда + // configure ещё не выполнялся (режим включили на работающем сервере) + НачатьКонфигурациюRag(); + Иначе + СтатусИндексации = "Очистка коллекции..."; + РагВызов("delete_collection", СтрокаУдаленияКоллекции(), "УдалениеКоллекцииЗавершение"); + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ОбновитьСтатус(Команда) +#Если НЕ ВебКлиент Тогда + Если Компонента = Неопределено Тогда + СтатусСервера = "Остановлен"; + Возврат; + КонецЕсли; + Попытка + Компонента.НачатьВызовПолучитьСтатус( + Новый ОписаниеОповещения("ПолучениеСтатусаЗавершение", ЭтотОбъект)); + Исключение + СтатусСервера = "Ошибка опроса статуса: " + ОписаниеОшибки(); + КонецПопытки; + Если РежимРаботы = "MCP+RAG" И ТранспортЗапущен = Истина Тогда + РагВызов("stats", "{}", "ОпросСтатусаИндексацииЗавершение"); + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +// Установка компоненты из скачанного бандла: выбор архива → выбор каталога → +// распаковка → проверка состава → подстановка каталога в КаталогКомпоненты. +&НаКлиенте +Процедура УстановитьИзАрхива(Команда) +#Если НЕ ВебКлиент Тогда + Диалог = Новый ДиалогВыбораФайла(РежимДиалогаВыбораФайла.Открытие); + Диалог.Заголовок = "Выберите http1c-addin-full.zip"; + Диалог.Фильтр = "Бандл компоненты (*.zip)|*.zip"; + Диалог.Показать(Новый ОписаниеОповещения("УстановкаИзАрхиваВыборАрхиваЗавершение", ЭтотОбъект)); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура УстановкаИзАрхиваВыборАрхиваЗавершение(ВыбранныеФайлы, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Если ТипЗнч(ВыбранныеФайлы) <> Тип("Массив") ИЛИ ВыбранныеФайлы.Количество() = 0 Тогда + Возврат; + КонецЕсли; + + ПараметрыУстановки = Новый Структура("ПутьКАрхиву", ВыбранныеФайлы[0]); + + Диалог = Новый ДиалогВыбораФайла(РежимДиалогаВыбораФайла.ВыборКаталога); + Диалог.Заголовок = "Куда распаковать компоненту"; + Если ЗначениеЗаполнено(КаталогКомпоненты) Тогда + Диалог.Каталог = КаталогКомпоненты; + КонецЕсли; + Диалог.Показать(Новый ОписаниеОповещения( + "УстановкаИзАрхиваВыборКаталогаЗавершение", ЭтотОбъект, ПараметрыУстановки)); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура УстановкаИзАрхиваВыборКаталогаЗавершение(ВыбранныеФайлы, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Если ТипЗнч(ВыбранныеФайлы) <> Тип("Массив") ИЛИ ВыбранныеФайлы.Количество() = 0 Тогда + Возврат; + КонецЕсли; + + ЦелевойКаталог = ВыбранныеФайлы[0]; + + Попытка + ЧтениеАрхива = Новый ЧтениеZipФайла(ДополнительныеПараметры.ПутьКАрхиву); + ЧтениеАрхива.ИзвлечьВсе(ЦелевойКаталог, РежимВосстановленияПутейФайловZIP.Восстанавливать); + ЧтениеАрхива.Закрыть(); + Исключение + Сообщить("Не удалось распаковать архив: " + ОписаниеОшибки()); + Возврат; + КонецПопытки; + + КаталогСРазделителем = ЦелевойКаталог; + Если Прав(КаталогСРазделителем, 1) <> "\" И Прав(КаталогСРазделителем, 1) <> "/" Тогда + КаталогСРазделителем = КаталогСРазделителем + "\"; + КонецЕсли; + + ФайлКомпоненты = Новый Файл(КаталогСРазделителем + "libhttp1cWin.dll"); + Если НЕ ФайлКомпоненты.Существует() Тогда + Сообщить("В архиве нет libhttp1cWin.dll — это не бандл компоненты http1c."); + Возврат; + КонецЕсли; + + ФайлЯдра = Новый Файл(КаталогСРазделителем + "rcore.dll"); + Если НЕ ФайлЯдра.Существует() Тогда + Сообщить("В бандле нет rcore.dll — это lite-бандл, семантический поиск (режим MCP+RAG) будет недоступен."); + КонецЕсли; + + КаталогКомпоненты = ЦелевойКаталог; + Сообщить("Компонента установлена в " + ЦелевойКаталог + ". Нажмите ""Запустить""."); +#КонецЕсли +КонецПроцедуры + +#КонецОбласти + +#Область ЗапускТранспорта + +&НаКлиенте +Процедура ПодключениеКомпонентыЗавершение(Подключение, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Если НЕ Подключение Тогда + СтатусСервера = "Ошибка: не удалось подключить libhttp1cWin.dll"; + Сообщить("Не удалось подключить внешнюю компоненту из каталога: " + КаталогКомпоненты); + Возврат; + КонецЕсли; + + Попытка + Компонента = Новый("AddIn.VAHttp1c.HttpServer"); + Исключение + СтатусСервера = "Ошибка создания объекта компоненты"; + Сообщить(ОписаниеОшибки()); + Возврат; + КонецПопытки; + + Попытка + СобратьИнструментыMCP(); + Исключение + СтатусСервера = "Ошибка сбора инструментов MCP"; + Сообщить(ОписаниеОшибки()); + Возврат; + КонецПопытки; + + Конфигурация = Новый Структура; + Конфигурация.Вставить("tools_json", ЗаписатьJSONВСтроку(СписокИнструментовСбор)); + Конфигурация.Вставить("prompts_json", ПромптыJson()); + Конфигурация.Вставить("timeout", 600); + Конфигурация.Вставить("logging_enabled", ВестиЛог = Истина); + Если ЗначениеЗаполнено(ФайлЛога) Тогда + Конфигурация.Вставить("log_path", ФайлЛога); + КонецЕсли; + Если ЗначениеЗаполнено(ТокенАвторизации) Тогда + Конфигурация.Вставить("auth_token", ТокенАвторизации); + КонецЕсли; + + СтатусСервера = "Применение настроек..."; + Попытка + Компонента.НачатьВызовПрименитьНастройки( + Новый ОписаниеОповещения("ПрименениеНастроекЗавершение", ЭтотОбъект), + ЗаписатьJSONВСтроку(Конфигурация)); + Исключение + СтатусСервера = "Ошибка применения настроек"; + Сообщить(ОписаниеОшибки()); + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ПрименениеНастроекЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + СтатусСервера = "Запуск прослушивания..."; + Попытка + Компонента.НачатьВызовНачатьПрослушивание( + Новый ОписаниеОповещения("ПрослушиваниеЗапущено", ЭтотОбъект), Порт); + Исключение + СтатусСервера = "Ошибка запуска прослушивания"; + Сообщить(ОписаниеОшибки()); + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ПрослушиваниеЗапущено(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + ТранспортЗапущен = Истина; + МодульMCPVA.УстановитьВнешнийФасадМсп(ЭтотОбъект); + СтатусСервера = "Запущен: http://localhost:" + Формат(Порт, "ЧГ=0") + "/mcp" + + ", инструментов: " + Формат(СчетчикИнструментов, "ЧН=0; ЧГ=0"); + ОбновитьДоступностьЭлементов(); + + // Заполняем выпадающий список моделей из ядра (на lite-компоненте вернётся ошибка — игнорируем) + РагВызов("list_models", "{}", "ПолучениеСпискаМоделейЗавершение"); + + Если РежимРаботы = "MCP+RAG" Тогда + НачатьКонфигурациюRag(); + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура СобратьИнструментыMCP() +#Если НЕ ВебКлиент Тогда + КартаОбработчиков = Новый Соответствие; + СписокИнструментовСбор = Новый Массив; + + ПараметрыСозданияMCP = Новый Структура; + ПараметрыСозданияMCP.Вставить("МодульМсп_Сервер", ЭтотОбъект); + ПараметрыСозданияMCP.Вставить("МодульМсп_СхемаПараметров", ЭтотОбъект); + + РежимСбораИнструментов = Истина; + Попытка + МодульMCPVA.ВызватьРегистрациюИнструментовMCP(ПараметрыСозданияMCP); + Исключение + РежимСбораИнструментов = Ложь; + ВызватьИсключение; + КонецПопытки; + РежимСбораИнструментов = Ложь; + + // собственный инструмент плагина: скил по поиску (см. ОбработатьВызовИнструмента) + Инструмент = Новый Структура; + Инструмент.Вставить("name", "get_search_guide"); + Инструмент.Вставить("description", + "Вернуть инструкцию (skill), как искать шаги, примеры и правила Vanessa Automation" + + " через семантический поиск: коллекции, режимы search, фильтры, workflow написания" + + " сценария. Вызови один раз перед подбором шагов или написанием сценария."); + Инструмент.Вставить("inputSchema", НовыйСхемаОбъекта(Новый Структура, Новый Массив)); + СписокИнструментовСбор.Добавить(Инструмент); + + СчетчикИнструментов = СписокИнструментовСбор.Количество(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ОстановитьТранспорт() +#Если НЕ ВебКлиент Тогда + Если ТранспортЗапущен = Истина И МодульMCPVA <> Неопределено Тогда + Попытка + МодульMCPVA.УстановитьВнешнийФасадМсп(Неопределено); + Исключение + КонецПопытки; + КонецЕсли; + Если Компонента <> Неопределено Тогда + Попытка + Компонента.НачатьВызовОстановитьПрослушивание(ПустоеОповещение()); + Исключение + КонецПопытки; + КонецЕсли; + ТранспортЗапущен = Ложь; + RagНастроен = Ложь; + Попытка + ОтключитьОбработчикОжидания("КонтрольАктуальностиИндекса"); + Исключение + КонецПопытки; + СтатусСервера = "Остановлен"; + СтатусИндексации = ""; + ОбновитьДоступностьЭлементов(); +#КонецЕсли +КонецПроцедуры + +// Fallback: форма открыта не как плагин — ищем открытое окно Vanessa Automation. +&НаКлиенте +Процедура НайтиКонтекстВанессы() +#Если НЕ ВебКлиент Тогда + // "Окно" как имя переменной нельзя: это свойство формы (read-only, Window) + Для Каждого ТекущееОкноКлиента Из ПолучитьОкна() Цикл + Для Каждого ФормаОкна Из ТекущееОкноКлиента.Содержимое Цикл + Попытка + Если СтрНайти(ФормаОкна.ИмяФормы, "VanessaAutomation") = 0 Тогда + Продолжить; + КонецЕсли; + // Проверяем, что это главная форма VA, через известный экспортный метод + Проверка = ФормаОкна.ПолучитьКонтекстСохраняемый(); + Ванесса = ФормаОкна; + Возврат; + Исключение + // не форма VA (нет ИмяФормы или метода) — пропускаем + КонецПопытки; + КонецЦикла; + КонецЦикла; +#КонецЕсли +КонецПроцедуры + +#КонецОбласти + +#Область ФасадМспСервера + +// Экспортные функции — фасад модулей Мсп_Сервер/Мсп_СхемаПараметров для MCPVA. +// В режиме сбора инструментов накапливают определения, в режиме диспетчеризации +// замыкают ответы на транспорт http1c, иначе делегируют в реальное расширение МСП. + +&НаКлиенте +Функция НовыйСвойствоСхемы(Тип, Описание) Экспорт + Возврат Новый Структура("type, description", Тип, Описание); +КонецФункции + +&НаКлиенте +Функция НовыйСхемаОбъекта(Свойства, ТребуемыеПоля) Экспорт + Схема = Новый Структура; + Схема.Вставить("type", "object"); + Схема.Вставить("properties", Свойства); + Схема.Вставить("required", ТребуемыеПоля); + Возврат Схема; +КонецФункции + +&НаКлиенте +Функция НовыйИнструмент(Имя, Описание, Схема, Оповещение, Знач Асинхронный = Ложь) Экспорт + Если РежимСбораИнструментов = Истина Тогда + Инструмент = Новый Структура; + Инструмент.Вставить("name", Имя); + Инструмент.Вставить("description", Описание); + Инструмент.Вставить("inputSchema", Схема); + Инструмент.Вставить("_обработчик", Оповещение.ИмяПроцедуры); + Возврат Инструмент; + КонецЕсли; + + МодульМсп = Вычислить("Мсп_Сервер"); + Возврат МодульМсп.НовыйИнструмент(Имя, Описание, Схема, Оповещение, Асинхронный); +КонецФункции + +&НаКлиенте +Процедура ЗарегистрироватьИнструмент(Инструмент, ИмяПровайдера) Экспорт + Если РежимСбораИнструментов = Истина Тогда + КартаОбработчиков.Вставить(Инструмент.name, Инструмент._обработчик); + // в tools_json уходит чистое определение, без служебного поля + Определение = Новый Структура; + Определение.Вставить("name", Инструмент.name); + Определение.Вставить("description", Инструмент.description); + Определение.Вставить("inputSchema", Инструмент.inputSchema); + СписокИнструментовСбор.Добавить(Определение); + Возврат; + КонецЕсли; + + МодульМсп = Вычислить("Мсп_Сервер"); + МодульМсп.ЗарегистрироватьИнструмент(Инструмент, ИмяПровайдера); +КонецПроцедуры + +&НаКлиенте +Функция ОшибкаИнструмента(Код, Текст) Экспорт + Если РежимДиспетчеризации = Истина Тогда + Возврат Новый Структура("Http1cОшибкаИнструмента, Код, Сообщение", Истина, Код, Текст); + КонецЕсли; + + МодульМсп = Вычислить("Мсп_Сервер"); + Возврат МодульМсп.ОшибкаИнструмента(Код, Текст); +КонецФункции + +&НаКлиенте +Процедура ЗавершитьВызовИнструментаУспешно(Идентификатор, Текст) Экспорт + ИнициализироватьСостояние(); + Если ОжидающиеЗапросы[Идентификатор] <> Неопределено Тогда + ОтправитьРезультатИнструмента(Идентификатор, Текст, Ложь); + Возврат; + КонецЕсли; + + // чужой/устаревший идентификатор: без client_mcp делегировать некуда + Попытка + МодульМсп = Вычислить("Мсп_Сервер"); + МодульМсп.ЗавершитьВызовИнструментаУспешно(Идентификатор, Текст); + Исключение + КонецПопытки; +КонецПроцедуры + +&НаКлиенте +Процедура ЗавершитьВызовИнструментаСОшибкой(Идентификатор, Код, Текст) Экспорт + ИнициализироватьСостояние(); + Если ОжидающиеЗапросы[Идентификатор] <> Неопределено Тогда + ОтправитьРезультатИнструмента(Идентификатор, Код + ": " + Текст, Истина); + Возврат; + КонецЕсли; + + // чужой/устаревший идентификатор: без client_mcp делегировать некуда + Попытка + МодульМсп = Вычислить("Мсп_Сервер"); + МодульМсп.ЗавершитьВызовИнструментаСОшибкой(Идентификатор, Код, Текст); + Исключение + КонецПопытки; +КонецПроцедуры + +&НаКлиенте +Процедура УведомитьОПрогрессе(Идентификатор, Номер, Всего, Сообщение) Экспорт +#Если НЕ ВебКлиент Тогда + ИнициализироватьСостояние(); + Если ОжидающиеЗапросы[Идентификатор] <> Неопределено Тогда + Если Компонента <> Неопределено Тогда + Компонента.НачатьВызовОтправитьПрогресс(ПустоеОповещение(), Идентификатор, Номер, Всего, Сообщение); + КонецЕсли; + Возврат; + КонецЕсли; + + Попытка + МодульМсп = Вычислить("Мсп_Сервер"); + МодульМсп.УведомитьОПрогрессе(Идентификатор, Номер, Всего, Сообщение); + Исключение + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +#КонецОбласти + +#Область ДиспетчеризацияИнструментов + +&НаКлиенте +Процедура ОбработатьВызовИнструмента(Данные) +#Если НЕ ВебКлиент Тогда + Сообщение = ПрочитатьJSONИзСтроки(Данные); + Если ТипЗнч(Сообщение) <> Тип("Соответствие") Тогда + Возврат; + КонецЕсли; + + Идентификатор = Сообщение["id"]; + ИмяИнструмента = Сообщение["tool"]; + ОжидающиеЗапросы.Вставить(Идентификатор, Истина); + + // собственный инструмент плагина: скил по поиску из макета (дубль промпта + // search-guide — инструменты, в отличие от промптов, агент видит всегда) + Если ИмяИнструмента = "get_search_guide" Тогда + Попытка + ОтправитьРезультатИнструмента(Идентификатор, ТекстМакета("СкилПоиска"), Ложь); + Исключение + ОтправитьРезультатИнструмента(Идентификатор, "ACTION_FAILED: " + ОписаниеОшибки(), Истина); + КонецПопытки; + Возврат; + КонецЕсли; + + ИмяОбработчика = КартаОбработчиков[ИмяИнструмента]; + Если ИмяОбработчика = Неопределено Тогда + ОтправитьРезультатИнструмента(Идентификатор, "Unknown tool: " + Строка(ИмяИнструмента), Истина); + Возврат; + КонецЕсли; + + // Аргументы для обработчиков MCPVA — Структура (Запрос.Аргументы.Свойство()) + Аргументы = Новый Структура; + АргументыСообщения = Сообщение["arguments"]; + Если ТипЗнч(АргументыСообщения) = Тип("Соответствие") Тогда + Для Каждого КлючИЗначение Из АргументыСообщения Цикл + Попытка + Аргументы.Вставить(КлючИЗначение.Ключ, КлючИЗначение.Значение); + Исключение + // ключ не является допустимым идентификатором — пропускаем + КонецПопытки; + КонецЦикла; + КонецЕсли; + + Запрос = Новый Структура("Идентификатор, Аргументы", Идентификатор, Аргументы); + + РежимДиспетчеризации = Истина; + Попытка + Результат = Вычислить("МодульMCPVA." + ИмяОбработчика + "(Запрос, Неопределено)"); + Исключение + РежимДиспетчеризации = Ложь; + ОтправитьРезультатИнструмента(Идентификатор, "ACTION_FAILED: " + ОписаниеОшибки(), Истина); + Возврат; + КонецПопытки; + РежимДиспетчеризации = Ложь; + + Если Результат = Неопределено Тогда + // отложенный ответ: обработчик завершит вызов через ЗавершитьВызовИнструмента* + ИначеЕсли ТипЗнч(Результат) = Тип("Структура") И Результат.Свойство("Http1cОшибкаИнструмента") Тогда + ОтправитьРезультатИнструмента(Идентификатор, Результат.Код + ": " + Результат.Сообщение, Истина); + Иначе + ОтправитьРезультатИнструмента(Идентификатор, Строка(Результат), Ложь); + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ОтправитьРезультатИнструмента(Идентификатор, Текст, ЭтоОшибка) +#Если НЕ ВебКлиент Тогда + ОжидающиеЗапросы.Удалить(Идентификатор); + + Если Компонента = Неопределено Тогда + Возврат; + КонецЕсли; + + ЭлементСодержимого = Новый Структура("type, text", "text", Текст); + Содержимое = Новый Массив; + Содержимое.Добавить(ЭлементСодержимого); + + Результат = Новый Структура; + Результат.Вставить("content", Содержимое); + Если ЭтоОшибка Тогда + Результат.Вставить("isError", Истина); + КонецЕсли; + + // body — строка с уже сериализованным результатом (двойная сериализация) + Конверт = Новый Структура; + Конверт.Вставить("id", Идентификатор); + Конверт.Вставить("body", ЗаписатьJSONВСтроку(Результат)); + + Компонента.НачатьВызовОтправитьОтвет(ПустоеОповещение(), ЗаписатьJSONВСтроку(Конверт)); +#КонецЕсли +КонецПроцедуры + +#КонецОбласти + +#Область ПромптыMCP + +// Определения промптов MCP для prompts_json компоненты (сериализованный массив +// [{name, description, arguments}]). Один промпт search-guide без аргументов: +// инструкция для ИИ-агента, текст отдаёт ОбработатьЗапросПромпта из макета СкилПоиска. +&НаКлиенте +Функция ПромптыJson() + Промпты = Новый Массив; + + Промпт = Новый Структура; + Промпт.Вставить("name", "search-guide"); + Промпт.Вставить("title", "Vanessa: как искать шаги (RAG)"); + Промпт.Вставить("description", + "Инструкция для ИИ-агента: как искать шаги, примеры и правила Vanessa Automation" + + " через семантический поиск (list_collections, search, grep, get_segment)"); + Промпт.Вставить("arguments", Новый Массив); + Промпты.Добавить(Промпт); + + Возврат ЗаписатьJSONВСтроку(Промпты); +КонецФункции + +// PromptGet: запрос {id, name} -> ответ prompts/get +// {"messages":[{"role":"user","content":{"type":"text","text":"..."}}]}, +// при ошибке/неизвестном имени — {"error":"<текст>"} (протокол http1c). +&НаКлиенте +Процедура ОбработатьЗапросПромпта(Данные) +#Если НЕ ВебКлиент Тогда + Сообщение = ПрочитатьJSONИзСтроки(Данные); + Если ТипЗнч(Сообщение) <> Тип("Соответствие") Тогда + Возврат; + КонецЕсли; + + Идентификатор = Сообщение["id"]; + ИмяПромпта = Строка(Сообщение["name"]); + ОжидающиеЗапросы.Вставить(Идентификатор, Истина); + + Если ИмяПромпта <> "search-guide" И ИмяПромпта <> "search_guide" Тогда + ОтправитьРезультатПромпта(Идентификатор, + Новый Структура("error", "Unknown prompt: " + ИмяПромпта)); + Возврат; + КонецЕсли; + + Попытка + ТекстПромпта = ТекстМакета("СкилПоиска"); + Исключение + ОтправитьРезультатПромпта(Идентификатор, + Новый Структура("error", "Prompt error: " + ОписаниеОшибки())); + Возврат; + КонецПопытки; + + Содержимое = Новый Структура("type, text", "text", ТекстПромпта); + СообщениеПромпта = Новый Структура("role, content", "user", Содержимое); + Сообщения = Новый Массив; + Сообщения.Добавить(СообщениеПромпта); + + ОтправитьРезультатПромпта(Идентификатор, Новый Структура("messages", Сообщения)); +#КонецЕсли +КонецПроцедуры + +// Конверт {id, body} как у ОтправитьРезультатИнструмента, но body — это +// сериализованный результат prompts/get целиком (без обёртки content/isError). +&НаКлиенте +Процедура ОтправитьРезультатПромпта(Идентификатор, Результат) +#Если НЕ ВебКлиент Тогда + ОжидающиеЗапросы.Удалить(Идентификатор); + + Если Компонента = Неопределено Тогда + Возврат; + КонецЕсли; + + Конверт = Новый Структура; + Конверт.Вставить("id", Идентификатор); + Конверт.Вставить("body", ЗаписатьJSONВСтроку(Результат)); + + Компонента.НачатьВызовОтправитьОтвет(ПустоеОповещение(), ЗаписатьJSONВСтроку(Конверт)); +#КонецЕсли +КонецПроцедуры + +&НаСервере +Функция ТекстМакета(ИмяМакета) + Возврат РеквизитФормыВЗначение("Объект").ПолучитьМакет(ИмяМакета).ПолучитьТекст(); +КонецФункции + +#КонецОбласти + +#Область RAG + +&НаКлиенте +Процедура НачатьКонфигурациюRag() +#Если НЕ ВебКлиент Тогда + СтатусИндексации = "Подготовка семантического поиска: загрузка модели «" + Модель + + "» и индексация шагов. При первом запуске модель скачивается — это может занять несколько минут..."; + + Настройки = Новый Структура; + Если СтрНайти(Модель, "\") > 0 ИЛИ СтрНайти(Модель, "/") > 0 ИЛИ СтрНайти(Модель, ":") > 0 Тогда + Настройки.Вставить("model_path", Модель); + Иначе + Настройки.Вставить("model", Модель); + КонецЕсли; + Если ЗначениеЗаполнено(КаталогМоделей) Тогда + Настройки.Вставить("cache_dir", КаталогМоделей); + КонецЕсли; + Настройки.Вставить("device", Устройство); + Настройки.Вставить("embed_workers", 0); // 0 = авто (ncpu/2, максимум 4) + Настройки.Вставить("normalize", Истина); + + РагВызов("configure", ЗаписатьJSONВСтроку(Настройки), "КонфигурацияRagЗавершение"); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура КонфигурацияRagЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Конверт = ПрочитатьJSONИзСтроки(РезультатВызова); + Если ТипЗнч(Конверт) <> Тип("Соответствие") ИЛИ Конверт["ok"] <> Истина Тогда + // MCP транспорт продолжает работать без семантического поиска + СтатусИндексации = "Ошибка конфигурации RAG: " + ТекстОшибкиКонверта(Конверт, РезультатВызова); + Возврат; + КонецЕсли; + + RagНастроен = Истина; + ПодключитьОбработчикОжидания("КонтрольАктуальностиИндекса", 30, Истина); + СтатусИндексации = "Очистка коллекции..."; + РагВызов("delete_collection", СтрокаУдаленияКоллекции(), "УдалениеКоллекцииЗавершение"); +#КонецЕсли +КонецПроцедуры + +// Следит за актуальностью индекса шагов: при смене языка генератора Gherkin или +// изменении размера библиотеки шагов (подключили/перечитали библиотеки) запускает +// переиндексацию автоматически. Перевзводится сам, пока транспорт запущен. +&НаКлиенте +Процедура КонтрольАктуальностиИндекса() Экспорт +#Если НЕ ВебКлиент Тогда + Если ТранспортЗапущен <> Истина ИЛИ RagНастроен <> Истина ИЛИ РежимРаботы <> "MCP+RAG" Тогда + Возврат; + КонецЕсли; + + ТекущийЯзык = ТекущийЯзыкГенератора(); + ТекущееКоличествоШагов = КоличествоИзвестныхШагов(); + + НужнаПереиндексация = Ложь; + Если ЗначениеЗаполнено(ЯзыкИндексации) И ТекущийЯзык <> ЯзыкИндексации Тогда + Сообщить("Язык генератора Gherkin изменился (" + ЯзыкИндексации + " -> " + ТекущийЯзык + + ") — запускаю переиндексацию."); + НужнаПереиндексация = Истина; + ИначеЕсли КоличествоШаговПриИндексации <> Неопределено И ТекущееКоличествоШагов <> Неопределено + И ТекущееКоличествоШагов <> КоличествоШаговПриИндексации Тогда + Сообщить("Библиотека шагов изменилась (" + Формат(КоличествоШаговПриИндексации, "ЧН=0; ЧГ=0") + + " -> " + Формат(ТекущееКоличествоШагов, "ЧН=0; ЧГ=0") + " шагов) — запускаю переиндексацию."); + НужнаПереиндексация = Истина; + КонецЕсли; + + Если НужнаПереиндексация Тогда + СтатусИндексации = "Данные изменились: переиндексация..."; + РагВызов("delete_collection", СтрокаУдаленияКоллекции(), "УдалениеКоллекцииЗавершение"); + КонецЕсли; + + ПодключитьОбработчикОжидания("КонтрольАктуальностиИндекса", 30, Истина); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Функция ТекущийЯзыкГенератора() + Язык = "ru"; + Попытка + ЯзыкНастройки = СокрЛП(Ванесса.Объект.ЯзыкГенератораGherkin); + Если ЗначениеЗаполнено(ЯзыкНастройки) Тогда + Язык = ЯзыкНастройки; + КонецЕсли; + Исключение + // настройка недоступна (старая версия VA) — считаем ru + КонецПопытки; + Возврат Язык; +КонецФункции + +&НаКлиенте +Функция КоличествоИзвестныхШагов() + Попытка + Возврат Ванесса.ТаблицаИзвестныхStepDefinition.Количество(); + Исключение + Возврат Неопределено; + КонецПопытки; +КонецФункции + +&НаКлиенте +Процедура УдалениеКоллекцииЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + // результат не важен: коллекции могло не быть + НачатьИндексациюШагов(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура НачатьИндексациюШагов() +#Если НЕ ВебКлиент Тогда + СтатусИндексации = "Выгрузка шагов из Vanessa Automation..."; + + Попытка + ШагиJson = Ванесса.ВсеДоступныеШагиMCP(); + Исключение + СтатусИндексации = "Ошибка выгрузки шагов: " + ОписаниеОшибки(); + Возврат; + КонецПопытки; + + Шаги = ПрочитатьJSONИзСтроки(ШагиJson); + Если ТипЗнч(Шаги) <> Тип("Массив") ИЛИ Шаги.Количество() = 0 Тогда + СтатусИндексации = "Шаги не получены (пустой ответ Vanessa Automation)"; + Возврат; + КонецЕсли; + + // запоминаем язык и размер библиотеки шагов — КонтрольАктуальностиИндекса + // сравнит их с текущими и при расхождении запустит переиндексацию + ЯзыкИндексации = ТекущийЯзыкГенератора(); + КоличествоШаговПриИндексации = КоличествоИзвестныхШагов(); + + ОчередьИндексации = Новый Массив; + НомерБатча = 0; + Сегменты = Новый Массив; + ВсегоСегментов = 0; + + Для Каждого Шаг Из Шаги Цикл + ИмяШага = Строка(Шаг["ИмяШага"]); + Если НЕ ЗначениеЗаполнено(ИмяШага) Тогда + Продолжить; + КонецЕсли; + ОписаниеШага = Строка(Шаг["ОписаниеШага"]); + ТипШага = Строка(Шаг["ПолныйТипШага"]); + + Сегмент = Новый Структура; + Сегмент.Вставить("text", ИмяШага); + Сегмент.Вставить("embed_text", ИмяШага + Символы.ПС + СинонимыДействий(ИмяШага) + + Символы.ПС + Лев(ОписаниеШага, 500)); + Сегмент.Вставить("meta", Новый Структура("type, lang", ТипШага, ЯзыкИндексации)); + Сегменты.Добавить(Сегмент); + ВсегоСегментов = ВсегоСегментов + 1; + + Если Сегменты.Количество() >= 400 Тогда + НомерБатча = НомерБатча + 1; + ОчередьИндексации.Добавить(НовыйПакетИндексации(НомерБатча, Сегменты)); + Сегменты = Новый Массив; + КонецЕсли; + КонецЦикла; + + Если Сегменты.Количество() > 0 Тогда + НомерБатча = НомерБатча + 1; + ОчередьИндексации.Добавить(НовыйПакетИндексации(НомерБатча, Сегменты)); + КонецЕсли; + + СтатусИндексации = "Индексация: " + Формат(ВсегоСегментов, "ЧГ=0") + " шагов, " + + Формат(НомерБатча, "ЧГ=0") + " пакетов..."; + ОтправитьСледующийПакетИндексации(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Функция НовыйПакетИндексации(НомерБатча, Сегменты) + Пакет = Новый Структура; + Пакет.Вставить("collection", "vanessa_steps"); + Пакет.Вставить("collection_description", "Шаги Vanessa Automation (Gherkin/BDD)"); + Пакет.Вставить("doc_id", "vanessa_steps_" + Формат(НомерБатча, "ЧГ=0")); + Пакет.Вставить("segments", Сегменты); + Возврат Пакет; +КонецФункции + +// Синонимы действий для embed_text: «кликаю/click» семантически ложатся на +// «нажимаю на кнопку» и т.п. — закрывает запросы, где модель не дотягивает +// лексику (текст шага в выдаче не меняется, обогащается только эмбеддинг). +&НаКлиенте +Функция СинонимыДействий(ИмяШага) + ИмяНРег = НРег(ИмяШага); + Термины = Новый Массив; + ДобавитьСинонимы(Термины, ИмяНРег, "нажимаю", "кликаю щёлкаю щелкаю клик мышью press click"); + ДобавитьСинонимы(Термины, ИмяНРег, "ввожу", "заполняю печатаю набираю type fill input"); + ДобавитьСинонимы(Термины, ИмяНРег, "заполняю", "ввожу устанавливаю значение set fill"); + ДобавитьСинонимы(Термины, ИмяНРег, "выбираю", "выбор choose select"); + ДобавитьСинонимы(Термины, ИмяНРег, "открываю", "запускаю open"); + ДобавитьСинонимы(Термины, ИмяНРег, "закрываю", "завершаю close"); + ДобавитьСинонимы(Термины, ИмяНРег, "жду", "ожидаю ожидание пауза wait"); + ДобавитьСинонимы(Термины, ИмяНРег, "проверяю", "проверка убеждаюсь assert check verify"); + ДобавитьСинонимы(Термины, ИмяНРег, "кнопк", "button"); + ДобавитьСинонимы(Термины, ИмяНРег, "поле", "field input"); + ДобавитьСинонимы(Термины, ИмяНРег, "таблиц", "список grid table list"); + ДобавитьСинонимы(Термины, ИмяНРег, "флаж", "галочка чекбокс checkbox"); + ДобавитьСинонимы(Термины, ИмяНРег, "флаг", "галочка чекбокс checkbox"); + ДобавитьСинонимы(Термины, ИмяНРег, "гиперссылк", "ссылка link"); + ДобавитьСинонимы(Термины, ИмяНРег, "переключатель", "радиокнопка radio"); + ДобавитьСинонимы(Термины, ИмяНРег, "окн", "window"); + Возврат СтрСоединить(Термины, " "); +КонецФункции + +&НаКлиенте +Процедура ДобавитьСинонимы(Термины, ИмяНРег, Подстрока, Синонимы) + Если СтрНайти(ИмяНРег, Подстрока) > 0 Тогда + Термины.Добавить(Синонимы); + КонецЕсли; +КонецПроцедуры + +&НаКлиенте +Процедура ОтправитьСледующийПакетИндексации() +#Если НЕ ВебКлиент Тогда + Если ОчередьИндексации.Количество() = 0 Тогда + // шаги отправлены — дальше база знаний, затем коллекции файлов + НачатьИндексациюБазыЗнаний(); + Возврат; + КонецЕсли; + + Пакет = ОчередьИндексации[0]; + ОчередьИндексации.Удалить(0); + РагВызов("index_segments", ЗаписатьJSONВСтроку(Пакет), "ПакетИндексацииЗавершение"); +#КонецЕсли +КонецПроцедуры + +// База знаний Vanessa Automation (макет БазаЗнанийДляMCP, формат "Вопрос: ...\n<ответ>") +// индексируется коллекцией vanessa_kb: один документ, сегмент = вопрос + ответ, +// embed_text = вопрос + начало ответа, meta = {type: "qa", num: <номер>}. +&НаКлиенте +Процедура НачатьИндексациюБазыЗнаний() +#Если НЕ ВебКлиент Тогда + СтатусИндексации = "Индексация базы знаний..."; + РагВызов("delete_collection", "{""collection"":""vanessa_kb""}", "УдалениеБазыЗнанийЗавершение"); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура УдалениеБазыЗнанийЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + // результат не важен: коллекции могло не быть + Попытка + Запрос = Новый Структура("Идентификатор, Аргументы", "", Новый Структура("format", "all")); + ТекстБазы = МодульMCPVA.ПолучитьДанныеИзБазыЗнаний(Запрос, Неопределено); + Исключение + Сообщить("База знаний недоступна, коллекция vanessa_kb пропущена: " + ОписаниеОшибки()); + НачатьИндексациюКоллекцийФайлов(); + Возврат; + КонецПопытки; + + Если ТипЗнч(ТекстБазы) <> Тип("Строка") ИЛИ НЕ ЗначениеЗаполнено(ТекстБазы) Тогда + Сообщить("База знаний пуста, коллекция vanessa_kb пропущена."); + НачатьИндексациюКоллекцийФайлов(); + Возврат; + КонецЕсли; + + Сегменты = СегментыБазыЗнаний(ТекстБазы); + Если Сегменты.Количество() = 0 Тогда + Сообщить("В базе знаний не найдено вопросов (маркер ""Вопрос:""), коллекция vanessa_kb пропущена."); + НачатьИндексациюКоллекцийФайлов(); + Возврат; + КонецЕсли; + + Пакет = Новый Структура; + Пакет.Вставить("collection", "vanessa_kb"); + Пакет.Вставить("collection_description", + "База знаний Vanessa Automation: частые вопросы и ответы — как правильно писать шаги и сценарии, типовые проблемы и их решения"); + Пакет.Вставить("doc_id", "vanessa_kb"); + Пакет.Вставить("segments", Сегменты); + + СтатусИндексации = "Индексация базы знаний: " + Формат(Сегменты.Количество(), "ЧН=0; ЧГ=0") + " вопросов..."; + РагВызов("index_segments", ЗаписатьJSONВСтроку(Пакет), "ИндексацияБазыЗнанийЗавершение"); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ИндексацияБазыЗнанийЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Конверт = ПрочитатьJSONИзСтроки(РезультатВызова); + Если ТипЗнч(Конверт) <> Тип("Соответствие") ИЛИ Конверт["ok"] <> Истина Тогда + Сообщить("Ошибка индексации базы знаний: " + ТекстОшибкиКонверта(Конверт, РезультатВызова)); + КонецЕсли; + НачатьИндексациюКоллекцийФайлов(); +#КонецЕсли +КонецПроцедуры + +// Разбор текста базы знаний на сегменты "вопрос + ответ" по маркеру "Вопрос:" +// в начале строки (без учёта регистра — как в MCPVA.ПолучитьМассивВопросов). +&НаКлиенте +Функция СегментыБазыЗнаний(ТекстБазы) + Сегменты = Новый Массив; + ТекущийВопрос = ""; + СтрокиОтвета = Новый Массив; + + Строки = СтрРазделить(ТекстБазы, Символы.ПС, Истина); + Для Каждого Строка Из Строки Цикл + Если СтрНайти(НРег(Строка), "вопрос:") = 1 Тогда + ДобавитьСегментБазыЗнаний(Сегменты, ТекущийВопрос, СтрокиОтвета); + ТекущийВопрос = СокрЛП(Сред(Строка, СтрДлина("Вопрос:") + 1)); + СтрокиОтвета = Новый Массив; + ИначеЕсли ЗначениеЗаполнено(ТекущийВопрос) Тогда + СтрокиОтвета.Добавить(Строка); + КонецЕсли; + КонецЦикла; + ДобавитьСегментБазыЗнаний(Сегменты, ТекущийВопрос, СтрокиОтвета); + + Возврат Сегменты; +КонецФункции + +&НаКлиенте +Процедура ДобавитьСегментБазыЗнаний(Сегменты, Вопрос, СтрокиОтвета) + Если НЕ ЗначениеЗаполнено(Вопрос) Тогда + Возврат; + КонецЕсли; + + Ответ = СокрЛП(СтрСоединить(СтрокиОтвета, Символы.ПС)); + + Сегмент = Новый Структура; + Сегмент.Вставить("text", "Вопрос: " + Вопрос + Символы.ПС + Ответ); + Сегмент.Вставить("embed_text", Вопрос + Символы.ПС + Лев(Ответ, 600)); + Сегмент.Вставить("meta", Новый Структура("type, num", "qa", Сегменты.Количество() + 1)); + Сегменты.Добавить(Сегмент); +КонецПроцедуры + +&НаКлиенте +Процедура ПакетИндексацииЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Конверт = ПрочитатьJSONИзСтроки(РезультатВызова); + Если ТипЗнч(Конверт) <> Тип("Соответствие") ИЛИ Конверт["ok"] <> Истина Тогда + ОчередьИндексации = Новый Массив; + СтатусИндексации = "Ошибка индексации: " + ТекстОшибкиКонверта(Конверт, РезультатВызова); + Возврат; + КонецЕсли; + ОтправитьСледующийПакетИндексации(); +#КонецЕсли +КонецПроцедуры + +// Коллекции файлов из таблицы формы: на каждую валидную строку (Коллекция и +// Каталог заполнены) — delete_collection → скан каталога по маскам расширений → +// index_raw по одному файлу на вызов (последовательная цепочка колбэков). +&НаКлиенте +Процедура НачатьИндексациюКоллекцийФайлов() +#Если НЕ ВебКлиент Тогда + ОчередьКоллекцийФайлов = Новый Массив; + + НомерСтроки = 0; + Для Каждого СтрокаКоллекции Из ТаблицаКоллекций Цикл + НомерСтроки = НомерСтроки + 1; + ИмяКоллекции = СокрЛП(СтрокаКоллекции.Коллекция); + КаталогФайлов = СокрЛП(СтрокаКоллекции.Каталог); + Если НЕ ЗначениеЗаполнено(ИмяКоллекции) И НЕ ЗначениеЗаполнено(КаталогФайлов) + И НЕ ЗначениеЗаполнено(СокрЛП(СтрокаКоллекции.Описание)) + И НЕ ЗначениеЗаполнено(СокрЛП(СтрокаКоллекции.Расширения)) Тогда + Продолжить; // полностью пустая строка — молча пропускаем + КонецЕсли; + Если НЕ ЗначениеЗаполнено(ИмяКоллекции) ИЛИ НЕ ЗначениеЗаполнено(КаталогФайлов) Тогда + Сообщить("Коллекции файлов: строка " + Формат(НомерСтроки, "ЧГ=0") + + " пропущена — заполните «Коллекция» и «Каталог»."); + Продолжить; + КонецЕсли; + + ОписаниеКоллекции = Новый Структура; + ОписаниеКоллекции.Вставить("Имя", ИмяКоллекции); + ОписаниеКоллекции.Вставить("Описание", СокрЛП(СтрокаКоллекции.Описание)); + ОписаниеКоллекции.Вставить("Каталог", КаталогФайлов); + ОписаниеКоллекции.Вставить("Расширения", СокрЛП(СтрокаКоллекции.Расширения)); + ОчередьКоллекцийФайлов.Добавить(ОписаниеКоллекции); + КонецЦикла; + + ОбработатьСледующуюКоллекциюФайлов(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ОбработатьСледующуюКоллекциюФайлов() +#Если НЕ ВебКлиент Тогда + Если ОчередьКоллекцийФайлов.Количество() = 0 Тогда + // все коллекции обработаны — переходим к опросу stats + ТекущаяКоллекцияФайлов = Неопределено; + СтатусИндексации = "Индексация в фоне (эмбеддинг)..."; + ПодключитьОбработчикОжидания("ОпроситьСтатусИндексации", 5, Истина); + Возврат; + КонецЕсли; + + ТекущаяКоллекцияФайлов = ОчередьКоллекцийФайлов[0]; + ОчередьКоллекцийФайлов.Удалить(0); + + СтатусИндексации = "Коллекция """ + ТекущаяКоллекцияФайлов.Имя + """: очистка..."; + РагВызов("delete_collection", + ЗаписатьJSONВСтроку(Новый Структура("collection", ТекущаяКоллекцияФайлов.Имя)), + "УдалениеКоллекцииФайловЗавершение"); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура УдалениеКоллекцииФайловЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + // результат не важен: коллекции могло не быть + СобратьФайлыКоллекции(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура СобратьФайлыКоллекции() +#Если НЕ ВебКлиент Тогда + ФайлыКоллекции = Новый Массив; + УникальныеПути = Новый Соответствие; + Для Каждого Маска Из МаскиИзРасширений(ТекущаяКоллекцияФайлов.Расширения) Цикл + Попытка + НайденныеФайлы = НайтиФайлы(ТекущаяКоллекцияФайлов.Каталог, Маска, Истина); + Исключение + НайденныеФайлы = Новый Массив; + КонецПопытки; + Для Каждого НайденныйФайл Из НайденныеФайлы Цикл + Если НЕ НайденныйФайл.ЭтоФайл() Тогда + Продолжить; + КонецЕсли; + КлючПути = НРег(НайденныйФайл.ПолноеИмя); + Если УникальныеПути[КлючПути] = Неопределено Тогда + УникальныеПути.Вставить(КлючПути, Истина); + ФайлыКоллекции.Добавить(НайденныйФайл.ПолноеИмя); + КонецЕсли; + КонецЦикла; + КонецЦикла; + + Если ФайлыКоллекции.Количество() = 0 Тогда + Сообщить("Коллекция " + ТекущаяКоллекцияФайлов.Имя + ": файлы не найдены в " + + ТекущаяКоллекцияФайлов.Каталог); + ОбработатьСледующуюКоллекциюФайлов(); + Возврат; + КонецЕсли; + + ТекущаяКоллекцияФайлов.Вставить("Файлы", ФайлыКоллекции); + ТекущаяКоллекцияФайлов.Вставить("Индекс", 0); + ОтправитьСледующийФайлКоллекции(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ОтправитьСледующийФайлКоллекции() +#Если НЕ ВебКлиент Тогда + ФайлыКоллекции = ТекущаяКоллекцияФайлов.Файлы; + Если ТекущаяКоллекцияФайлов.Индекс >= ФайлыКоллекции.Количество() Тогда + ОбработатьСледующуюКоллекциюФайлов(); + Возврат; + КонецЕсли; + + ПутьКФайлу = ФайлыКоллекции[ТекущаяКоллекцияФайлов.Индекс]; + ТекущаяКоллекцияФайлов.Индекс = ТекущаяКоллекцияФайлов.Индекс + 1; + ТекущаяКоллекцияФайлов.Вставить("ТекущийФайл", ПутьКФайлу); + + СтатусИндексации = "Коллекция """ + ТекущаяКоллекцияФайлов.Имя + """: файл " + + Формат(ТекущаяКоллекцияФайлов.Индекс, "ЧН=0; ЧГ=0") + " из " + + Формат(ФайлыКоллекции.Количество(), "ЧН=0; ЧГ=0") + "..."; + + ОписаниеФайла = Новый Файл(ПутьКФайлу); + Попытка + РазмерФайла = ОписаниеФайла.Размер(); + Исключение + Сообщить("Коллекция " + ТекущаяКоллекцияФайлов.Имя + ": не удалось прочитать " + + ПутьКФайлу + ": " + ОписаниеОшибки()); + ОтправитьСледующийФайлКоллекции(); + Возврат; + КонецПопытки; + Если РазмерФайла > 5 * 1024 * 1024 Тогда + Сообщить("Коллекция " + ТекущаяКоллекцияФайлов.Имя + ": пропущен " + ПутьКФайлу + + " (больше 5 МБ)"); + ОтправитьСледующийФайлКоллекции(); + Возврат; + КонецЕсли; + + Попытка + ЧтениеФайла = Новый ЧтениеТекста(ПутьКФайлу, КодировкаТекста.UTF8); + ТекстФайла = ЧтениеФайла.Прочитать(); + ЧтениеФайла.Закрыть(); + Исключение + Сообщить("Коллекция " + ТекущаяКоллекцияФайлов.Имя + ": не удалось прочитать " + + ПутьКФайлу + ": " + ОписаниеОшибки()); + ОтправитьСледующийФайлКоллекции(); + Возврат; + КонецПопытки; + Если ТекстФайла = Неопределено Тогда + ТекстФайла = ""; // пустой файл + КонецЕсли; + + РасширениеФайла = ОписаниеФайла.Расширение; + Если Лев(РасширениеФайла, 1) = "." Тогда + РасширениеФайла = Сред(РасширениеФайла, 2); + КонецЕсли; + + Пакет = Новый Структура; + Пакет.Вставить("collection", ТекущаяКоллекцияФайлов.Имя); + // collection_description передаём на каждом вызове: последний непустой побеждает + Пакет.Вставить("collection_description", ТекущаяКоллекцияФайлов.Описание); + Пакет.Вставить("doc_id", ПутьКФайлу); + Пакет.Вставить("name", ОписаниеФайла.Имя); + Пакет.Вставить("text", ТекстФайла); + Пакет.Вставить("meta", Новый Структура("path, ext", ПутьКФайлу, РасширениеФайла)); + РагВызов("index_raw", ЗаписатьJSONВСтроку(Пакет), "ФайлКоллекцииЗавершение"); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ФайлКоллекцииЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Конверт = ПрочитатьJSONИзСтроки(РезультатВызова); + Если ТипЗнч(Конверт) <> Тип("Соответствие") ИЛИ Конверт["ok"] <> Истина Тогда + // ошибка одного файла не прерывает обработку остальных + Сообщить("Коллекция " + ТекущаяКоллекцияФайлов.Имя + ": ошибка индексации " + + ТекущаяКоллекцияФайлов.ТекущийФайл + ": " + + ТекстОшибкиКонверта(Конверт, РезультатВызова)); + КонецЕсли; + ОтправитьСледующийФайлКоллекции(); +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ОпроситьСтатусИндексации() Экспорт +#Если НЕ ВебКлиент Тогда + Если Компонента = Неопределено ИЛИ ТранспортЗапущен <> Истина Тогда + Возврат; + КонецЕсли; + РагВызов("stats", "{}", "ОпросСтатусаИндексацииЗавершение"); +#КонецЕсли +КонецПроцедуры + +// Агрегирует прогресс по ВСЕМ коллекциям из result.collections: суммы embedded +// и n_segments; готово — когда у всех vector_status = "ready"; ошибка — если +// хотя бы у одной "error" (коллекция называется в статусе). +&НаКлиенте +Процедура ОпросСтатусаИндексацииЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Эмбеддинг = 0; + ВсегоСегментов = 0; + КоличествоКоллекций = 0; + ВсеГотовы = Истина; + КоллекцияСОшибкой = ""; + Попытка + Конверт = ПрочитатьJSONИзСтроки(РезультатВызова); + Коллекции = Конверт["result"]["collections"]; + Если ТипЗнч(Коллекции) = Тип("Соответствие") Тогда + Для Каждого КлючИЗначение Из Коллекции Цикл + ДанныеКоллекции = КлючИЗначение.Значение; + КоличествоКоллекций = КоличествоКоллекций + 1; + Эмбеддинг = Эмбеддинг + ДанныеКоллекции["embedded"]; + ВсегоСегментов = ВсегоСегментов + ДанныеКоллекции["n_segments"]; + ВекторныйСтатус = Строка(ДанныеКоллекции["vector_status"]); + Если ВекторныйСтатус = "error" И НЕ ЗначениеЗаполнено(КоллекцияСОшибкой) Тогда + КоллекцияСОшибкой = Строка(КлючИЗначение.Ключ); + КонецЕсли; + Если ВекторныйСтатус <> "ready" Тогда + ВсеГотовы = Ложь; + КонецЕсли; + КонецЦикла; + КонецЕсли; + Исключение + ВсеГотовы = Ложь; + КонецПопытки; + + Если ЗначениеЗаполнено(КоллекцияСОшибкой) Тогда + СтатусИндексации = "Ошибка эмбеддинга в коллекции """ + КоллекцияСОшибкой + + """ (vector_status=error)"; + ИначеЕсли ВсеГотовы И КоличествоКоллекций > 0 Тогда + СтатусИндексации = "Готово: " + Формат(ВсегоСегментов, "ЧН=0; ЧГ=0") + " сегментов в " + + Формат(КоличествоКоллекций, "ЧН=0; ЧГ=0") + " коллекциях"; + ИначеЕсли ТранспортЗапущен = Истина Тогда + СтатусИндексации = "Эмбеддинг: " + Формат(Эмбеддинг, "ЧН=0; ЧГ=0") + " из " + + Формат(ВсегоСегментов, "ЧН=0; ЧГ=0") + " (коллекций: " + + Формат(КоличествоКоллекций, "ЧН=0; ЧГ=0") + ")..."; + ПодключитьОбработчикОжидания("ОпроситьСтатусИндексации", 5, Истина); + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ПолучениеСпискаМоделейЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Конверт = ПрочитатьJSONИзСтроки(РезультатВызова); + Если ТипЗнч(Конверт) <> Тип("Соответствие") ИЛИ Конверт["ok"] <> Истина Тогда + Возврат; // lite-компонента или метод недоступен — остаётся список по умолчанию + КонецЕсли; + + Модели = Новый Массив; + Попытка + СписокИзЯдра = Конверт["result"]["models"]; + Если ТипЗнч(СписокИзЯдра) = Тип("Массив") Тогда + Для Каждого ЭлементСписка Из СписокИзЯдра Цикл + Если ТипЗнч(ЭлементСписка) = Тип("Строка") Тогда + Модели.Добавить(ЭлементСписка); + ИначеЕсли ТипЗнч(ЭлементСписка) = Тип("Соответствие") Тогда + Модели.Добавить(Строка(ЭлементСписка["name"])); + КонецЕсли; + КонецЦикла; + КонецЕсли; + Исключение + КонецПопытки; + + Если Модели.Количество() > 0 Тогда + ЗаполнитьСписокВыбораМоделей(Модели); + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура РагВызов(Метод, ДанныеJson, ИмяОбработчика) +#Если НЕ ВебКлиент Тогда + Если Компонента = Неопределено Тогда + Возврат; + КонецЕсли; + Попытка + Компонента.НачатьВызовRagВыполнить( + Новый ОписаниеОповещения(ИмяОбработчика, ЭтотОбъект), Метод, ДанныеJson); + Исключение + СтатусИндексации = "Ошибка вызова RagВыполнить(" + Метод + "): " + ОписаниеОшибки(); + КонецПопытки; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Функция СтрокаУдаленияКоллекции() + Возврат "{""collection"":""vanessa_steps""}"; +КонецФункции + +// Маски НайтиФайлы из фильтра расширений: "feature" → "*.feature"; +// "feature;md;txt" → три маски; ".md" и "*.md" тоже допустимы. Пусто — "*.feature". +&НаКлиенте +Функция МаскиИзРасширений(СтрокаРасширений) + Маски = Новый Массив; + Для Каждого ЧастьСтроки Из СтрРазделить(СтрокаРасширений, ";,", Ложь) Цикл + РасширениеФайла = СокрЛП(ЧастьСтроки); + Если НЕ ЗначениеЗаполнено(РасширениеФайла) Тогда + Продолжить; + КонецЕсли; + Если Лев(РасширениеФайла, 1) = "*" Тогда + Маски.Добавить(РасширениеФайла); + ИначеЕсли Лев(РасширениеФайла, 1) = "." Тогда + Маски.Добавить("*" + РасширениеФайла); + Иначе + Маски.Добавить("*." + РасширениеФайла); + КонецЕсли; + КонецЦикла; + Если Маски.Количество() = 0 Тогда + Маски.Добавить("*.feature"); + КонецЕсли; + Возврат Маски; +КонецФункции + +&НаКлиенте +Функция ТекстОшибкиКонверта(Конверт, РезультатВызова) + Текст = ""; + Попытка + Текст = Строка(Конверт["error"]["code"]) + ": " + Строка(Конверт["error"]["message"]); + Исключение + КонецПопытки; + Если НЕ ЗначениеЗаполнено(Текст) Тогда + Текст = Лев(Строка(РезультатВызова), 300); + КонецЕсли; + Возврат Текст; +КонецФункции + +#КонецОбласти + +#Область СтатусИОформление + +&НаКлиенте +Процедура ПолучениеСтатусаЗавершение(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт +#Если НЕ ВебКлиент Тогда + Статус = ПрочитатьJSONИзСтроки(РезультатВызова); + Если ТипЗнч(Статус) <> Тип("Соответствие") Тогда + Возврат; + КонецЕсли; + Если Статус["running"] = Истина Тогда + СтатусСервера = "Запущен: http://localhost:" + Формат(Статус["port"], "ЧГ=0") + "/mcp" + + ", инструментов: " + Формат(Статус["tools_registered"], "ЧН=0; ЧГ=0") + + ", запросов в работе: " + Формат(Статус["pending_requests"], "ЧН=0; ЧГ=0"); + Иначе + СтатусСервера = "Остановлен"; + КонецЕсли; +#КонецЕсли +КонецПроцедуры + +&НаКлиенте +Процедура ОбновитьДоступностьЭлементов() + РежимRag = (РежимРаботы = "MCP+RAG"); + НастройкиДоступны = (ТранспортЗапущен <> Истина); // пока сервер работает, настройки не меняем + Элементы.ГруппаНастройки.Доступность = НастройкиДоступны; + Элементы.ГруппаRAG.Доступность = РежимRag И НастройкиДоступны; + Элементы.Переиндексировать.Видимость = РежимRag; +КонецПроцедуры + +&НаКлиенте +Процедура ОбновитьСсылкуНаМодель() + СсылкаНаМодельURL = ""; + ИмяМодели = СокрЛП(Модель); + Описание = Неопределено; + Если СтрНайти(ИмяМодели, "\") = 0 И СтрНайти(ИмяМодели, "/") = 0 И СтрНайти(ИмяМодели, ":") = 0 Тогда + Описание = ОписанияМоделей()[ИмяМодели]; + КонецЕсли; + Если Описание = Неопределено Тогда + // путь к офлайн-модели или неизвестное имя — страницы для скачивания нет + Элементы.СсылкаНаМодель.Видимость = Ложь; + Возврат; + КонецЕсли; + СсылкаНаМодельURL = Описание.URL; + Элементы.СсылкаНаМодель.Заголовок = "Страница модели: " + Описание.Репозиторий + + " (" + Описание.Размер + ")"; + Элементы.СсылкаНаМодель.Видимость = Истина; +КонецПроцедуры + +// Репозитории Hugging Face, из которых fastembed 5.16.0 скачивает модели. +&НаКлиенте +Функция ОписанияМоделей() + Модели = Новый Соответствие; + Модели.Вставить("multilingual-e5-small", НовоеОписаниеМодели("intfloat/multilingual-e5-small", "~0.5 ГБ")); + Модели.Вставить("multilingual-e5-base", НовоеОписаниеМодели("intfloat/multilingual-e5-base", "~1.1 ГБ")); + Модели.Вставить("multilingual-e5-large", НовоеОписаниеМодели("Qdrant/multilingual-e5-large-onnx", "~2.2 ГБ")); + Возврат Модели; +КонецФункции + +&НаКлиенте +Функция НовоеОписаниеМодели(Репозиторий, Размер) + Возврат Новый Структура("URL, Репозиторий, Размер", + "https://huggingface.co/" + Репозиторий, Репозиторий, Размер); +КонецФункции + +&НаКлиенте +Процедура ЗаполнитьСписокВыбораМоделей(Модели) + Элементы.Модель.СписокВыбора.Очистить(); + Для Каждого ИмяМодели Из Модели Цикл + Элементы.Модель.СписокВыбора.Добавить(ИмяМодели); + КонецЦикла; + ОбновитьСсылкуНаМодель(); +КонецПроцедуры + +&НаКлиенте +Функция СписокМоделейПоУмолчанию() + Модели = Новый Массив; + Модели.Добавить("multilingual-e5-small"); + Модели.Добавить("multilingual-e5-base"); + Модели.Добавить("multilingual-e5-large"); + Возврат Модели; +КонецФункции + +#КонецОбласти + +#Область Помощники + +&НаКлиенте +Процедура ИнициализироватьСостояние() + Если ОжидающиеЗапросы = Неопределено Тогда + ОжидающиеЗапросы = Новый Соответствие; + КартаОбработчиков = Новый Соответствие; + СписокИнструментовСбор = Новый Массив; + ОчередьИндексации = Новый Массив; + ОчередьКоллекцийФайлов = Новый Массив; + ТекущаяКоллекцияФайлов = Неопределено; + ТранспортЗапущен = Ложь; + РежимДиспетчеризации = Ложь; + РежимСбораИнструментов = Ложь; + RagНастроен = Ложь; + КонецЕсли; +КонецПроцедуры + +&НаКлиенте +Функция ЗаписатьJSONВСтроку(Значение) + Запись = Новый ЗаписьJSON; + Запись.УстановитьСтроку(); + ЗаписатьJSON(Запись, Значение); + Возврат Запись.Закрыть(); +КонецФункции + +&НаКлиенте +Функция ПрочитатьJSONИзСтроки(ИсходнаяСтрока) + Если НЕ ЗначениеЗаполнено(ИсходнаяСтрока) Тогда + Возврат Неопределено; + КонецЕсли; + Попытка + Чтение = Новый ЧтениеJSON; + Чтение.УстановитьСтроку(ИсходнаяСтрока); + Возврат ПрочитатьJSON(Чтение, Истина); + Исключение + Возврат Неопределено; + КонецПопытки; +КонецФункции + +&НаКлиенте +Функция ПустоеОповещение() + Возврат Новый ОписаниеОповещения("ПустойОбработчикКолбэка", ЭтотОбъект); +КонецФункции + +&НаКлиенте +Процедура ПустойОбработчикКолбэка(РезультатВызова, ПараметрыВызова, ДополнительныеПараметры) Экспорт + // fire-and-forget для НачатьВызов* компоненты +КонецПроцедуры + +#КонецОбласти diff --git "a/vanessa-plugin/MCPRagSearch/MCPRagSearch/Templates/\320\241\320\272\320\270\320\273\320\237\320\276\320\270\321\201\320\272\320\260.xml" "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Templates/\320\241\320\272\320\270\320\273\320\237\320\276\320\270\321\201\320\272\320\260.xml" new file mode 100644 index 0000000..7b69b15 --- /dev/null +++ "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Templates/\320\241\320\272\320\270\320\273\320\237\320\276\320\270\321\201\320\272\320\260.xml" @@ -0,0 +1,20 @@ + + + + \ No newline at end of file diff --git "a/vanessa-plugin/MCPRagSearch/MCPRagSearch/Templates/\320\241\320\272\320\270\320\273\320\237\320\276\320\270\321\201\320\272\320\260/Ext/Template.txt" "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Templates/\320\241\320\272\320\270\320\273\320\237\320\276\320\270\321\201\320\272\320\260/Ext/Template.txt" new file mode 100644 index 0000000..94a2dfd --- /dev/null +++ "b/vanessa-plugin/MCPRagSearch/MCPRagSearch/Templates/\320\241\320\272\320\270\320\273\320\237\320\276\320\270\321\201\320\272\320\260/Ext/Template.txt" @@ -0,0 +1,37 @@ +# Поиск шагов, примеров и правил Vanessa Automation (RAG) + +Этот MCP-сервер — Vanessa Automation (BDD-тестирование 1С) с семантическим поиском. + +## Коллекции (актуальный список — инструмент list_collections) +- vanessa_steps — все шаги Ванессы. Текст хита = готовый текст шага; meta.type = категория, meta.lang = язык. +- vanessa_kb — база знаний: вопросы-ответы «как правильно писать сценарии» с Gherkin-примерами; meta.num = номер вопроса. +- прочие — файловые коллекции проекта (например, фича-файлы): doc_id = путь к файлу, line_start/line_end = строки файла. + +## Как подобрать шаг +1. Сформулируй действие по-русски своими словами и вызови search: + {"query":"нажать на кнопку формы","collection":"vanessa_steps","mode":"dense","k":10} + - dense — поиск по смыслу (падежи, синонимы, языки не мешают); хорошие скоры 0.75–0.92. + - hybrid/keyword — для точных подстрок и идентификаторов; скоры hybrid — это RRF (~0.03), min_score не задавай. +2. Бери ТОЧНЫЙ текст шага из поля text хита; значения параметров подставляй в той же форме кавычек ("..." или '...'). +3. Описание, сниппет и параметры шага: инструмент search_for_steps_by_keywords с search_name = точный текст шага. +4. Фильтр по категории: "filter":{"all":{"type":"UI.Формы.Кнопки"}} (категория видна в meta.type хита). + +## Примеры использования шагов в реальных сценариях +- grep: {"pattern":"нажимаю на кнопку","collection":"<файловая>","ignore_case":true,"context_lines":2} — найдёт строки фича-файлов. +- get_segment: {"doc_id":"<путь из хита>","line_start":N,"line_end":M} — вырезает кусок сценария целиком. + +## Правила и типовые проблемы +Сначала спроси vanessa_kb (mode dense, вопрос своими словами): проведение документов, ожидание окон, +очистка тестовых данных, переключатели/гиперссылки, проверка отчётов — там готовые ответы с примерами. + +## Workflow написания сценария +1. list_collections — что доступно и готово (vector_status=ready). +2. Составь план действий; на каждое действие — search по vanessa_steps (dense). +3. Сомнения — vanessa_kb; примеры — grep/get_segment по файловым коллекциям. +4. Проверь синтаксис: check_syntax. Выполняй пошагово execute_feature_step, целиком — run_scenario. +5. Состояние и результаты: get_VanessaAutomation_state, get_test_results. + +## Частые ошибки +- Не подбирай шаги точной подстрокой — русская морфология ломает такой поиск; используй dense. +- Не выдумывай шаги: если лучший dense-скор < ~0.75 — переформулируй запрос или спроси vanessa_kb. +- Не меняй формулировку найденного шага (кроме первого слова И/Когда/Тогда и значений параметров). diff --git a/vanessa-plugin/MCPRagSearch/MCPRagSearch/filename b/vanessa-plugin/MCPRagSearch/MCPRagSearch/filename new file mode 100644 index 0000000..7c0a46e --- /dev/null +++ b/vanessa-plugin/MCPRagSearch/MCPRagSearch/filename @@ -0,0 +1 @@ +MCPRagSearch.epf \ No newline at end of file diff --git a/vanessa-plugin/QUICKSTART.md b/vanessa-plugin/QUICKSTART.md new file mode 100644 index 0000000..322207b --- /dev/null +++ b/vanessa-plugin/QUICKSTART.md @@ -0,0 +1,71 @@ +# MCPRagSearch — быстрый старт + +## Зачем это сделано + +Штатный MCP Ванессы живёт в расширении `client_mcp`, а поиск шагов там — подстрочный: +«кликаю по кнопке» не найдёт «нажимаю на кнопку», морфология, синонимы и смена языка — +мимо. Плагин **MCPRagSearch** поднимает MCP-сервер целиком на внешней компоненте +**http1c** (C++ транспорт + Rust-ядро поиска) и добавляет **семантический (RAG) поиск**: +ИИ-агент находит шаги по смыслу, задаёт вопросы базе знаний VA и ищет по фича-файлам +проекта. Ядро Ванессы при этом не правится (микро-хук уже в ядре — PR +[Pr-Mex/vanessa-automation#2614](https://github.com/Pr-Mex/vanessa-automation/pull/2614)), +расширение `client_mcp` не требуется. + +## Что умеет + +- **Все 36 MCP-инструментов VA** (запуск сценариев, анализ форм, шаги, база знаний…) — + регистрируются из MCPVA автоматически, обслуживаются компонентой. +- **Семантический поиск** (режим `MCP + RAG`), без round-trip в 1С: + `search` (по смыслу: dense / hybrid / keyword), `grep` (регулярные выражения), + `get_segment` (чтение фрагмента файла по строкам), `list_collections` + (какие коллекции проиндексированы). +- **Коллекции из коробки**: `vanessa_steps` (все шаги VA, эмбеддинг обогащён синонимами + действий) и `vanessa_kb` (база знаний «как правильно писать сценарии», 98 Q&A); + плюс **свои файловые коллекции** — таблица «Коллекции файлов» (каталог + расширения + + описание), например фичи проекта. +- **Скил для агента**: MCP-промпт `search-guide` и инструмент-дубль `get_search_guide` — + готовая инструкция агенту, как и где искать. +- **Автоактуальность**: смена языка генератора Gherkin или библиотек шагов → + автоматическая переиндексация (проверка раз в 30 секунд). + +## Требования + +Windows x86-64, платформа 1С 8.3.19+, Vanessa Automation с микро-хуком MCPVA +(ветка `mcp-rag` / PR #2614). + +## Запуск + +1. **Плагин**: собранный `MCPRagSearch.epf` положить в `plugins/` рядом с + `vanessa-automation.epf` — или просто открыть как внешнюю обработку в том же сеансе, + где открыта VA. +2. **Компонента**: на форме плагина нажать гиперссылку **«Скачать компоненту (последний + релиз)»** (`http1c-addin-full.zip`), затем **«Установить из архива…»** — плагин + распакует бандл, проверит состав и сам подставит каталог в настройки. +3. **Режим работы**: + - `MCP` — только инструменты VA (быстрый старт, ничего не скачивается); + - `MCP + RAG` — плюс семантический поиск: выбрать модель (по умолчанию + `multilingual-e5-small`, рядом ссылка на её страницу), каталог для скачивания + моделей и устройство (`auto` = GPU/DirectML с откатом на CPU). +4. Нажать **«Запустить»**. Первый старт в режиме RAG скачает модель и проиндексирует + шаги — подготовка может занять время, прогресс и счётчики видны на форме. + Кнопка «Остановить» (или закрытие формы) останавливает сервер. +5. **Подключить MCP-клиента** (Claude Code / VS Code / Cursor…): + + ```json + { + "mcpServers": { + "vanessa": { "type": "http", "url": "http://localhost:8765/mcp" } + } + } + ``` + +## Подводные камни + +- **Лайт-бандл** (без `rcore.dll`): поисковые инструменты отвечают `rag_not_installed` — + доустановите полный бандл в тот же каталог, компонента подхватит его без перезапуска 1С. +- **Клиент не видит новые инструменты/промпт**: MCP-клиенты кэшируют список при + подключении — переподключите сервер в клиенте после его запуска/перезапуска. +- **Закрытие формы плагина = остановка сервера** — форму можно свернуть, но не закрывать. +- **Порт занят**: поменяйте «Порт» на форме (по умолчанию 8765) и URL у клиента. + +Подробное описание всех настроек, коллекций и протокола — в [README.md](README.md). diff --git a/vanessa-plugin/README.md b/vanessa-plugin/README.md new file mode 100644 index 0000000..72dfca9 --- /dev/null +++ b/vanessa-plugin/README.md @@ -0,0 +1,253 @@ +# MCPRagSearch — плагин Vanessa Automation + +Внешняя обработка (плагин VA типа «Утилита», или просто внешняя обработка, +открытая в том же сеансе), которая поднимает MCP сервер на внешней компоненте +**http1c** (`libhttp1cWin.dll`): компонента обслуживает **весь MCP-контракт +Ванессы**. Опционально — семантический поиск на Rust-ядре (`rcore.dll`): по +шагам VA (коллекция `vanessa_steps`), по базе знаний VA (коллекция +`vanessa_kb`) и по произвольным файлам из настраиваемых коллекций (таблица +«Коллекции файлов»). + +Инструменты MCP не дублируются: плагин вызывает у MCPVA регистрацию всех +инструментов (`ВызватьРегистрациюИнструментовMCP`) через свой фасад, передаёт их +определения в компоненту (`tools_json`), а входящие `ToolCall` диспетчеризует +обратно в обработчики MCPVA. Ответ уходит конвертом +`{"id": ..., "body": "<сериализованный результат MCP>"}` (body — JSON-строка, +двойная сериализация — протокол http1c). + +Нужен микро-хук в MCPVA (уже в форке VA): `УстановитьВнешнийФасадМсп` + +`ВызватьРегистрациюИнструментовMCP`. Там же в форке исправлена схема +инструмента `get_data_from_knowledge_base` (`search_string`: number → string — +типизированные клиенты роняли аргумент). + +Итого инструментов: 36 инструментов VA + 4 нативных (`search`, `grep`, +`get_segment`, `list_collections` — обслуживаются компонентой без round-trip +в 1С) + `get_search_guide` = **41**. + +## Сравнение со штатным поиском (зачем вообще RAG) + +Перед внедрением был снят бейзлайн на живом MCP Ванессы: одни и те же запросы +задавались штатному `search_for_steps_by_keywords` (подстрочный поиск) и +нативному `search` (hybrid, k=5) по коллекции `vanessa_steps`: + +| Запрос | Штатный (подстрока) | RAG (hybrid) | +|---|---|---| +| «нажимаю кнопку» | 16 шагов, **все мимо** («кнопку очистить», «кнопку выбора»), главных нет | ✅ топ-5 = главные шаги: «я нажимаю на кнопку "…"», с именем, в таблице | +| «нажать кнопку» (инфинитив) | **0** | ✅ топ-5 релевантный | +| «click button» (англ. запрос к ru-корпусу) | **0** | ✅ кросс-языковой поиск работает | +| «кнопкa» (опечатка: латинская «a») | **0** | ✅ dense — точный топ (score ≈ 0.88) | +| «заполняю поле и нажимаю кнопку» (составной) | **0**, без намёка, что нужно два шага | ✅ выдача даёт материал по обоим действиям | +| «проверить всплывающее уведомление» (по смыслу) | **0** | ✅ нашло «Если появилось предупреждение…» — попадание при нуле общих слов | +| «выбрать значение из выпадающего списка» | работал (по слову «выпадающ») | ✅ топ чище и ранжирован | +| «кликаю по кнопке» (синоним) | уводил в экзотику (UI Automation) | ✅ закрыто обогащением `embed_text` синонимами действий: dense находит «нажимаю на кнопку» (score ≈ 0.88) | + +**Почему подстрока проваливается.** Русская морфология: «кнопка» ≠ «кнопку», и +все шаги вида «нажимаю на кнопку…» по запросу «кнопка» просто не находятся +(5 результатов из 36+ реальных). Порядок слов и предлоги: «нажимаю кнопку» без +«на» давал выдачу, которая *выглядит* успешной, но не содержит ни одного нужного +шага. Ранжирования нет: порядок выдачи не связан с релевантностью, при лимите +обрезка произвольная. Синонимы, инфинитивы, другой язык и опечатки — гарантированный +ноль, потому что сравниваются буквальные байты строки. + +**Следствия для ИИ-агента.** Два режима отказа. Нули — агент начинает вручную +подбирать стемы и перечислять падежи через `|` (лишние итерации, лишние токены, +об этом приёме надо ещё догадаться). И опаснее — «успешная» нерелевантная выдача: +агент молча берёт неправильный шаг, и ошибка всплывает только при прогоне +сценария. Семантический поиск закрывает оба: эмбеддинги сводят запрос и шаг по +смыслу независимо от формы слова и языка, а скор даёт честное ранжирование — +6 из 7 провальных кейсов закрылись одним механизмом, седьмой («кликаю») — +обогащением эмбеддингов синонимами при индексации (текст шага в выдаче не +меняется). + +**Что ещё дала сверка.** Найденное по её итогам закрыто в этом же релизе: +сегментная мета в хитах (раньше `meta` шагов приходила пустой — фильтры по +`meta.type`/`meta.lang` не работали), база знаний как отдельная коллекция +`vanessa_kb` (поиск примеров через штатный инструмент был полностью сломан +багом схемы `search_string` number → string — исправлен в форке VA), описания +коллекций в `list_collections` (агенту видно, где что искать), и пояснение про +скоры: у hybrid RRF-скоры ≈ 0.03 — это норма, для абсолютных порогов +`min_score` используйте режим dense. + +## Структура исходников + +``` +vanessa-plugin/MCPRagSearch/ +├── MCPRagSearch.xml ← корневой дескриптор внешней обработки +└── MCPRagSearch/ + ├── filename ← имя целевого .epf (как у плагинов VA) + ├── Ext/ObjectModule.bsl ← ОписаниеПлагина / Инициализация + ├── Forms/ + │ ├── Форма.xml ← метаданные формы + │ └── Форма/Ext/ + │ ├── Form.xml ← макет формы (панель управления) + │ └── Form/Module.bsl ← вся логика (клиент) + └── Templates/ + ├── СкилПоиска.xml ← метаданные макета + └── СкилПоиска/Ext/Template.txt ← текст скила (промпт search-guide / инструмент get_search_guide) +``` + +## Сборка .epf + +Любым стандартным способом сборки из XML-выгрузки, например конфигуратором: + +``` +1cv8.exe DESIGNER /F <путь_к_базе> /N admin /P admin + /LoadExternalDataProcessorOrReportFromFiles "vanessa-plugin\MCPRagSearch\MCPRagSearch.xml" "MCPRagSearch.epf" + /DisableStartupDialogs /Out build.log +``` + +или `vrunner compileepf` / ibcmd — формат выгрузки тот же, что у плагинов VA +(version 2.10, как `plugins/УтвержденияBDD`). + +## Установка + +1. Собранный `MCPRagSearch.epf` положить в каталог `plugins/` рядом с + `vanessa-automation.epf` (VA подхватывает плагины из этого каталога), либо + открыть как обычную внешнюю обработку в той же сессии, где открыта VA. +2. Компонента — рекомендуемый путь, прямо с формы плагина: + 1. в группе «Настройки сервера» нажать гиперссылку + «Скачать компоненту (последний релиз)» — браузер скачает + `http1c-addin-full.zip` (`libhttp1cWin.dll` + `rcore.dll` + `DirectML.dll`) + из последнего релиза GitHub (релизный workflow публикует архивы и под + стабильными неверсионированными именами, поэтому ссылка + `releases/latest/download/http1c-addin-full.zip` всегда актуальна); если + прямая ссылка не работает — страница релизов: + ; + 2. нажать кнопку «Установить из архива…», выбрать скачанный zip и каталог, + куда распаковать — плагин распакует бандл, проверит состав + (`libhttp1cWin.dll` обязателен; без `rcore.dll` это lite-бандл, RAG будет + недоступен) и сам подставит каталог в поле «Каталог компоненты»; + 3. нажать «Запустить». + + Вручную: в любой каталог положить `libhttp1cWin.dll` (транспорт), а для + режима RAG ещё `rcore.dll` и `DirectML.dll` (полный пакет http1c). + Этот каталог указывается в поле «Каталог компоненты». + +Требования: Windows x86-64, платформа 8.3.19+ (используются +`НачатьПодключениеВнешнейКомпоненты`, `Выполнить`/`Вычислить` на клиенте). + +## Панель управления + +| Элемент | Назначение | +|---|---| +| Режим работы (тумблер) | `MCP` — только сервер инструментов; `MCP+RAG` — плюс индексация шагов и семантический поиск | +| Запустить | Подключает компоненту, собирает инструменты из MCPVA, применяет настройки, запускает прослушивание; в режиме RAG — конфигурирует модель и индексирует шаги | +| Остановить | Останавливает прослушивание, снимает внешний фасад МСП | +| Переиндексировать | (только RAG) Пересоздаёт коллекции `vanessa_steps`, `vanessa_kb` и все коллекции файлов, индексирует всё заново; если RAG включили на работающем сервере — сначала выполнит configure | +| Обновить статус | Опрашивает компоненту (`ПолучитьСтатус`) и, в режиме RAG, прогресс эмбеддинга (`stats`) | +| Скачать компоненту (последний релиз) | Гиперссылка: качает `http1c-addin-full.zip` из последнего релиза GitHub | +| Установить из архива… | Распаковывает скачанный zip в выбранный каталог, проверяет состав бандла и подставляет каталог в «Каталог компоненты» (доступна, пока сервер остановлен) | + +Если форма открыта не как плагин, при «Запустить» плагин ищет открытое окно +Vanessa Automation сам; если не нашёл — сообщение «Откройте Vanessa Automation». + +## Параметры (автосохраняются в настройках формы) + +| Параметр | По умолчанию | Описание | +|---|---|---| +| Порт | 8765 | MCP endpoint: `http://localhost:<порт>/mcp` | +| Каталог компоненты | — | Каталог с `libhttp1cWin.dll` (+ `rcore.dll`, `DirectML.dll` для RAG) | +| Токен авторизации | пусто | Если задан — Bearer-токен (`auth_token`) | +| Вести лог / Файл лога | выкл | Логирование компоненты (`logging_enabled` / `log_path`) | +| Модель | multilingual-e5-small | Имя встроенной модели (список подтягивается из `list_models`; рядом — гиперссылка на страницу модели на Hugging Face с размером скачивания) или путь к локальной модели | +| Каталог моделей | пусто | `cache_dir` — куда скачивать модели; пусто = каталог по умолчанию | +| Устройство | auto | `auto` (DirectML с откатом на CPU) / `dml` / `cpu` | +| Коллекции файлов | пусто | Таблица дополнительных коллекций для индексации файлов (см. ниже) | + +Статусные поля (не сохраняются): статус сервера, статус индексации, число +зарегистрированных инструментов. + +## Коллекции + +В режиме `MCP+RAG` плагин индексирует три типа коллекций (порядок индексации: +шаги → база знаний → коллекции файлов): + +| Коллекция | Содержимое | +|---|---| +| `vanessa_steps` | Все шаги VA. Текст хита = готовый текст шага; в `meta` хита: `meta.type` = категория шага, `meta.lang` = язык генератора Gherkin. `embed_text` обогащается синонимами действий («кликаю»/«click» семантически ложатся на «нажимаю» и т.п.) — текст шага в выдаче не меняется, обогащается только эмбеддинг | +| `vanessa_kb` | База знаний VA (макет `БазаЗнанийДляMCP`, 98 вопросов-ответов «как правильно писать сценарии»): сегмент = вопрос + ответ, `meta.type` = `qa`, `meta.num` = номер вопроса | +| файловые | Произвольные текстовые файлы из таблицы «Коллекции файлов» (см. ниже) | + +### Вотчер актуальности + +Раз в 30 секунд плагин сверяет язык генератора Gherkin и размер библиотеки +шагов с состоянием на момент индексации; при расхождении (сменили язык, +подключили/перечитали библиотеки шагов) автоматически запускает +переиндексацию. Вотчер работает, пока сервер запущен в режиме `MCP+RAG`. + +## Коллекции файлов + +Таблица «Коллекции файлов» в группе «Семантический поиск (RAG)» позволяет, кроме +шагов VA, проиндексировать произвольные текстовые файлы — например, фича-файлы +проекта. Каждая строка таблицы — отдельная коллекция в `rcore`; индексация +выполняется при запуске в режиме `MCP+RAG` и по кнопке «Переиндексировать» +(после шагов, до опроса статуса). + +Колонки: + +| Колонка | Назначение | +|---|---| +| Коллекция | Имя коллекции (желательно латиница/цифры/подчёркивание, например `project_features`) — под этим именем коллекция видна MCP-инструментам | +| Описание | Описание коллекции для ИИ-агента — уходит в `collection_description` и попадает в описание инструментов поиска | +| Каталог | Папка с файлами; сканируется рекурсивно (включая подпапки) | +| Расширения | Фильтр расширений: `feature`, `feature;md;txt`, допускается и форма `*.feature`; пусто = `feature` | + +Каждый файл индексируется целиком одним вызовом `index_raw` (`doc_id` — полный +путь, `name` — имя файла, `meta.path`/`meta.ext`): ядро само режет текст на +чанки по строкам, поэтому для коллекции работают все поисковые инструменты — +`search` (семантический поиск), `grep` (регулярные выражения) и `get_segment` +(чтение фрагмента файла по строкам, с контекстом до/после). Файлы читаются в +UTF-8; файлы больше 5 МБ пропускаются с сообщением. Строки таблицы без имени +коллекции или каталога пропускаются, перед заливкой коллекция пересоздаётся +(`delete_collection`). + +Пример: коллекция `irp_features`, описание «Фича-файлы проекта ИРП (Gherkin, +русские шаги)», каталог `D:\Projects\IRP\features`, расширения `feature` — после +индексации ИИ-агент через MCP ищет по содержимому всех фич проекта семантически +(`search`), по регулярке (`grep`) и читает найденные места из файлов +(`get_segment`). + +## Промпты и скил поиска + +Кроме инструментов, сервер публикует один MCP-промпт (`prompts_json` +компоненты): + +| Промпт | Назначение | +|---|---| +| `search-guide` (title «Vanessa: как искать шаги (RAG)») | Инструкция для ИИ-агента: как искать шаги, примеры и правила Vanessa Automation через семантический поиск (`list_collections`, `search`, `grep`, `get_segment`) | + +Текст хранится в текстовом макете `СкилПоиска` (правится в конфигураторе) и +описывает весь рабочий процесс: подбор шагов по коллекции `vanessa_steps` +(режим dense), вопросы к базе знаний `vanessa_kb`, поиск примеров по файловым +коллекциям (`grep` / `get_segment`) и частые ошибки. Аргументов у промпта нет; +запрос `prompts/get` обслуживается событием `PromptGet` (ответ — +`{"messages":[{"role":"user","content":{"type":"text","text":"..."}}]}` +тем же конвертом `{id, body}`; для совместимости принимается и старое имя +`search_guide`). + +Тот же текст отдаёт инструмент-дубль **`get_search_guide`**: не все клиенты +показывают MCP-промпты (известный случай — VS Code через `mcp-remote`), а +инструменты агент видит всегда — поэтому скил доступен обоими путями. + +MCP-клиенты показывают промпты как slash-команды: например, в Claude Code это +`/mcp__vanessa__search-guide` (префикс зависит от имени сервера в конфигурации +клиента) — вызов вставляет текст инструкции в контекст диалога перед +написанием сценария. + +## Подключение MCP-клиента + +```json +{ + "mcpServers": { + "vanessa": { + "type": "http", + "url": "http://localhost:8765/mcp" + } + } +} +``` + +В режиме RAG поисковые инструменты (`search` / `grep` / `get_segment` / +`list_collections`) обслуживаются самой компонентой без round-trip в 1С — по +коллекциям `vanessa_steps`, `vanessa_kb` и по всем коллекциям файлов из +таблицы.