From 50e19cdb18ea77438915ce280b9715cf45e29b96 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 May 2026 21:57:55 +0900 Subject: [PATCH 1/7] Position LocalCrab as local ontology factory --- Makefile | 11 +- README.md | 622 ++++++------------------ apps/api/Dockerfile | 17 - apps/api/main.py | 2 +- apps/web/Dockerfile | 35 -- apps/web/lib/api.ts | 2 +- docker-compose.yml | 158 ------ docs/localcrab-factory-workflow.md | 257 ++++++++++ docs/localcrab-opencrab-relationship.md | 115 +++++ docs/opencrab-pack-v1.md | 381 +++++++++++++++ opencrab/cli.py | 85 ++-- opencrab/config.py | 20 +- opencrab/mcp/tools.py | 2 +- opencrab/ontology/bm25.py | 71 ++- opencrab/ontology/query.py | 144 +++++- opencrab/ontology/reranker.py | 113 ++++- opencrab/pack/__init__.py | 5 + opencrab/pack/neo4j_export.py | 163 +++++++ opencrab/stores/chroma_store.py | 10 +- opencrab/stores/factory.py | 42 +- opencrab/stores/local_doc_store.py | 2 +- opencrab/stores/local_graph_store.py | 22 +- opencrab/stores/neo4j_store.py | 24 +- railway.toml | 9 - tests/test_pack_neo4j_export.py | 65 +++ tests/test_query_retrieval.py | 127 +++++ 26 files changed, 1707 insertions(+), 797 deletions(-) delete mode 100644 apps/api/Dockerfile delete mode 100644 apps/web/Dockerfile delete mode 100644 docker-compose.yml create mode 100644 docs/localcrab-factory-workflow.md create mode 100644 docs/localcrab-opencrab-relationship.md create mode 100644 docs/opencrab-pack-v1.md create mode 100644 opencrab/pack/__init__.py create mode 100644 opencrab/pack/neo4j_export.py delete mode 100644 railway.toml create mode 100644 tests/test_pack_neo4j_export.py create mode 100644 tests/test_query_retrieval.py diff --git a/Makefile b/Makefile index 8843fa3..c8100e0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install dev-install up down status serve query manifest lint format test coverage seed +.PHONY: help install dev-install status serve query manifest lint format test coverage seed PYTHON := python PIP := pip @@ -10,8 +10,6 @@ help: @echo "Usage:" @echo " make install Install package" @echo " make dev-install Install with dev extras" - @echo " make up Start all Docker services" - @echo " make down Stop all Docker services" @echo " make status Check store connections" @echo " make serve Start MCP server on stdio" @echo " make manifest Print MetaOntology grammar" @@ -27,13 +25,6 @@ install: dev-install: $(PIP) install -e ".[dev]" -up: - docker-compose up -d - @echo "Services starting... run 'make status' to check readiness." - -down: - docker-compose down - status: $(PYTHON) -m opencrab.cli status diff --git a/README.md b/README.md index 48e4972..bd579ba 100644 --- a/README.md +++ b/README.md @@ -4,532 +4,234 @@ # OpenCrab -**MetaOntology OS MCP Server Plugin** - -> Carcinization is the evolutionary tendency for crustaceans to converge on a crab-like body plan. -> OpenCrab applies the same principle to agent environments: -> all sufficiently advanced AI systems eventually evolve toward ontology-structured forms. - -OpenCrab is an MCP (Model Context Protocol) server that exposes the MetaOntology OS grammar -to any OpenClaw-compatible agent environment — Claude Code, n8n, LangGraph, and beyond. - -**Companion:** [`crabharness/`](./crabharness/) — a plugin-based mission control plane that plans evidence collection, delegates heavy crawling to Codex workers, validates artifacts through a three-gate pipeline, and emits OpenCrab-ready promotion packages. See [CrabHarness README](./crabharness/README.md). - ---- - -## What's New (v1.5.0) - -### Phase 1 — Core Stabilization -- **Grammar versioning**: `GRAMMAR_VERSION = "1.0.0"` in every manifest response -- **Type Schema Registry**: YAML schemas in `schemas/types/` — `required`, `enum` validation on every node write -- **Receipt IDs**: every `add_node` / `add_edge` returns `receipt_id + receipt_ts` for provenance - -### Phase 2 — Action / Workflow Runtime -- **WorkflowEngine**: SQL state machine (`pending → running → approved/rejected → completed/failed`) with full audit log -- **ApprovalEngine**: three-state approval queue linked to workflow runs -- **CrabHarness `promotion-apply`**: CLI command + MCP tool to apply PromotionPackages inline - -### Phase 3 — Identity / Canonicalization / Promotion -- **IdentityEngine**: alias table + fuzzy duplicate detection — no auto-merge, human review first -- **CanonicalizeEngine**: tombstone-based node merge — alias nodes preserved, `resolve_canonical()` for lookups -- **PromotionEngine**: full extraction lifecycle `candidate → validated → promoted | rejected` with evidence linking - -### Phase 4 — Query / Reasoning Upgrade -- **BM25 Index**: pure Python keyword search over all node properties (no external deps) -- **RRF Reranker**: Reciprocal Rank Fusion merges vector + BM25 + graph results; BM25 cross-score boosts query-relevant hits -- **Policy-aware filtering**: pass `subject_id` to `ontology_query` and results are filtered by ReBAC `view` permission - -### Phase 5 — Productization -- **Tenant isolation**: `tenant_id` context stamped on writes; `X-Tenant-Id` header support -- **Billing hooks**: `billing_events` table tracks node_write / query / ingest / promotion per tenant -- **Schema packs**: domain bundles (`saas`, `biomedical`, `legal`) installable with one MCP call - -**Total MCP tools: 30** - ---- - -## Architecture - -``` - ┌─────────────────────────────────────────────┐ - │ OpenCrab MCP Server │ - │ (stdio JSON-RPC) │ - └──────────────────┬──────────────────────────┘ - │ - ┌────────────────────────────┼────────────────────────────┐ - │ │ │ - ┌───────▼──────┐ ┌────────▼───────┐ ┌────────▼───────┐ - │ grammar/ │ │ ontology/ │ │ stores/ │ - │ manifest.py │ │ builder.py │ │ │ - │ validator.py│ │ rebac.py │ │ neo4j_store │ - │ glossary.py │ │ impact.py │ │ chroma_store │ - └──────────────┘ │ query.py │ │ mongo_store │ - │ identity.py │ │ sql_store │ - ┌───────────────┐ │ canonicalize │ └───────┬────────┘ - │ schemas/ │ │ promotion.py │ │ - │ types/*.yaml │ │ bm25.py │ ┌─────────────▼──────────┐ - │ packs/*.yaml │ │ reranker.py │ │ billing/ │ - │ loader.py │ │ tenant.py │ │ hooks.py │ - │ pack_registry│ └────────────────┘ └────────────────────────┘ - └───────────────┘ - ┌──────────────────────────────────┐ - │ execution/ │ - │ workflow.py approvals.py │ - │ action_registry.py │ - └──────────────────────────────────┘ -``` - -### MetaOntology OS — 9 Spaces - -| Space | Node Types | Role | -|------------|---------------------------------------------|-----------------------------------| -| subject | User, Team, Org, Agent | Actors with identity and agency | -| resource | Project, Document, File, Dataset, Tool, API | Artifacts that subjects act upon | -| evidence | TextUnit, LogEntry, Evidence | Raw empirical observations | -| concept | Entity, Concept, Topic, Class | Abstract knowledge | -| claim | Claim, Covariate | Derived assertions | -| community | Community, CommunityReport | Concept clusters | -| outcome | Outcome, KPI, Risk | Measurable results | -| lever | Lever | Tunable control variables | -| policy | Policy, Sensitivity, ApprovalRule | Governance rules | - -### MetaEdge Relationship Grammar - -``` -subject ──[owns, manages, can_view, can_edit, can_execute, can_approve]──► resource -resource ──[contains, derived_from, logged_as]──────────────────────────► evidence -evidence ──[mentions, describes, exemplifies]────────────────────────────► concept -evidence ──[supports, contradicts, timestamps]───────────────────────────► claim -concept ──[related_to, subclass_of, part_of, influences, depends_on]────► concept -concept ──[contributes_to, constrains, predicts, degrades]──────────────► outcome -lever ──[raises, lowers, stabilizes, optimizes]───────────────────────► outcome -lever ──[affects]─────────────────────────────────────────────────────► concept -community ──[clusters, summarizes]────────────────────────────────────────► concept -policy ──[protects, classifies, restricts]─────────────────────────────► resource -policy ──[permits, denies, requires_approval]──────────────────────────► subject -``` - ---- +**LocalCrab builds. OpenCrab SaaS distributes.** + +OpenCrab is the public integration repository for the LocalCrab ontology +factory and the OpenCrab hosted ecosystem at [opencrab.sh](https://opencrab.sh). + +This repository contains the local engine: MetaOntology OS grammar, MCP tools, +CrabHarness evidence collection, local stores, promotion lifecycle, and pack +export contracts. It does **not** contain the private implementation of the +hosted `opencrab.sh` SaaS product. + +Any sample app or API code in this repository is local/demo infrastructure for +developer testing. It is not the production `opencrab.sh` SaaS code. + +## What This Repo Is For + +| Layer | Role | Lives here? | +| --- | --- | --- | +| LocalCrab | Local ontology factory for crawling, parsing, evidence indexing, Neo4j validation, and ZIP pack export. | Yes | +| CrabHarness | Mission-first control plane for crawler planning, worker delegation, evidence validation, and promotion packages. | Yes | +| MetaOntology OS | Canonical grammar, schemas, ReBAC, identity/canonicalization, promotion lifecycle, and MCP server tools. | Yes | +| OpenCrab SaaS | Hosted ingestion, marketplace, profiles, MCP access, community, and paid/free pack circulation. | No, linked via [opencrab.sh](https://opencrab.sh) | + +The intended flow: + +```text +source material or crawl target + | + v +CrabHarness mission planning + | + v +evidence collection + OCR/CLIP indexing + | + v +MetaOntology grammar extraction + | + v +Neo4j/Cypher validation + | + v +OpenCrab Pack v1 ZIP + | + v +opencrab.sh ingest + marketplace + ecosystem distribution +``` + +## LocalCrab and OpenCrab SaaS + +LocalCrab is quality-first. It exists to produce ontology packs with strong +evidence coverage, traceable parsing, OCR/CLIP context, graph validation, and +promotion receipts. + +OpenCrab SaaS is ecosystem-first. It exists to ingest packs, make them useful +to users and agents, distribute them through marketplace/community surfaces, +and expose hosted MCP access. + +Read the full relationship model: + +- [LocalCrab and OpenCrab SaaS relationship](./docs/localcrab-opencrab-relationship.md) +- [LocalCrab factory workflow](./docs/localcrab-factory-workflow.md) +- [OpenCrab Pack v1 ZIP format](./docs/opencrab-pack-v1.md) ## Quick Start -### 1. Start the data services - -```bash -docker-compose up -d -``` - -This starts Neo4j, MongoDB, PostgreSQL, and ChromaDB. - -### 2. Install OpenCrab +### 1. Install LocalCrab ```bash pip install -e ".[dev]" ``` -### 3. Configure environment +### 2. Run LocalCrab ```bash -opencrab init # creates .env from template -# Edit .env if your credentials differ from defaults +opencrab serve ``` -**Local mode (no Docker required):** +LocalCrab runs locally by default. It uses SQLite, JSON files, and a local +Chroma persistent store under `./opencrab_data`. -```bash -STORAGE_MODE=local opencrab serve -``` - -Local mode uses SQLite + JSON files — no external services needed. - -### 4. Seed example data - -```bash -python scripts/seed_ontology.py -``` - -### 5. Verify connectivity +### 3. Verify the grammar and query path ```bash opencrab status +opencrab manifest +opencrab query "system performance and error rates" ``` -### 6. Add to Claude Code MCP +### 4. Add LocalCrab as an MCP server ```bash claude mcp add opencrab -- opencrab serve ``` -Or add to your `.claude/mcp.json` manually (see below). - -### 7. Run a query - -```bash -opencrab query "system performance and error rates" -opencrab manifest # see the full grammar -``` - ---- - -## Claude Code MCP Configuration - -Add to `~/.claude/mcp.json` (or project-level `.mcp.json`): - -```json -{ - "mcpServers": { - "opencrab": { - "command": "opencrab", - "args": ["serve"], - "env": { - "NEO4J_URI": "bolt://localhost:7687", - "NEO4J_USER": "neo4j", - "NEO4J_PASSWORD": "opencrab", - "MONGODB_URI": "mongodb://root:opencrab@localhost:27017", - "MONGODB_DB": "opencrab", - "POSTGRES_URL": "postgresql://opencrab:opencrab@localhost:5432/opencrab", - "CHROMA_HOST": "localhost", - "CHROMA_PORT": "8000" - } - } - } -} -``` - -**Local mode (SQLite + JSON, no Docker):** +Or add it manually: ```json { "mcpServers": { "opencrab": { "command": "opencrab", - "args": ["serve"], - "env": { - "STORAGE_MODE": "local" - } + "args": ["serve"] } } } ``` ---- - -## MCP Tool Reference - -### Core Ontology (9 tools) - -#### `ontology_manifest` -Returns the full MetaOntology grammar with version, spaces, meta-edges, impact categories, and ReBAC config. - -#### `ontology_add_node` -```json -{ - "space": "subject", - "node_type": "User", - "node_id": "user-alice", - "properties": { "name": "Alice Chen", "role": "analyst" }, - "tenant_id": "acme", - "subject_id": "user-alice" -} -``` -Returns `receipt_id + receipt_ts`. Properties validated against type schema if one exists. - -#### `ontology_add_edge` -```json -{ - "from_space": "subject", "from_id": "user-alice", - "relation": "owns", - "to_space": "resource", "to_id": "doc-spec" -} -``` -Validates the `(from_space, to_space, relation)` triple against the grammar before write. - -#### `ontology_query` — Hybrid Query (v2) -```json -{ - "question": "What factors degrade system performance?", - "spaces": ["concept", "outcome"], - "limit": 10, - "subject_id": "user-alice", - "tenant_id": "acme", - "use_bm25": true, - "use_rerank": true -} -``` -Pipeline: vector similarity → BM25 keyword → graph expansion → RRF reranking → policy filter. - -#### `query_bm25` -```json -{ "question": "machine learning", "spaces": ["concept"], "limit": 10 } -``` -BM25-only keyword search. Fast and deterministic, no embeddings. - -#### `ontology_impact` -```json -{ "node_id": "lever-cache-ttl", "change_type": "update" } -``` -Returns triggered I1–I7 impact categories and affected neighbours. - -#### `ontology_rebac_check` -```json -{ "subject_id": "user-alice", "permission": "edit", "resource_id": "ds-events" } -``` -Returns `{ "granted": true/false, "reason": "...", "path": [...] }`. - -#### `ontology_lever_simulate` -```json -{ "lever_id": "lever-cache-ttl", "direction": "lowers", "magnitude": 0.7 } -``` - -#### `ontology_ingest` -```json -{ "text": "...", "source_id": "incident-2026-01", "metadata": { "space": "evidence" } } -``` - ---- - -### Identity & Canonicalization (7 tools) - -#### `identity_add_alias` -Register `alias_id` as an alias for `canonical_id`. Types: `name`, `merge`, `external`. +## CrabHarness -#### `identity_resolve_canonical` -Resolve a node ID to its canonical form. Returns `is_alias: true` if it was an alias. +[`crabharness/`](./crabharness/) is the mission-first control plane for +evidence collection. It plans what to crawl, delegates heavy work to plugin +workers, validates the collected bundle, and emits OpenCrab-ready promotion +packages. -#### `identity_propose_duplicate` -Propose two nodes as potential duplicates for human review. +Core responsibilities: -#### `identity_resolve_duplicate` -Accept or reject a pending duplicate candidate. On accept, registers alias automatically. +- Decide crawl target, scope, depth, volume, rate limits, and success criteria. +- Store every collected page, document, file, image, and log as evidence. +- Preserve hashes, source URLs or paths, crawl timestamps, parser status, and + missing-context candidates. +- Promote only after completeness, semantic relevance, and autoresearch gates + pass. -#### `identity_list_pending_duplicates` -List all pending duplicate candidates sorted by similarity. +See the [CrabHarness README](./crabharness/README.md). -#### `canonicalize_merge_nodes` -Merge `alias_id` into `canonical_id` using tombstone pattern — alias node is preserved. +## MetaOntology OS -#### `canonicalize_find_and_propose` -Find nodes with similar names and auto-propose them as duplicate candidates. +LocalCrab keeps the existing MetaOntology OS grammar and MCP surface as the +canonical ontology contract. ---- +### 9 Spaces -### Promotion Lifecycle (4 tools) +| Space | Role | +| --- | --- | +| subject | Actors with identity, agency, roles, and permissions. | +| resource | Documents, datasets, tools, APIs, files, and projects. | +| evidence | Raw observations, logs, text units, parser/OCR outputs, and empirical records. | +| concept | Entities, concepts, topics, classes, and domain abstractions. | +| claim | Derived assertions grounded by evidence. | +| community | Clusters and summaries of related concepts or actors. | +| outcome | KPIs, risks, impacts, and measurable results. | +| lever | Tunable controls that affect outcomes or concepts. | +| policy | Access, sensitivity, approval, and governance rules. | -Tracks extracted entities through: `candidate → validated → promoted | rejected`. +### Core MCP Tools -#### `promotion_register_candidate` -Register an extracted entity as a promotion candidate (not visible in normal queries yet). +- `ontology_manifest`: return the full grammar. +- `ontology_add_node`: add or update a grammar-validated node. +- `ontology_add_edge`: add a grammar-validated edge. +- `ontology_query`: hybrid vector + BM25 + graph query. +- `ontology_impact`: I1-I7 impact analysis. +- `ontology_rebac_check`: relationship-based access check. +- `ontology_ingest`: ingest text into the local ontology stores. +- `harness_promotion_apply`: apply a CrabHarness promotion package. -#### `promotion_validate_candidate` -Mark a candidate as validated — ready for final review. +## OpenCrab Pack v1 -#### `promotion_promote` -Promote to `promoted` status. Optionally links evidence nodes via `supports` edges. +LocalCrab exports ontology deliveries as an OpenCrab Pack v1 ZIP. The pack is +designed to be recognized by OpenCrab SaaS while remaining reproducible in a +local Neo4j environment. -#### `promotion_reject` -Mark a candidate as rejected with an optional reason. +Required high-level layout: ---- - -### Workflow & Approvals (3 tools) - -#### `workflow_create_run` -Start an auditable workflow run in `pending` state before executing any sensitive action. - -#### `workflow_advance` -Advance a run to a new status (`pending → running → approved/rejected → completed/failed`). - -#### `approval_request` -Submit an approval request linked to a workflow run. - ---- - -### CrabHarness Integration (1 tool) - -#### `harness_promotion_apply` -```json -{ "package": { ... }, "dry_run": false } +```text +manifest.json +graph/nodes.jsonl +graph/edges.jsonl +evidence/index.jsonl +quality/report.json +neo4j/import.cypher +neo4j/opencrab_ingest.jsonl +neo4j/export_status.json +README.md +sample_queries.json +community_reports.json ``` -Apply a CrabHarness PromotionPackage inline. Returns `receipt_id + receipt_ts` per node/edge written. Use `dry_run: true` to validate without writing. - ---- -### Billing & Usage (2 tools) +The packaging pipeline is: -#### `billing_get_usage` -```json -{ "tenant_id": "acme", "event_type": "query", "since": "2026-04-01T00:00:00Z" } +```text +validate -> Neo4j import/check -> Neo4j graph export -> normalized SaaS export -> ZIP package ``` -Aggregated usage counts by event type for a tenant. - -#### `billing_list_events` -Recent raw billing events for a tenant (last N). - ---- - -### Schema Packs (3 tools) - -Domain-specific schema bundles that extend the type registry without touching core schemas. -#### `schema_pack_list` -List available packs with install status. Built-in packs: `saas`, `biomedical`, `legal`. - -#### `schema_pack_install` -```json -{ "name": "biomedical" } -``` -Generates stub YAML type schemas in `schemas/types/`. Existing user schemas are never overwritten. - -#### `schema_pack_uninstall` -```json -{ "name": "biomedical", "force": false } -``` - ---- - -## CLI Reference - -``` -opencrab init Create .env from template -opencrab serve Start MCP server (stdio) -opencrab status Check store connections -opencrab ingest Ingest files into vector store -opencrab query Run a hybrid query -opencrab manifest Print MetaOntology grammar -``` - -Global flags: - -``` -opencrab --version -opencrab query --json-output -opencrab manifest --json-output -opencrab ingest -r -opencrab ingest -e .txt,.md -``` - ---- - -## Impact Categories (I1–I7) - -| ID | Name | Question | -|----|--------------------------|-------------------------------------------------------| -| I1 | Data impact | What data values or records change? | -| I2 | Relation impact | What graph edges are affected? | -| I3 | Space impact | Which ontology spaces are touched? | -| I4 | Permission impact | Which access permissions change? | -| I5 | Logic impact | Which business rules are invalidated? | -| I6 | Cache/index impact | Which caches or indexes must be refreshed? | -| I7 | Downstream system impact | Which external systems or APIs are affected? | - ---- - -## Active Metadata Layers - -Every node and edge can carry orthogonal metadata attributes: - -| Layer | Attributes | -|------------|------------------------------------| -| existence | identity, provenance, lineage | -| quality | confidence, freshness, completeness| -| relational | dependency, sensitivity, maturity | -| behavioral | usage, mutation, effect | - ---- +See [OpenCrab Pack v1 ZIP format](./docs/opencrab-pack-v1.md). ## Development ```bash -make dev-install # install with dev extras -make up # start docker services -make seed # seed example data -make test # run test suite -make coverage # test + coverage report -make lint # ruff linter -make format # black + isort -make status # check store connections +make dev-install +make seed +make test +make status ``` -### Running integration tests +Run integration tests: ```bash OPENCRAB_INTEGRATION=1 pytest tests/ -v ``` -### Project structure +## Project Structure -``` +```text opencrab/ -├── grammar/ # MetaOntology grammar (manifest, validator, glossary) -├── schemas/ # Type schemas (YAML), schema packs, loader -│ ├── types/ # Per-node-type YAML schemas (required/enum validation) -│ └── packs/ # Domain packs: saas, biomedical, legal -├── ontology/ # Core engines -│ ├── builder.py # Node/edge write with receipt IDs + schema validation -│ ├── query.py # Hybrid query: vector + BM25 + graph + RRF reranker -│ ├── bm25.py # Pure Python BM25 index -│ ├── reranker.py # RRF + BM25 cross-score fusion -│ ├── identity.py # Alias table + duplicate candidate detection -│ ├── canonicalize.py # Tombstone-based node merge -│ ├── promotion.py # Extraction lifecycle (candidate → promoted) -│ ├── tenant.py # Tenant isolation context + property stamping -│ ├── rebac.py # Relationship-based access control -│ └── impact.py # I1–I7 impact analysis -├── execution/ # Workflow & approvals runtime -│ ├── workflow.py # WorkflowEngine state machine -│ ├── approvals.py # ApprovalEngine queue -│ └── action_registry.py # YAML action schemas -├── billing/ # Usage metering -│ └── hooks.py # BillingHooks — billing_events table -├── stores/ # Store adapters (Neo4j, ChromaDB, MongoDB, PostgreSQL, Local) -└── mcp/ # MCP server (stdio JSON-RPC) + 30 tool definitions -tests/ # Test suite -scripts/ # Seed script -crabharness/ # Evidence collection pipeline -docker-compose.yml # All data services -``` - ---- - -## CrabHarness — Mission Control Plane - -[`crabharness/`](./crabharness/) is the companion data collection pipeline for OpenCrab. It owns the "how do we *get* the evidence that fills the ontology" layer, while OpenCrab owns the "how is the ontology structured and queried" layer. - -### What CrabHarness adds - -| Capability | Description | -|------------|-------------| -| **Plugin workers** | Drop a `worker.manifest.json` + `adapter.py` into `codex_workers/` and a new collector appears in the catalog — no core changes. | -| **Mission-first planner** | Declarative `mission.json` picks workers by `target_object` + tag match instead of hardcoded pipelines. | -| **Three-gate validation** | Every artifact bundle scored on (1) completeness, (2) semantic relevance, (3) autoresearch verdict. | -| **Harvest dedupe** | `.seen.json` side-index with SHA256 IDs for `harvest` collection mode. | -| **Promotion packages** | Builds OpenCrab node/edge packages — **never mutates OpenCrab directly**. | -| **MCP-native apply** | `harness_promotion_apply` MCP tool applies packages inline from Claude without file I/O. | - -### Quickstart - -```bash -cd crabharness -pip install -e . -crabharness catalog -crabharness run missions/examples/github-trending-harvest.json -crabharness promotion-apply artifacts/runs///promotion_package.json -``` - -### Integration with OpenCrab - -CrabHarness produces promotion packages as pure JSON. Apply them via: -- **CLI**: `crabharness promotion-apply ` -- **MCP**: `harness_promotion_apply { "package": { ... } }` - -The `dry_run: true` flag validates grammar and schemas without writing. - ---- + grammar/ MetaOntology grammar, validator, glossary + schemas/ YAML type schemas, schema packs, action schemas + ontology/ builder, query, identity, canonicalization, promotion, ReBAC + execution/ workflow and approval runtime + billing/ local usage hooks + stores/ Neo4j, Chroma, Mongo, SQL, and local adapters + mcp/ MCP server and tool registry +crabharness/ + crabharness/ mission planner, runtime, validation, promotion package builder + codex_workers/ plugin workers for crawlers and collectors + missions/ example missions +docs/ public integration and pack delivery contracts +``` + +## Korean Summary + +이 리포지토리는 LocalCrab과 OpenCrab SaaS를 하나의 제품처럼 설명하는 공개 통합 +리포지토리입니다. LocalCrab은 온톨로지 공장입니다. 크롤링, 파싱, OCR, CLIP +이미지 컨텍스트, evidence 풀 인덱싱, Neo4j 검증, ZIP 팩 생성을 담당합니다. + +OpenCrab SaaS는 [opencrab.sh](https://opencrab.sh)의 생태계 허브입니다. 완성된 +팩을 인제스트하고, 마켓플레이스와 커뮤니티에서 배포하며, hosted MCP 접근을 +제공합니다. 단, `opencrab.sh`의 내부 SaaS 코드는 이 공개 리포지토리에 포함하지 +않습니다. ## License -MIT — see [LICENSE](LICENSE). - ---- - -*OpenCrab: resistance is futile. Your agent will become an ontology.* +MIT. diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile deleted file mode 100644 index 6fe8eb3..0000000 --- a/apps/api/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.11-slim - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 - -WORKDIR /app - -COPY . /app - -RUN pip install --upgrade pip \ - && pip install -e /app \ - && pip install -r /app/apps/api/requirements.txt - -EXPOSE 8080 - -CMD ["sh", "-c", "uvicorn apps.api.main:app --host 0.0.0.0 --port ${PORT:-8080}"] diff --git a/apps/api/main.py b/apps/api/main.py index 65c98b9..17fe5dd 100644 --- a/apps/api/main.py +++ b/apps/api/main.py @@ -447,7 +447,7 @@ def get_status(ctx: ApiContext = Depends(get_context)) -> dict[str, Any]: "ok": True, "service": "opencrab-api", "tier": _tier(), - "storage_mode": ctx.settings.storage_mode, + "storage_mode": "local", "stores": { "graph": {"available": bool(getattr(ctx.graph, "available", False)), "healthy": bool(_safe_count(ctx.graph.ping, 0))}, "vector": {"available": bool(getattr(ctx.vector, "available", False)), "healthy": bool(_safe_count(ctx.vector.ping, 0))}, diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile deleted file mode 100644 index b7ccfd0..0000000 --- a/apps/web/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM node:20-alpine AS deps - -RUN apk add --no-cache libc6-compat -WORKDIR /app - -COPY apps/web/package.json ./ -RUN npm install - -FROM node:20-alpine AS builder - -RUN apk add --no-cache libc6-compat -WORKDIR /app - -COPY --from=deps /app/node_modules ./node_modules -COPY apps/web ./ - -ENV NEXT_TELEMETRY_DISABLED=1 - -RUN npm run build - -FROM node:20-alpine AS runner - -WORKDIR /app -ENV NODE_ENV=production -ENV NEXT_TELEMETRY_DISABLED=1 -ENV PORT=3000 -ENV HOSTNAME=0.0.0.0 - -COPY --from=builder /app/public ./public -COPY --from=builder /app/.next/standalone ./ -COPY --from=builder /app/.next/static ./.next/static - -EXPOSE 3000 - -CMD ["node", "server.js"] diff --git a/apps/web/lib/api.ts b/apps/web/lib/api.ts index 9792f59..dda7bd2 100644 --- a/apps/web/lib/api.ts +++ b/apps/web/lib/api.ts @@ -1,4 +1,4 @@ -const BASE = process.env.NEXT_PUBLIC_API_URL || 'https://opencrabback.up.railway.app' +const BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000' function headers(apiKey?: string) { const h: Record = { 'Content-Type': 'application/json' } diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index de0b9b6..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,158 +0,0 @@ -version: "3.9" - -services: - neo4j: - image: neo4j:5 - container_name: opencrab-neo4j - ports: - - "7474:7474" # HTTP browser - - "7687:7687" # Bolt protocol - environment: - NEO4J_AUTH: neo4j/opencrab - NEO4J_PLUGINS: '["apoc"]' - NEO4J_dbms_security_procedures_unrestricted: "apoc.*" - NEO4J_dbms_security_procedures_allowlist: "apoc.*" - NEO4J_apoc_export_file_enabled: "true" - NEO4J_apoc_import_file_enabled: "true" - NEO4J_apoc_import_file_use__neo4j__config: "true" - volumes: - - neo4j_data:/data - - neo4j_logs:/logs - - neo4j_import:/var/lib/neo4j/import - - neo4j_plugins:/plugins - healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://localhost:7474"] - interval: 15s - timeout: 10s - retries: 5 - start_period: 30s - restart: unless-stopped - - mongodb: - image: mongo:7 - container_name: opencrab-mongodb - ports: - - "27017:27017" - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: opencrab - MONGO_INITDB_DATABASE: opencrab - volumes: - - mongo_data:/data/db - healthcheck: - test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping')"] - interval: 15s - timeout: 10s - retries: 5 - start_period: 20s - restart: unless-stopped - - postgres: - image: postgres:16 - container_name: opencrab-postgres - ports: - - "5432:5432" - environment: - POSTGRES_USER: opencrab - POSTGRES_PASSWORD: opencrab - POSTGRES_DB: opencrab - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U opencrab -d opencrab"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 15s - restart: unless-stopped - - chromadb: - image: chromadb/chroma:latest - container_name: opencrab-chromadb - ports: - - "8000:8000" - environment: - CHROMA_SERVER_HOST: 0.0.0.0 - CHROMA_SERVER_HTTP_PORT: 8000 - ANONYMIZED_TELEMETRY: "False" - ALLOW_RESET: "True" - volumes: - - chroma_data:/chroma/chroma - healthcheck: - # The Chroma image does not ship curl/wget. Use bash's /dev/tcp support - # so docker compose can reliably gate API/Web startup on Chroma readiness. - test: - [ - "CMD-SHELL", - "bash -ec \"exec 3<>/dev/tcp/127.0.0.1/8000 && printf 'GET /api/v2/heartbeat HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q '200'\"", - ] - interval: 10s - timeout: 5s - retries: 5 - start_period: 15s - restart: unless-stopped - - api: - build: - context: . - dockerfile: apps/api/Dockerfile - container_name: opencrab-api - ports: - - "8001:8001" - environment: - STORAGE_MODE: docker - LOG_LEVEL: ${LOG_LEVEL:-INFO} - OPENCRAB_API_KEY: ${OPENCRAB_API_KEY:-local-opencrab-key} - OPENCRAB_TIER: ${OPENCRAB_TIER:-free} - OPENCRAB_CORS_ORIGINS: ${OPENCRAB_CORS_ORIGINS:-http://localhost:3000,http://127.0.0.1:3000} - NEO4J_URI: bolt://neo4j:7687 - NEO4J_USER: ${NEO4J_USER:-neo4j} - NEO4J_PASSWORD: ${NEO4J_PASSWORD:-opencrab} - MONGODB_URI: mongodb://root:opencrab@mongodb:27017/opencrab?authSource=admin - MONGODB_DB: ${MONGODB_DB:-opencrab} - POSTGRES_URL: postgresql://opencrab:opencrab@postgres:5432/opencrab - CHROMA_HOST: chromadb - CHROMA_PORT: 8000 - CHROMA_COLLECTION: ${CHROMA_COLLECTION:-opencrab_vectors} - PORT: 8001 - depends_on: - neo4j: - condition: service_healthy - mongodb: - condition: service_healthy - postgres: - condition: service_healthy - chromadb: - condition: service_healthy - restart: unless-stopped - - web: - build: - context: . - dockerfile: apps/web/Dockerfile - container_name: opencrab-web - ports: - - "3000:3000" - environment: - PORT: 3000 - NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:8001} - NEXT_PUBLIC_OPENCRAB_API_KEY: ${NEXT_PUBLIC_OPENCRAB_API_KEY:-local-opencrab-key} - NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-pk_test_replace_me} - CLERK_SECRET_KEY: ${CLERK_SECRET_KEY:-sk_test_replace_me} - depends_on: - api: - condition: service_started - restart: unless-stopped - -volumes: - neo4j_data: - neo4j_logs: - neo4j_import: - neo4j_plugins: - mongo_data: - postgres_data: - chroma_data: - -networks: - default: - name: opencrab-network diff --git a/docs/localcrab-factory-workflow.md b/docs/localcrab-factory-workflow.md new file mode 100644 index 0000000..733ee7a --- /dev/null +++ b/docs/localcrab-factory-workflow.md @@ -0,0 +1,257 @@ +# LocalCrab Factory Workflow + +LocalCrab is quality-first. Its job is to deliver ontology packs with complete +evidence coverage, traceable parsing, strong OCR/image handling, graph +validation, and promotion receipts. + +The existing CrabHarness and MetaOntology grammar stay in place. This workflow +adds a stricter operating policy around evidence completeness and final +promotion quality. + +## Factory Pipeline + +```text +plan -> collect -> parse -> index evidence -> extract graph + -> canonicalize -> validate quality -> verify in Neo4j + -> export graph back from Neo4j -> export OpenCrab Pack v1 ZIP +``` + +## Path A: No Source Material Yet + +Use this path when the user only knows the target domain or source area, but no +complete source corpus has been prepared. + +### 1. Plan the Crawl + +Create a CrabHarness mission before collecting anything. + +The mission must freeze: + +- Crawl target: domains, repositories, websites, datasets, APIs, or search + spaces. +- Scope: allowed and excluded locations. +- Depth: link depth, folder depth, pagination depth, API expansion depth. +- Volume: max pages, files, bytes, images, records, and runtime. +- Rate limits: concurrency, delay, retry policy, robots/API constraints. +- Required evidence: pages, source files, documents, images, logs, metadata. +- Success criteria: minimum artifacts, required fields, semantic questions, + minimum completeness score, and minimum semantic score. + +### 2. Collect Evidence + +Every crawler output is evidence, not just successful documents. + +Store: + +- Raw pages and files. +- Extracted text. +- Parser logs. +- Crawl logs. +- HTTP/API metadata. +- Images and media metadata. +- Failed or skipped URLs with reason codes. +- Missing-context candidates. + +Each evidence artifact must include: + +- Stable evidence id. +- Source URL or local path. +- Crawl timestamp. +- Content hash. +- Parser status. +- Media type. +- Byte size. +- Parent source or crawl run id. + +### 3. Full Evidence Indexing + +Before ontology extraction, build `evidence/index.jsonl`. + +The index should connect: + +- Source artifact -> parsed document. +- Parsed document -> chunks. +- Chunk -> extracted node ids. +- Chunk -> extracted edge ids. +- Image -> CLIP context ids. +- OCR page/region -> text chunks. +- Parser failure -> missing-context candidate. + +No node or edge should be promoted without evidence refs. + +### 4. Extract Under MetaOntology Grammar + +Use the existing grammar path: + +- `ontology_manifest` to inspect valid spaces and relations. +- `ontology_add_node` and `ontology_add_edge` semantics for graph shape. +- Schema registry for type-specific required fields. +- Identity and canonicalization tools for duplicate handling. +- Promotion lifecycle for candidate, validated, promoted, or rejected status. + +### 5. Quality Gate + +The final promotion gate must strongly weight: + +- Parsing completeness. +- OCR completeness when applicable. +- Evidence coverage. +- Chunk coverage. +- Node evidence reference integrity. +- Edge evidence reference integrity. +- Orphan node count. +- Broken edge count. +- Missing source-map count. + +Promotion should fail if graph entities cannot be traced back to evidence. + +### 6. Neo4j Export Snapshot + +After Cypher import/check succeeds, export the loaded graph back from Neo4j into +`neo4j/opencrab_ingest.jsonl`. + +This gives the pack two graph views: + +- `graph/nodes.jsonl` and `graph/edges.jsonl`: LocalCrab's canonical planned + graph. +- `neo4j/opencrab_ingest.jsonl`: the actual graph snapshot after Neo4j accepted + it. + +If the counts diverge, `quality/report.json` must explain why. + +## Path B: Source Material Already Exists + +Use this path when the user provides files, folders, PDFs, images, exports, or +datasets. + +### 1. Native Parsing First + +Use native parsers when possible: + +- Markdown, text, HTML, CSV, JSON, JSONL, XML, YAML. +- Office documents where text can be extracted. +- PDFs with embedded text. +- Structured datasets and metadata files. + +Store parser output and parser status as evidence. + +### 2. OCR Fallback and Double OCR + +If a document cannot be read directly, or if it is scanned/image-heavy, run OCR +with at least two passes or engines. + +The evidence index should preserve: + +- OCR engine name. +- OCR version/config when available. +- Page or region id. +- Confidence score. +- Text output. +- Differences between OCR passes. +- Chosen merged text. +- Low-confidence spans. + +Do not silently discard OCR disagreements. Treat them as reviewable evidence. + +### 3. Image Context With CLIP + +Image-heavy sources need visual context, not just OCR text. + +For each image, record: + +- Image hash and source path. +- CLIP embedding id. +- Captions or generated descriptions. +- Tags/classes. +- Nearby document context. +- Related page/section/chunk ids. + +Image context can support concepts, claims, or evidence nodes, but it should +remain traceable to the original image. + +### 4. Full Evidence Indexing + +Every source item must be represented in `evidence/index.jsonl`, including: + +- Native parser output. +- OCR output. +- CLIP image context. +- Chunk records. +- Node and edge references. +- Parser and OCR failures. +- Missing or skipped resources. + +The central question for the final gate is: + +> The material exists. Did any context disappear before graph promotion? + +### 5. Quality Gate + +Promotion requires: + +- No unexplained missing source files. +- No promoted node without evidence refs. +- No promoted edge without evidence refs. +- No broken evidence refs. +- No broken node references in edges. +- OCR coverage for unreadable pages. +- CLIP context for meaningful image assets. +- Neo4j import/check success or an explicit failure reason. + +## Recommended Quality Scores + +Use these as defaults unless a mission explicitly sets stricter thresholds. + +| Metric | Recommended minimum | +| --- | --- | +| Parsing completeness | 0.95 | +| OCR completeness for scanned/image docs | 0.90 | +| Evidence coverage | 0.98 | +| Chunk coverage | 0.95 | +| Node evidence ref integrity | 1.00 | +| Edge evidence ref integrity | 1.00 | +| Graph reference integrity | 1.00 | +| Broken evidence refs | 0 | +| Broken edges | 0 | + +## Benchmark-Steered Quality Gates + +LocalCrab packs should be evaluated against the failure modes that appear in +real RAG benchmarks, not only against schema validity. + +The current priority benchmark weaknesses are: + +- Relationship questions: "changed reason", "revision background", + "why not applicable", and "company-specific restriction". +- Multi-hop questions: material + method + defect + law/regulation + risk. +- Fire/material questions: combinations that may fail fire performance and the + evidence or regulation behind the risk. +- Hallucination traps: nonexistent methods, nonexistent standards, or + unsupported installation rules. + +For every pack, add benchmark steering checks before promotion: + +- Important standards and construction methods must have explicit `reason`, + `rationale`, `revision_reason`, or `not_applicable_reason` evidence when the + source material contains that context. +- Nodes that represent materials, methods, defects, risks, or laws should be + connected through evidence-backed edges, not left as isolated keyword hits. +- If a source says a method is unavailable, prohibited, deleted, restricted, or + company-specific, that negative applicability must become a claim. +- Fire-performance and legal-risk claims must include evidence refs and should + connect material nodes to law/standard nodes where source material permits. +- Missing information must be represented as absence with evidence, not filled + by model inference. + +These gates are designed so OpenCrab does not merely retrieve documents. It +retrieves the relationship structure that explains why the answer is true. + +## Promotion Rule + +LocalCrab should be conservative: + +> If evidence traceability is incomplete, keep the ontology in candidate or +> validated state. Promote only when missing context is explained or repaired. + +That is the key difference between a casual graph export and a LocalCrab +factory delivery. diff --git a/docs/localcrab-opencrab-relationship.md b/docs/localcrab-opencrab-relationship.md new file mode 100644 index 0000000..0dab7c6 --- /dev/null +++ b/docs/localcrab-opencrab-relationship.md @@ -0,0 +1,115 @@ +# LocalCrab and OpenCrab SaaS Relationship + +OpenCrab is designed as one product with two deployment surfaces. + +LocalCrab is the local ontology factory. OpenCrab SaaS, available at +[opencrab.sh](https://opencrab.sh), is the hosted ecosystem where finished +packs are ingested, distributed, installed, queried, and sold or shared. + +The hosted SaaS implementation is private and is intentionally not included in +this public repository. + +## Responsibilities + +| System | Primary responsibility | Optimized for | +| --- | --- | --- | +| LocalCrab | Build high-quality ontology packs from local files, crawled sources, OCR/CLIP outputs, and Neo4j validation. | Quality, evidence coverage, reproducibility. | +| OpenCrab SaaS | Ingest OpenCrab packs, manage users and profiles, expose hosted MCP access, and distribute packs through marketplace/community surfaces. | Ecosystem growth, distribution, hosted usability. | +| GitHub repository | Provide public grammar, LocalCrab runtime, CrabHarness, pack format, examples, and developer onboarding. | International developer access and trust. | + +## Boundary + +This repository may contain: + +- LocalCrab MCP server code. +- MetaOntology OS grammar, schemas, validators, stores, and local runtime. +- CrabHarness crawler planning and evidence validation workflow. +- OpenCrab Pack v1 delivery contract. +- Example missions, example packs, and public documentation. +- Links and calls to action for [opencrab.sh](https://opencrab.sh). + +This repository must not contain: + +- Private hosted SaaS business logic. +- Production `opencrab.sh` deployment secrets. +- SaaS billing, profile, marketplace, or community implementation details that + are not intended to be public. +- Private customer packs or source corpora. + +Any local app or API surface in this repository should be treated as demo or +developer infrastructure. It must not be described as the production +`opencrab.sh` implementation. + +## Data Flow + +```text +1. Source selection + - Existing local files, PDFs, images, datasets, or a crawl target. + +2. CrabHarness mission planning + - Target, scope, depth, volume, rate limit, and success criteria are frozen. + +3. Evidence collection + - Pages, documents, files, images, logs, parser outputs, OCR outputs, and + CLIP image context are stored as evidence artifacts. + +4. Full evidence indexing + - Every evidence item receives a stable id, source reference, timestamp, + hash, parser status, and links to chunks, nodes, and edges. + +5. MetaOntology extraction + - Nodes and edges are extracted under the existing grammar. + +6. Canonicalization and promotion + - Duplicate candidates, aliases, evidence-backed claims, and promotion + status are handled before final delivery. + +7. Neo4j validation + - Cypher import/check confirms graph integrity and reproducibility. + +8. OpenCrab Pack v1 ZIP + - A SaaS-ingestible ZIP is produced. + +9. OpenCrab SaaS ingestion + - The pack is uploaded or imported into opencrab.sh for hosted use, + marketplace distribution, community discovery, and MCP access. +``` + +## Public Positioning + +Use this sentence when explaining the system publicly: + +> LocalCrab is the open ontology factory; OpenCrab SaaS is the hosted ecosystem. + +For developers: + +> Build and validate ontology packs locally, then bring them to opencrab.sh for +> hosted ingestion, distribution, and agent access. + +For pack creators: + +> LocalCrab gives you quality control before your ontology reaches users. + +For SaaS users: + +> OpenCrab SaaS lets you install, query, share, and monetize ontology packs +> without running the local factory. + +## Compatibility Rule + +LocalCrab and OpenCrab SaaS should communicate through stable artifacts, not +private implementation coupling. The canonical bridge is OpenCrab Pack v1: + +- `manifest.json` declares the pack identity, counts, limits, grammar version, + hashes, and quality summary. +- `graph/nodes.jsonl` and `graph/edges.jsonl` are the SaaS-ingestible graph. +- `evidence/index.jsonl` preserves traceability from source artifacts to graph + entities. +- `quality/report.json` records whether the pack is safe to promote. +- `neo4j/import.cypher` preserves local reproducibility. +- `neo4j/opencrab_ingest.jsonl` is extracted back out of Neo4j after + import/check, so the pack includes the actual graph snapshot that passed + verification. + +This keeps LocalCrab useful as open source while letting OpenCrab SaaS evolve +as the hosted ecosystem. diff --git a/docs/opencrab-pack-v1.md b/docs/opencrab-pack-v1.md new file mode 100644 index 0000000..2d3c4c8 --- /dev/null +++ b/docs/opencrab-pack-v1.md @@ -0,0 +1,381 @@ +# OpenCrab Pack v1 ZIP Format + +OpenCrab Pack v1 is the delivery contract between LocalCrab and OpenCrab SaaS. + +The ZIP must be useful in two places: + +- Local reproducibility: a developer can inspect the evidence and replay the + graph in Neo4j. +- Hosted ingestion: OpenCrab SaaS can read the normalized graph and evidence + index without depending on private LocalCrab internals. + +## Packaging Pipeline + +```text +validate -> Neo4j import/check -> Neo4j graph export -> package +``` + +The ZIP is not just an archive. It is the promotion artifact. + +## Required Layout + +```text +manifest.json +graph/nodes.jsonl +graph/edges.jsonl +evidence/index.jsonl +quality/report.json +neo4j/import.cypher +neo4j/opencrab_ingest.jsonl +neo4j/export_status.json +README.md +sample_queries.json +community_reports.json +``` + +Optional but recommended: + +```text +raw/ +parsed/ +ocr/ +images/ +clip/ +scripts/import_to_neo4j.py +neo4j/import_status.json +neo4j/export_status.json +``` + +## `manifest.json` + +The manifest identifies the pack and tells OpenCrab SaaS how to handle it. + +Required fields: + +```json +{ + "format_version": "opencrab-pack-v1", + "pack_id": "example_pack", + "title": "Example Ontology Pack", + "version": "1.0.0", + "grammar_version": "1.0.0", + "created_at": "2026-05-17T00:00:00Z", + "created_by": "LocalCrab", + "license": { + "scope": "personal", + "name": "MIT" + }, + "source": { + "mode": "crawl", + "label": "Example source", + "url": "https://example.com", + "description": "What was collected or provided." + }, + "counts": { + "documents": 0, + "chunks": 0, + "images": 0, + "evidence": 0, + "nodes": 0, + "edges": 0, + "files": 0, + "bytes": 0 + }, + "limits": { + "split_recommended": false, + "staged_ingest_recommended": false, + "reason": null + }, + "quality": { + "parsing_completeness": 1.0, + "ocr_completeness": null, + "clip_coverage": null, + "evidence_coverage": 1.0, + "chunk_coverage": 1.0, + "node_evidence_integrity": 1.0, + "edge_evidence_integrity": 1.0, + "relationship_evidence_coverage": 1.0, + "multihop_path_coverage": 1.0, + "graph_reference_integrity": 1.0, + "promotion_status": "validated" + }, + "retrieval_hints": { + "relation_cues": ["reason", "revision", "not_applicable", "risk", "law"], + "benchmark_focus": ["relationship_questions", "multi_hop", "hallucination_guard"] + }, + "hashes": { + "nodes_sha256": "...", + "edges_sha256": "...", + "evidence_sha256": "...", + "pack_sha256": "..." + }, + "artifacts": { + "nodes": "graph/nodes.jsonl", + "edges": "graph/edges.jsonl", + "evidence_index": "evidence/index.jsonl", + "quality_report": "quality/report.json", + "neo4j_cypher": "neo4j/import.cypher", + "opencrab_ingest": "neo4j/opencrab_ingest.jsonl", + "neo4j_export_status": "neo4j/export_status.json" + } +} +``` + +## `graph/nodes.jsonl` + +One JSON object per line. + +Required fields: + +```json +{ + "id": "node:example", + "label": "Example", + "space": "concept", + "node_type": "Entity", + "properties": {}, + "evidence_refs": ["evidence:example:chunk:001"], + "quality": { + "confidence": 0.95, + "parser": "native", + "promotion_status": "validated" + } +} +``` + +Rules: + +- `space` must exist in the MetaOntology grammar. +- `node_type` must be valid for the selected space or accepted by an installed + schema pack. +- `id` must be stable inside the pack. +- `evidence_refs` should not be empty for promoted nodes. +- `properties` must not contain secrets or private SaaS-only fields. +- Relationship-heavy nodes should expose searchable fields such as `reason`, + `rationale`, `revision_reason`, `not_applicable_reason`, `risk`, `law`, + `standard`, or equivalent nested properties when those facts exist. + +## `graph/edges.jsonl` + +One JSON object per line. + +Required fields: + +```json +{ + "id": "edge:example:mentions:target", + "from_id": "node:example", + "to_id": "node:target", + "from_space": "evidence", + "to_space": "concept", + "relation": "mentions", + "confidence": 0.92, + "evidence_refs": ["evidence:example:chunk:001"], + "properties": {} +} +``` + +Rules: + +- `from_id` and `to_id` must exist in `graph/nodes.jsonl`. +- `(from_space, to_space, relation)` must validate against the grammar. +- `evidence_refs` should not be empty for promoted edges. +- Broken edges must fail validation. +- Edges that explain changes, restrictions, risk, law, defect chains, or + material compatibility should preserve relation-specific evidence. These + edges are critical for relationship and multi-hop RAG benchmarks. + +## `evidence/index.jsonl` + +The evidence index is the traceability layer. Every source artifact, parser +output, OCR output, CLIP output, chunk, and graph reference should be reachable +from here. + +Example: + +```json +{ + "evidence_id": "evidence:example:chunk:001", + "kind": "text_chunk", + "source": { + "url": "https://example.com/page", + "path": null, + "title": "Example page" + }, + "hash": "sha256:...", + "collected_at": "2026-05-17T00:00:00Z", + "parser": { + "status": "ok", + "method": "native_html", + "warnings": [] + }, + "ocr": null, + "clip": null, + "location": { + "document_id": "doc:example", + "page": null, + "section": "Introduction", + "chunk_index": 1 + }, + "links": { + "document_id": "doc:example", + "chunk_ids": ["chunk:example:001"], + "node_ids": ["node:example"], + "edge_ids": ["edge:example:mentions:target"] + } +} +``` + +For OCR evidence, include engine, confidence, page or region id, low-confidence +spans, and pass number. For CLIP evidence, include image hash, embedding id, +caption or tags, and related chunk ids. + +## `quality/report.json` + +The quality report records why the pack is safe, unsafe, or not yet ready to +promote. + +Required top-level fields: + +```json +{ + "status": "pass", + "summary": { + "parsing_completeness": 1.0, + "ocr_completeness": null, + "clip_coverage": null, + "evidence_coverage": 1.0, + "chunk_coverage": 1.0, + "node_evidence_integrity": 1.0, + "edge_evidence_integrity": 1.0, + "relationship_evidence_coverage": 1.0, + "multihop_path_coverage": 1.0, + "graph_reference_integrity": 1.0 + }, + "checks": { + "grammar": "pass", + "schema": "pass", + "evidence_refs": "pass", + "orphan_nodes": "pass", + "broken_edges": "pass", + "neo4j_import": "pass" + }, + "counts": { + "missing_evidence_refs": 0, + "broken_edges": 0, + "orphan_nodes": 0, + "parser_failures": 0, + "ocr_low_confidence_spans": 0 + }, + "issues": [] +} +``` + +Recommended defaults: + +| Metric | Minimum | +| --- | --- | +| parsing completeness | 0.95 | +| OCR completeness for scanned/image docs | 0.90 | +| evidence coverage | 0.98 | +| chunk coverage | 0.95 | +| node evidence integrity | 1.00 | +| edge evidence integrity | 1.00 | +| relationship evidence coverage | 0.95 | +| multi-hop path coverage | 0.90 | +| graph reference integrity | 1.00 | + +## `neo4j/import.cypher` + +This file is for local reproducibility. It should be possible to load the pack +into Neo4j and verify graph counts and relationship integrity. + +The Cypher import does not replace `graph/nodes.jsonl` and `graph/edges.jsonl`. +Those JSONL files are the canonical SaaS-ingestible graph. + +## `neo4j/opencrab_ingest.jsonl` + +This file is the normalized graph export extracted from Neo4j after +`neo4j/import.cypher` has been applied and checked. + +It is not merely a copy of `graph/nodes.jsonl` and `graph/edges.jsonl`. +It records the graph state that actually passed Neo4j import/check, including +relationship types and properties as Neo4j loaded them. + +Each line should be one of: + +```json +{ "kind": "node", "payload": { "...": "..." } } +{ "kind": "edge", "payload": { "...": "..." } } +{ "kind": "evidence", "payload": { "...": "..." } } +``` + +The node and edge counts in `opencrab_ingest.jsonl` must match the canonical +graph files unless the quality report explicitly explains a filtered item. + +Generate it with: + +```bash +opencrab export-neo4j-pack \ + --pack-id my_pack_id \ + --output build/my_pack/neo4j/opencrab_ingest.jsonl +``` + +The command also writes `neo4j/export_status.json` with exported node and edge +counts, timestamp, pack filter, and output path. + +## SaaS Limits and Splitting + +`manifest.json` must record pack size and counts so OpenCrab SaaS can choose +normal ingest, staged ingest, or split-pack ingest. + +Recommended warning thresholds: + +| Dimension | Warning threshold | +| --- | --- | +| ZIP size | 100 MB | +| Nodes | 100,000 | +| Edges | 300,000 | +| Evidence rows | 500,000 | +| Files | 20,000 | + +If a pack exceeds a threshold, set: + +```json +{ + "limits": { + "split_recommended": true, + "staged_ingest_recommended": true, + "reason": "Pack exceeds recommended SaaS ingest threshold for evidence rows." + } +} +``` + +## Validation Failures + +A pack must fail validation when: + +- `manifest.json` is missing or has an unsupported `format_version`. +- Required graph or evidence files are missing. +- A node uses an invalid grammar space. +- An edge uses an invalid grammar relation. +- An edge references a missing node. +- A promoted node or edge has no evidence refs. +- Relationship claims do not preserve reason/revision/applicability evidence + that exists in the source material. +- Multi-hop risk questions cannot traverse material, method, defect, standard, + and law nodes where those source entities exist. +- `evidence/index.jsonl` references missing artifacts without explanation. +- The ZIP exceeds declared limits and does not request staged or split ingest. +- Neo4j import/check fails without an explicit failure reason. + +## Marketplace Metadata + +The pack should include human-facing metadata for opencrab.sh: + +- `README.md`: public explanation, source, scope, quality status, and usage. +- `sample_queries.json`: starter questions for users and agents. +- `community_reports.json`: GraphRAG/community summaries when available. + +These files help OpenCrab SaaS present the pack in marketplace and community +surfaces without exposing private SaaS implementation details. diff --git a/opencrab/cli.py b/opencrab/cli.py index 229d7cf..6e3ec6b 100644 --- a/opencrab/cli.py +++ b/opencrab/cli.py @@ -74,12 +74,10 @@ def init(force: bool) -> None: console.print( Panel( "[bold]Next steps:[/bold]\n\n" - "1. Edit [cyan].env[/cyan] with your credentials.\n" - "2. Start services:\n" - " [cyan]docker-compose up -d[/cyan]\n" - "3. Add to Claude Code MCP config:\n" + "1. Optional: edit [cyan].env[/cyan] to change LOCAL_DATA_DIR.\n" + "2. Add to Claude Code MCP config:\n" " [cyan]claude mcp add opencrab -- opencrab serve[/cyan]\n" - "4. Seed example data:\n" + "3. Seed example data:\n" " [cyan]python scripts/seed_ontology.py[/cyan]", title="OpenCrab Setup", border_style="green", @@ -89,14 +87,7 @@ def init(force: bool) -> None: def _write_default_env(path: Path) -> None: content = """\ -NEO4J_URI=bolt://localhost:7687 -NEO4J_USER=neo4j -NEO4J_PASSWORD=opencrab -MONGODB_URI=mongodb://root:opencrab@localhost:27017 -MONGODB_DB=opencrab -POSTGRES_URL=postgresql://opencrab:opencrab@localhost:5432/opencrab -CHROMA_HOST=localhost -CHROMA_PORT=8000 +LOCAL_DATA_DIR=./opencrab_data CHROMA_COLLECTION=opencrab_vectors MCP_SERVER_NAME=opencrab MCP_SERVER_VERSION=0.1.0 @@ -138,8 +129,8 @@ def status() -> None: ) cfg = get_settings() - mode_label = f"[bold cyan]{cfg.storage_mode.upper()} MODE[/bold cyan]" - storage_loc = cfg.local_data_dir if cfg.is_local else "Docker services" + mode_label = "[bold cyan]LOCAL MODE[/bold cyan]" + storage_loc = cfg.local_data_dir console.print(f"\n{mode_label} - storage at: {storage_loc}\n") graph = make_graph_store(cfg) @@ -147,20 +138,12 @@ def status() -> None: docs = make_doc_store(cfg) sql = make_sql_store(cfg) - if cfg.is_local: - store_rows: list[tuple[str, str, Any]] = [ - ("Graph (SQLite)", cfg.local_data_dir + "/graph.db", graph), - ("Vector (ChromaDB)", cfg.local_data_dir + "/chroma", vector), - ("Docs (JSON files)", cfg.local_data_dir + "/docs", docs), - ("SQL (SQLite)", cfg.local_data_dir + "/opencrab.db", sql), - ] - else: - store_rows = [ - ("Graph (Neo4j)", cfg.neo4j_uri, graph), - ("Docs (MongoDB)", cfg.mongodb_uri.split("@")[-1], docs), - ("SQL (PostgreSQL)", cfg.postgres_url.split("@")[-1], sql), - ("Vector (ChromaDB)", cfg.chroma_url, vector), - ] + store_rows: list[tuple[str, str, Any]] = [ + ("Graph (SQLite)", cfg.local_data_dir + "/graph.db", graph), + ("Vector (ChromaDB)", cfg.local_data_dir + "/chroma", vector), + ("Docs (JSON files)", cfg.local_data_dir + "/docs", docs), + ("SQL (SQLite)", cfg.local_data_dir + "/opencrab.db", sql), + ] table = Table(title="OpenCrab Store Status", show_header=True, header_style="bold cyan") table.add_column("Store", style="bold") @@ -458,6 +441,50 @@ def manifest(json_output: bool) -> None: ) +# --------------------------------------------------------------------------- +# export-neo4j-pack +# --------------------------------------------------------------------------- + + +@main.command("export-neo4j-pack") +@click.option( + "--output", + "-o", + required=True, + type=click.Path(), + help="Output JSONL path, usually neo4j/opencrab_ingest.jsonl.", +) +@click.option("--pack-id", default=None, help="Optional pack_id/source filter.") +@click.option("--node-limit", default=500_000, show_default=True, type=int) +@click.option("--edge-limit", default=1_000_000, show_default=True, type=int) +def export_neo4j_pack( + output: str, + pack_id: str | None, + node_limit: int, + edge_limit: int, +) -> None: + """Export a verified Neo4j graph snapshot for an OpenCrab pack.""" + from opencrab.config import get_settings + from opencrab.pack import export_neo4j_opencrab_ingest + from opencrab.stores.neo4j_store import Neo4jStore + + cfg = get_settings() + neo4j = Neo4jStore( + uri=cfg.neo4j_uri, + user=cfg.neo4j_user, + password=cfg.neo4j_password, + database=cfg.neo4j_database, + ) + status = export_neo4j_opencrab_ingest( + neo4j, + output, + pack_id=pack_id, + node_limit=node_limit, + edge_limit=edge_limit, + ) + console.print_json(json.dumps(status, ensure_ascii=False)) + + # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- diff --git a/opencrab/config.py b/opencrab/config.py index 14cd143..e144303 100644 --- a/opencrab/config.py +++ b/opencrab/config.py @@ -7,8 +7,6 @@ from __future__ import annotations from functools import lru_cache -from typing import Literal - from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -24,15 +22,14 @@ class Settings(BaseSettings): ) # ------------------------------------------------------------------ - # Storage mode: "local" (no Docker) or "docker" (full services) + # Local storage. LocalCrab intentionally runs without managed service + # dependencies; all default stores live under LOCAL_DATA_DIR. # ------------------------------------------------------------------ - storage_mode: Literal["local", "docker"] = Field( - default="local", alias="STORAGE_MODE" - ) local_data_dir: str = Field(default="./opencrab_data", alias="LOCAL_DATA_DIR") # ------------------------------------------------------------------ - # Neo4j (docker mode only) + # Optional Neo4j replay/export settings for pack verification. + # Normal LocalCrab operation does not require a Neo4j server. # ------------------------------------------------------------------ neo4j_uri: str = Field(default="bolt://localhost:7687", alias="NEO4J_URI") neo4j_user: str = Field(default="neo4j", alias="NEO4J_USER") @@ -40,7 +37,8 @@ class Settings(BaseSettings): neo4j_database: str | None = Field(default=None, alias="NEO4J_DATABASE") # ------------------------------------------------------------------ - # MongoDB (docker mode only) + # Legacy external-store settings retained for direct adapter use. + # The default LocalCrab factory path uses local JSON and SQLite stores. # ------------------------------------------------------------------ mongodb_uri: str = Field( default="mongodb://root:opencrab@localhost:27017", @@ -48,8 +46,6 @@ class Settings(BaseSettings): ) mongodb_db: str = Field(default="opencrab", alias="MONGODB_DB") - # ------------------------------------------------------------------ - # PostgreSQL (docker mode) / SQLite (local mode) # ------------------------------------------------------------------ postgres_url: str = Field( default="postgresql://opencrab:opencrab@localhost:5432/opencrab", @@ -57,7 +53,7 @@ class Settings(BaseSettings): ) # ------------------------------------------------------------------ - # ChromaDB (docker mode uses HttpClient; local mode uses PersistentClient) + # ChromaDB local persistent collection. # ------------------------------------------------------------------ chroma_host: str = Field(default="localhost", alias="CHROMA_HOST") chroma_port: int = Field(default=8000, alias="CHROMA_PORT") @@ -86,7 +82,7 @@ def chroma_url(self) -> str: @property def is_local(self) -> bool: - return self.storage_mode == "local" + return True @property def sqlite_url(self) -> str: diff --git a/opencrab/mcp/tools.py b/opencrab/mcp/tools.py index 9e529fd..c8179c3 100644 --- a/opencrab/mcp/tools.py +++ b/opencrab/mcp/tools.py @@ -55,7 +55,7 @@ def _clean_meta(meta: dict[str, Any]) -> dict[str, Any]: def _get_context() -> dict[str, Any]: - """Lazily initialise all stores and engines using the factory (respects STORAGE_MODE).""" + """Lazily initialise LocalCrab stores and engines using the local factory.""" global _context if _context: return _context diff --git a/opencrab/ontology/bm25.py b/opencrab/ontology/bm25.py index c6a54f6..9ba97e3 100644 --- a/opencrab/ontology/bm25.py +++ b/opencrab/ontology/bm25.py @@ -24,14 +24,74 @@ _B = 0.75 # Properties to include when building the text representation of a node -_TEXT_FIELDS = ("name", "description", "text", "title", "label", "summary", "content") +_TEXT_FIELDS = ( + "name", + "description", + "text", + "title", + "label", + "summary", + "content", + "reason", + "rationale", + "change_reason", + "revision_reason", + "applicability", + "limitation", + "limitations", + "risk", + "law", + "standard", + "evidence", + "source", + "heading_path", +) +_HANGUL_RE = re.compile(r"[가-힣]") +_MAX_PROPERTY_TEXT = 1200 def _tokenize(text: str) -> list[str]: - """Lowercase, strip punctuation, split on whitespace.""" + """Lowercase, strip punctuation, split on whitespace. + + Korean construction benchmarks contain many compound nouns and short + relation cues ("변경 이유", "적용 불가", "개정"). Whitespace tokenisation + alone misses those matches, so Hangul tokens also contribute 2- and 3-char + n-grams. This keeps exact English behaviour while making Korean recall + much less brittle. + """ text = text.lower() text = re.sub(r"[^\w\s]", " ", text) - return [t for t in text.split() if t] + tokens: list[str] = [] + for token in text.split(): + if not token: + continue + tokens.append(token) + if _HANGUL_RE.search(token) and len(token) >= 3: + for n in (2, 3): + tokens.extend(token[i : i + n] for i in range(0, len(token) - n + 1)) + return tokens + + +def _flatten_property_text(value: Any, depth: int = 0) -> list[str]: + """Collect searchable scalar text from nested property values.""" + if value is None or depth > 2: + return [] + if isinstance(value, str): + return [value[:_MAX_PROPERTY_TEXT]] + if isinstance(value, (int, float, bool)): + return [str(value)] + if isinstance(value, list): + parts: list[str] = [] + for item in value[:50]: + parts.extend(_flatten_property_text(item, depth + 1)) + return parts + if isinstance(value, dict): + parts = [] + for key, item in list(value.items())[:80]: + parts.append(str(key).replace("_", " ")) + parts.extend(_flatten_property_text(item, depth + 1)) + return parts + return [str(value)[:_MAX_PROPERTY_TEXT]] def _node_text(node: dict[str, Any]) -> str: @@ -47,6 +107,11 @@ def _node_text(node: dict[str, Any]) -> str: val = props.get(field) if val: parts.append(str(val)) + for key, val in props.items(): + if key in _TEXT_FIELDS: + continue + parts.append(str(key).replace("_", " ")) + parts.extend(_flatten_property_text(val)) return " ".join(parts) diff --git a/opencrab/ontology/query.py b/opencrab/ontology/query.py index 6c9447f..9c529ce 100644 --- a/opencrab/ontology/query.py +++ b/opencrab/ontology/query.py @@ -15,6 +15,7 @@ from __future__ import annotations import logging +import os from dataclasses import dataclass, field from typing import Any @@ -33,6 +34,45 @@ "CONTRADICTS": 0.5, } _DEFAULT_EDGE_SCORE: float = 0.5 +_BM25_NODE_LIMIT = int(os.getenv("OPENCRAB_BM25_NODE_LIMIT", "50000")) +_RELATION_QUERY_CUES = ( + "why", + "reason", + "rationale", + "change", + "revision", + "background", + "cannot", + "applicable", + "risk", + "law", + "regulation", + "이유", + "변경", + "개정", + "배경", + "불가", + "불가능", + "위험", + "법규", + "조합", + "관계", + "연결", +) +_MULTIHOP_QUERY_CUES = ( + "connect", + "relationship", + "multi", + "chain", + "cause", + "effect", + "연결", + "관계", + "원인", + "영향", + "단계", + "구분", +) # Lazily imported Phase 4 modules to avoid circular deps at module load _BM25Index: Any = None @@ -77,6 +117,82 @@ def to_dict(self) -> dict[str, Any]: } +@dataclass(frozen=True) +class _QueryProfile: + """Adaptive retrieval settings for the current question.""" + + vector_limit: int + bm25_limit: int + graph_limit: int + graph_depth: int + anchor_limit: int + rerank_limit: int + + +def _contains_any(text: str, cues: tuple[str, ...]) -> bool: + lowered = text.lower() + return any(cue in lowered for cue in cues) + + +def _profile_for_query(question: str, limit: int, graph_depth: int) -> _QueryProfile: + """Use higher recall for relationship and multi-hop questions.""" + relation_intent = _contains_any(question, _RELATION_QUERY_CUES) + multihop_intent = _contains_any(question, _MULTIHOP_QUERY_CUES) + depth = graph_depth + if relation_intent: + depth = max(depth, 2) + if multihop_intent: + depth = max(depth, 3) + + multiplier = 8 if relation_intent or multihop_intent else 4 + return _QueryProfile( + vector_limit=min(max(limit * multiplier, 24), 80), + bm25_limit=min(max(limit * (multiplier + 2), 40), 180), + graph_limit=min(max(limit * (multiplier + 2), 50), 220), + graph_depth=min(depth, 3), + anchor_limit=12 if relation_intent or multihop_intent else 6, + rerank_limit=min(max(limit * 4, 20), 80), + ) + + +def _ordered_unique(values: list[str | None], limit: int) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for value in values: + if not value or value in seen: + continue + seen.add(value) + output.append(value) + if len(output) >= limit: + break + return output + + +def _property_text(props: dict[str, Any], relation_type: str = "") -> str: + parts = [relation_type.replace("_", " ")] + for key in ( + "text", + "name", + "title", + "label", + "summary", + "description", + "reason", + "rationale", + "change_reason", + "revision_reason", + "evidence", + "source", + "heading_path", + ): + value = props.get(key) + if value: + parts.append(str(value)) + if len(parts) <= 1: + parts.append(str(props)) + return " ".join(parts)[:4000] + + class HybridQuery: """Orchestrates hybrid vector + graph queries.""" @@ -131,29 +247,35 @@ def query( list[QueryResult] sorted by descending score. """ result_lists: list[list[dict[str, Any]]] = [] + profile = _profile_for_query(question, limit, graph_depth) # --- Stage 1: Vector similarity search --- - vector_hits = self._vector_search(question, spaces, limit) + vector_hits = self._vector_search(question, spaces, profile.vector_limit) if vector_hits: result_lists.append([r.to_dict() for r in vector_hits]) # --- Stage 2: BM25 keyword search --- + bm25_hits: list[dict[str, Any]] = [] if use_bm25 and self._doc_store is not None: - bm25_hits = self._bm25_search(question, spaces, limit) + bm25_hits = self._bm25_search(question, spaces, profile.bm25_limit) if bm25_hits: result_lists.append(bm25_hits) - # --- Stage 3: Graph expansion from vector anchor nodes --- - anchor_ids = [hit.node_id for hit in vector_hits if hit.node_id] + # --- Stage 3: Graph expansion from vector and BM25 anchor nodes --- + anchor_ids = _ordered_unique( + [hit.node_id for hit in vector_hits if hit.node_id] + + [hit.get("node_id") for hit in bm25_hits], + profile.anchor_limit, + ) if anchor_ids and self._neo4j.available: - graph_results = self._graph_expand(anchor_ids, graph_depth, limit) + graph_results = self._graph_expand(anchor_ids, profile.graph_depth, profile.graph_limit) if graph_results: result_lists.append([r.to_dict() for r in graph_results]) # --- Stage 4: Rerank --- if use_rerank and result_lists: reranker = _get_reranker()() - merged = reranker.rerank(question, result_lists, top_k=limit * 2) + merged = reranker.rerank(question, result_lists, top_k=profile.rerank_limit) else: # Flat merge without reranking seen: set[str | None] = set() @@ -191,7 +313,7 @@ def _bm25_search( """Run BM25 search against doc store nodes, using a cached index.""" try: if self._bm25_dirty or self._bm25_cache is None: - nodes = self._doc_store.list_nodes(limit=5000) + nodes = self._doc_store.list_nodes(limit=_BM25_NODE_LIMIT) BM25Index = _get_bm25() self._bm25_cache = BM25Index.build(nodes) self._bm25_cache_size = len(nodes) @@ -312,19 +434,21 @@ def _graph_expand( if nid and nid not in seen: seen.add(nid) rel_type = n.get("relation_type") or n.get("relationship_type") or "" - base_score = _EDGE_WEIGHTS.get(rel_type, _DEFAULT_EDGE_SCORE) * hop_decay + rel_key = str(rel_type).upper() + base_score = _EDGE_WEIGHTS.get(rel_key, _DEFAULT_EDGE_SCORE) * hop_decay expanded.append( QueryResult( source="graph", node_id=nid, score=base_score, - text=props.get("text") or props.get("name"), + text=_property_text(props, rel_type), metadata=props, graph_context={ "anchor_id": anchor_id, "labels": n.get("labels", []), "relation_type": rel_type, - "depth": depth, + "relationship_types": n.get("relationship_types"), + "depth": n.get("depth") or depth, }, ) ) diff --git a/opencrab/ontology/reranker.py b/opencrab/ontology/reranker.py index 11ef6fe..68dc05c 100644 --- a/opencrab/ontology/reranker.py +++ b/opencrab/ontology/reranker.py @@ -34,7 +34,105 @@ def _tokenize(text: str) -> list[str]: text = text.lower() text = re.sub(r"[^\w\s]", " ", text) - return [t for t in text.split() if t] + tokens: list[str] = [] + for token in text.split(): + if not token: + continue + tokens.append(token) + if re.search(r"[가-힣]", token) and len(token) >= 3: + for n in (2, 3): + tokens.extend(token[i : i + n] for i in range(0, len(token) - n + 1)) + return tokens + + +_RELATION_CUES = ( + "why", + "reason", + "rationale", + "change", + "revision", + "background", + "because", + "cannot", + "risk", + "law", + "regulation", + "이유", + "변경", + "개정", + "배경", + "불가", + "불가능", + "위험", + "법규", + "조합", + "관계", + "연결", +) +_SOURCE_WEIGHTS = { + "bm25": 1.08, + "vector": 1.0, + "graph": 0.96, + "hybrid": 1.0, +} + + +def _query_has_relation_intent(query: str) -> bool: + q = query.lower() + return any(cue in q for cue in _RELATION_CUES) + + +def _item_text(item: dict[str, Any]) -> str: + parts = [str(item.get("text") or "")] + metadata = item.get("metadata") or {} + graph_context = item.get("graph_context") or {} + try: + parts.append(str(metadata)) + parts.append(str(graph_context)) + except Exception: + pass + return " ".join(parts).lower() + + +def _intent_boost(query: str, item: dict[str, Any]) -> float: + if not _query_has_relation_intent(query): + return 1.0 + + source = str(item.get("source") or "hybrid") + sources = set(item.get("sources") or [source]) + boost = max(_SOURCE_WEIGHTS.get(str(candidate), 1.0) for candidate in sources) + haystack = _item_text(item) + + matched_cues = sum(1 for cue in _RELATION_CUES if cue in query.lower() and cue in haystack) + if matched_cues: + boost += min(0.24, matched_cues * 0.08) + if "graph" in sources: + boost += 0.12 + + return min(boost, 1.35) + + +def _merge_duplicate(existing: dict[str, Any], item: dict[str, Any]) -> None: + """Merge duplicate node hits so consensus survives reranking.""" + existing_sources = set(existing.get("sources") or [existing.get("source", "hybrid")]) + existing_sources.add(item.get("source", "hybrid")) + existing["sources"] = sorted(str(s) for s in existing_sources if s) + if len(existing["sources"]) > 1: + existing["source"] = "hybrid" + + existing["score"] = max(float(existing.get("score") or 0.0), float(item.get("score") or 0.0)) + + old_text = existing.get("text") or "" + new_text = item.get("text") or "" + if new_text and new_text not in old_text: + existing["text"] = f"{old_text}\n{new_text}".strip()[:4000] + + metadata = dict(existing.get("metadata") or {}) + metadata.update(item.get("metadata") or {}) + existing["metadata"] = metadata + + if not existing.get("graph_context") and item.get("graph_context"): + existing["graph_context"] = item["graph_context"] def _bm25_cross_score(query_tokens: list[str], doc_tokens: list[str], avgdl: float) -> float: @@ -95,8 +193,13 @@ def rerank( for results in result_lists: for item in results: nid = item.get("node_id") - if nid and nid not in all_results: - all_results[nid] = item + if not nid: + continue + if nid not in all_results: + all_results[nid] = dict(item) + all_results[nid]["sources"] = [item.get("source", "hybrid")] + else: + _merge_duplicate(all_results[nid], item) if not all_results: return [] @@ -142,6 +245,10 @@ def rerank( score = _ALPHA * rrf + (1 - _ALPHA) * bm25 else: score = rrf + source_count = len(all_results[nid].get("sources") or []) + if source_count > 1: + score += min(0.1, 0.04 * (source_count - 1)) + score *= _intent_boost(query, all_results[nid]) final.append((nid, round(score, 4))) final.sort(key=lambda x: x[1], reverse=True) diff --git a/opencrab/pack/__init__.py b/opencrab/pack/__init__.py new file mode 100644 index 0000000..e07ae2b --- /dev/null +++ b/opencrab/pack/__init__.py @@ -0,0 +1,5 @@ +"""OpenCrab pack export helpers.""" + +from opencrab.pack.neo4j_export import export_neo4j_opencrab_ingest + +__all__ = ["export_neo4j_opencrab_ingest"] diff --git a/opencrab/pack/neo4j_export.py b/opencrab/pack/neo4j_export.py new file mode 100644 index 0000000..0b32882 --- /dev/null +++ b/opencrab/pack/neo4j_export.py @@ -0,0 +1,163 @@ +"""Export a verified Neo4j graph snapshot into OpenCrab Pack v1 JSONL.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def _stable_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + + +def _sha_id(prefix: str, value: Any) -> str: + digest = hashlib.sha256(_stable_json(value).encode("utf-8")).hexdigest()[:16] + return f"{prefix}:{digest}" + + +def _clean_props(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _node_id(props: dict[str, Any]) -> str: + for key in ("id", "node_id", "uuid"): + value = props.get(key) + if value: + return str(value) + return _sha_id("neo4j-node", props) + + +def _edge_id(payload: dict[str, Any]) -> str: + props = _clean_props(payload.get("properties")) + for key in ("id", "edge_id", "uuid"): + value = props.get(key) + if value: + return str(value) + return _sha_id("neo4j-edge", payload) + + +def _pack_filter_clause(entity: str) -> str: + return ( + "$pack_id IS NULL " + f"OR {entity}.pack_id = $pack_id " + f"OR {entity}.source = $pack_id " + f"OR {entity}.source_id = $pack_id" + ) + + +def _node_query(limit: int) -> str: + return f""" + MATCH (n) + WHERE {_pack_filter_clause("n")} + RETURN properties(n) AS props, labels(n) AS labels + LIMIT {int(limit)} + """ + + +def _edge_query(limit: int) -> str: + node_filter = ( + f"({_pack_filter_clause('a')}) " + f"OR ({_pack_filter_clause('b')}) " + f"OR ({_pack_filter_clause('r')})" + ) + return f""" + MATCH (a)-[r]->(b) + WHERE {node_filter} + RETURN properties(a) AS source_props, + labels(a) AS source_labels, + properties(b) AS target_props, + labels(b) AS target_labels, + properties(r) AS rel_props, + type(r) AS relation + LIMIT {int(limit)} + """ + + +def _normalise_node(row: dict[str, Any]) -> dict[str, Any]: + props = _clean_props(row.get("props")) + labels = row.get("labels") or [] + labels = labels if isinstance(labels, list) else list(labels) + node_id = _node_id(props) + return { + "kind": "node", + "payload": { + "id": node_id, + "label": props.get("label") or props.get("name") or props.get("title") or node_id, + "space": props.get("space") or props.get("ontology_space") or "", + "node_type": props.get("node_type") or props.get("type") or (labels[0] if labels else ""), + "labels": labels, + "properties": props, + "evidence_refs": props.get("evidence_refs") or props.get("evidence_ids") or [], + }, + } + + +def _normalise_edge(row: dict[str, Any]) -> dict[str, Any]: + source_props = _clean_props(row.get("source_props")) + target_props = _clean_props(row.get("target_props")) + rel_props = _clean_props(row.get("rel_props")) + source_id = _node_id(source_props) + target_id = _node_id(target_props) + payload = { + "from_id": source_id, + "to_id": target_id, + "from_space": source_props.get("space") or source_props.get("ontology_space") or "", + "to_space": target_props.get("space") or target_props.get("ontology_space") or "", + "relation": str(row.get("relation") or rel_props.get("relation") or "").lower(), + "properties": rel_props, + "confidence": rel_props.get("confidence"), + "evidence_refs": rel_props.get("evidence_refs") or rel_props.get("evidence_ids") or [], + "source_labels": row.get("source_labels") or [], + "target_labels": row.get("target_labels") or [], + } + payload["id"] = _edge_id(payload) + return {"kind": "edge", "payload": payload} + + +def export_neo4j_opencrab_ingest( + neo4j_store: Any, + output_path: str | Path, + *, + pack_id: str | None = None, + node_limit: int = 500_000, + edge_limit: int = 1_000_000, +) -> dict[str, Any]: + """Export Neo4j's loaded graph into OpenCrab Pack v1 ingest JSONL. + + The output is meant to be stored at `neo4j/opencrab_ingest.jsonl` inside a + pack. It is derived from Neo4j after import/check, so the pack contains the + actual graph state that passed graph verification. + """ + if not getattr(neo4j_store, "available", False): + raise RuntimeError("Neo4j store is not available.") + + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + params = {"pack_id": pack_id} + + node_rows = neo4j_store.run_cypher(_node_query(node_limit), params) + edge_rows = neo4j_store.run_cypher(_edge_query(edge_limit), params) + + node_count = 0 + edge_count = 0 + with output.open("w", encoding="utf-8") as handle: + for row in node_rows: + handle.write(_stable_json(_normalise_node(row)) + "\n") + node_count += 1 + for row in edge_rows: + handle.write(_stable_json(_normalise_edge(row)) + "\n") + edge_count += 1 + + status = { + "status": "ok", + "pack_id": pack_id, + "output": str(output), + "nodes": node_count, + "edges": edge_count, + "exported_at": datetime.now(UTC).isoformat(), + } + output.with_name("export_status.json").write_text(_stable_json(status) + "\n", encoding="utf-8") + return status diff --git a/opencrab/stores/chroma_store.py b/opencrab/stores/chroma_store.py index d5c2815..95a6609 100644 --- a/opencrab/stores/chroma_store.py +++ b/opencrab/stores/chroma_store.py @@ -1,11 +1,9 @@ """ ChromaDB vector store adapter. -Supports two modes: -- local: uses ChromaDB PersistentClient (no server needed, data in local dir) -- docker: uses ChromaDB HttpClient (requires running ChromaDB container) - -All methods degrade gracefully when unavailable. +LocalCrab uses ChromaDB PersistentClient by default, so no Chroma server is +required. HttpClient remains available for direct adapter use, but the +LocalCrab factory always selects persistent local mode. """ from __future__ import annotations @@ -19,7 +17,7 @@ class ChromaStore: - """ChromaDB adapter — local (PersistentClient) or docker (HttpClient).""" + """ChromaDB adapter — persistent local by default, HttpClient optional.""" def __init__( self, diff --git a/opencrab/stores/factory.py b/opencrab/stores/factory.py index 55c34bc..aa86b01 100644 --- a/opencrab/stores/factory.py +++ b/opencrab/stores/factory.py @@ -1,5 +1,5 @@ """ -Store factory — returns the right backend based on STORAGE_MODE setting. +Store factory — returns LocalCrab's local-only backends. Usage: from opencrab.stores.factory import make_graph_store, make_vector_store, ... @@ -19,25 +19,15 @@ def make_graph_store(settings: Settings) -> Any: - """Return LocalGraphStore (local) or Neo4jStore (docker).""" - if settings.is_local: - from opencrab.stores.local_graph_store import LocalGraphStore + """Return the local SQLite graph store.""" + from opencrab.stores.local_graph_store import LocalGraphStore - db_path = os.path.join(settings.local_data_dir, "graph.db") - return LocalGraphStore(db_path=db_path) - else: - from opencrab.stores.neo4j_store import Neo4jStore - - return Neo4jStore( - uri=settings.neo4j_uri, - user=settings.neo4j_user, - password=settings.neo4j_password, - database=settings.neo4j_database, - ) + db_path = os.path.join(settings.local_data_dir, "graph.db") + return LocalGraphStore(db_path=db_path) def make_vector_store(settings: Settings) -> Any: - """Return ChromaStore in local or docker mode.""" + """Return the local persistent Chroma store.""" from opencrab.stores.chroma_store import ChromaStore chroma_path = os.path.join(settings.local_data_dir, "chroma") @@ -45,27 +35,21 @@ def make_vector_store(settings: Settings) -> Any: host=settings.chroma_host, port=settings.chroma_port, collection_name=settings.chroma_collection, - local_mode=settings.is_local, + local_mode=True, local_path=chroma_path, ) def make_doc_store(settings: Settings) -> Any: - """Return LocalDocStore (local) or MongoStore (docker).""" - if settings.is_local: - from opencrab.stores.local_doc_store import LocalDocStore - - docs_path = os.path.join(settings.local_data_dir, "docs") - return LocalDocStore(data_dir=docs_path) - else: - from opencrab.stores.mongo_store import MongoStore + """Return the local JSON document store.""" + from opencrab.stores.local_doc_store import LocalDocStore - return MongoStore(uri=settings.mongodb_uri, db_name=settings.mongodb_db) + docs_path = os.path.join(settings.local_data_dir, "docs") + return LocalDocStore(data_dir=docs_path) def make_sql_store(settings: Settings) -> Any: - """Return SQLStore with SQLite (local) or PostgreSQL (docker).""" + """Return the local SQLite SQL store.""" from opencrab.stores.sql_store import SQLStore - url = settings.sqlite_url if settings.is_local else settings.postgres_url - return SQLStore(url=url) + return SQLStore(url=settings.sqlite_url) diff --git a/opencrab/stores/local_doc_store.py b/opencrab/stores/local_doc_store.py index 8916655..70d6a4c 100644 --- a/opencrab/stores/local_doc_store.py +++ b/opencrab/stores/local_doc_store.py @@ -1,5 +1,5 @@ """ -Local document store — JSON-file-backed store for local (no-Docker) mode. +Local document store — JSON-file-backed store for local-only mode. Implements the same interface as MongoStore so consumers are agnostic of the backend. Each "collection" is a single JSON file on disk. diff --git a/opencrab/stores/local_graph_store.py b/opencrab/stores/local_graph_store.py index 8edfa7b..1387f65 100644 --- a/opencrab/stores/local_graph_store.py +++ b/opencrab/stores/local_graph_store.py @@ -1,5 +1,5 @@ """ -Local graph store — SQLite-backed graph for local (no-Docker) mode. +Local graph store — SQLite-backed graph for local-only mode. Implements the same interface as Neo4jStore so store consumers are agnostic of the backend. Nodes and edges are stored in SQLite tables; @@ -209,7 +209,7 @@ def find_neighbors( # Outgoing edges if direction in ("out", "both"): cur.execute( - "SELECT to_type, to_id FROM graph_edges WHERE from_id=?", + "SELECT to_type, to_id, relation FROM graph_edges WHERE from_id=?", (current_id,), ) for row in cur.fetchall(): @@ -218,13 +218,19 @@ def find_neighbors( visited.add(nid) node = self._fetch_node_props(cur, row["to_type"], nid) if node: - results.append({"properties": node, "labels": [row["to_type"]]}) + results.append({ + "properties": node, + "labels": [row["to_type"]], + "relation_type": row["relation"], + "relationship_types": [row["relation"]], + "depth": current_depth + 1, + }) queue.append((nid, current_depth + 1)) # Incoming edges if direction in ("in", "both"): cur.execute( - "SELECT from_type, from_id FROM graph_edges WHERE to_id=?", + "SELECT from_type, from_id, relation FROM graph_edges WHERE to_id=?", (current_id,), ) for row in cur.fetchall(): @@ -233,7 +239,13 @@ def find_neighbors( visited.add(nid) node = self._fetch_node_props(cur, row["from_type"], nid) if node: - results.append({"properties": node, "labels": [row["from_type"]]}) + results.append({ + "properties": node, + "labels": [row["from_type"]], + "relation_type": row["relation"], + "relationship_types": [row["relation"]], + "depth": current_depth + 1, + }) queue.append((nid, current_depth + 1)) return results[:limit] diff --git a/opencrab/stores/neo4j_store.py b/opencrab/stores/neo4j_store.py index 5148912..71725f4 100644 --- a/opencrab/stores/neo4j_store.py +++ b/opencrab/stores/neo4j_store.py @@ -229,20 +229,30 @@ def find_neighbors( raise RuntimeError("Neo4j is not available.") arrow = { - "out": "-[r*1..{depth}]->", - "in": "<-[r*1..{depth}]-", - "both": "-[r*1..{depth}]-", - }.get(direction, "-[r*1..{depth}]-").format(depth=depth) + "out": "-[rels*1..{depth}]->", + "in": "<-[rels*1..{depth}]-", + "both": "-[rels*1..{depth}]-", + }.get(direction, "-[rels*1..{depth}]-").format(depth=depth) cypher = f""" - MATCH (start {{id: $id}}){arrow}(neighbor) - RETURN DISTINCT properties(neighbor) AS props, labels(neighbor) AS labels + MATCH path = (start {{id: $id}}){arrow}(neighbor) + WITH neighbor, labels(neighbor) AS labels, properties(neighbor) AS props, + [rel IN relationships(path) | type(rel)] AS relationship_types, + length(path) AS depth + RETURN DISTINCT props, labels, relationship_types, depth LIMIT $limit """ with self._session() as session: result = session.run(cypher, id=node_id, limit=limit) return [ - {"properties": dict(record["props"]), "labels": list(record["labels"])} + { + "properties": dict(record["props"]), + "labels": list(record["labels"]), + "relationship_types": list(record["relationship_types"]), + "relation_type": list(record["relationship_types"])[-1] + if record["relationship_types"] else "", + "depth": record["depth"], + } for record in result ] diff --git a/railway.toml b/railway.toml deleted file mode 100644 index 291bf01..0000000 --- a/railway.toml +++ /dev/null @@ -1,9 +0,0 @@ -[build] -builder = "dockerfile" -dockerfilePath = "apps/api/Dockerfile" - -[deploy] -startCommand = "uvicorn apps.api.main:app --host 0.0.0.0 --port $PORT" -healthcheckPath = "/api/status" -healthcheckTimeout = 30 -restartPolicyType = "on_failure" diff --git a/tests/test_pack_neo4j_export.py b/tests/test_pack_neo4j_export.py new file mode 100644 index 0000000..b3df66f --- /dev/null +++ b/tests/test_pack_neo4j_export.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json + +import pytest + +from opencrab.pack import export_neo4j_opencrab_ingest + + +class FakeNeo4jStore: + available = True + + def __init__(self) -> None: + self.calls: list[tuple[str, dict[str, str | None]]] = [] + + def run_cypher(self, cypher: str, params: dict[str, str | None]): + self.calls.append((cypher, params)) + if "MATCH (n)" in cypher: + return [ + { + "props": { + "id": "node:fire-risk", + "label": "내화성능 미달 위험", + "space": "claim", + "node_type": "Claim", + "pack_id": "bench-pack", + "evidence_refs": ["evidence:1"], + }, + "labels": ["Claim"], + } + ] + return [ + { + "source_props": {"id": "node:material", "space": "concept", "node_type": "Entity"}, + "source_labels": ["Entity"], + "target_props": {"id": "node:law", "space": "policy", "node_type": "Policy"}, + "target_labels": ["Policy"], + "rel_props": {"confidence": 0.93, "evidence_refs": ["evidence:2"]}, + "relation": "CONSTRAINS", + } + ] + + +def test_export_neo4j_opencrab_ingest_writes_nodes_edges_and_status(tmp_path) -> None: + output = tmp_path / "neo4j" / "opencrab_ingest.jsonl" + status = export_neo4j_opencrab_ingest(FakeNeo4jStore(), output, pack_id="bench-pack") + + assert status["nodes"] == 1 + assert status["edges"] == 1 + assert (tmp_path / "neo4j" / "export_status.json").exists() + + lines = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert lines[0]["kind"] == "node" + assert lines[0]["payload"]["id"] == "node:fire-risk" + assert lines[1]["kind"] == "edge" + assert lines[1]["payload"]["relation"] == "constrains" + assert lines[1]["payload"]["evidence_refs"] == ["evidence:2"] + + +def test_export_neo4j_opencrab_ingest_requires_available_store(tmp_path) -> None: + class Unavailable: + available = False + + with pytest.raises(RuntimeError, match="Neo4j store is not available"): + export_neo4j_opencrab_ingest(Unavailable(), tmp_path / "out.jsonl") diff --git a/tests/test_query_retrieval.py b/tests/test_query_retrieval.py new file mode 100644 index 0000000..2d8d6c6 --- /dev/null +++ b/tests/test_query_retrieval.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from opencrab.ontology.bm25 import BM25Index +from opencrab.ontology.query import HybridQuery +from opencrab.ontology.reranker import Reranker + + +def test_bm25_handles_korean_relation_questions() -> None: + nodes = [ + { + "node_id": "fireproof-reason", + "space": "claim", + "node_type": "Claim", + "properties": { + "title": "가구표면 방염 변경 사유", + "description": "방염공사 시 가구표면 방염으로 변경된 이유와 사내 기준 배경.", + }, + }, + { + "node_id": "balcony-waterproofing", + "space": "resource", + "node_type": "Document", + "properties": { + "title": "발코니 바닥 방수 시공 기준", + "description": "난방 발코니의 방수 시공 기준과 체크사항.", + }, + }, + ] + + hits = BM25Index.build(nodes).search( + "방염공사 시 가구표면 방염으로 변경된 이유 알려줘", + limit=2, + ) + + assert hits + assert hits[0]["node_id"] == "fireproof-reason" + + +def test_hybrid_query_expands_graph_from_bm25_anchor_for_relation_intent() -> None: + class FakeChroma: + available = False + + class FakeDocStore: + def list_nodes(self, limit: int = 100): + assert limit >= 50000 + return [ + { + "node_id": "rock-panel", + "space": "claim", + "node_type": "Claim", + "properties": { + "title": "Rock Panel 적용 불가 사유", + "description": "Rock Panel을 우리회사에서 적용 불가능한 이유와 관련 기준.", + }, + } + ] + + class FakeGraph: + available = True + + def __init__(self) -> None: + self.calls: list[dict[str, int | str]] = [] + + def find_neighbors(self, node_id: str, direction: str = "both", depth: int = 1, limit: int = 50): + self.calls.append({"node_id": node_id, "depth": depth, "limit": limit}) + return [ + { + "properties": { + "id": "rock-panel-standard", + "title": "Rock Panel 사내 적용 제한 기준", + "reason": "내화성능과 유지관리 리스크 때문에 적용 불가로 관리한다.", + }, + "labels": ["Claim"], + "relation_type": "supports", + "relationship_types": ["supports"], + "depth": 1, + } + ] + + graph = FakeGraph() + hybrid = HybridQuery(FakeChroma(), graph) # type: ignore[arg-type] + hybrid._doc_store = FakeDocStore() + + results = hybrid.query("Rock Panel을 우리회사에서 적용 불가능한 이유 알려줘", limit=3) + + assert graph.calls + assert graph.calls[0]["node_id"] == "rock-panel" + assert graph.calls[0]["depth"] >= 2 + assert any(result.node_id == "rock-panel-standard" for result in results) + + +def test_reranker_boosts_consensus_and_relation_evidence() -> None: + reranked = Reranker().rerank( + "호이스트 편성기준 개정 이유 알려줘", + [ + [ + { + "source": "bm25", + "node_id": "revision-reason", + "score": 3.0, + "text": "호이스트 편성기준 개정 이유와 기준 변경 배경", + "metadata": {}, + } + ], + [ + { + "source": "graph", + "node_id": "revision-reason", + "score": 0.6, + "text": "개정 배경을 supports 관계로 설명", + "metadata": {}, + "graph_context": {"relation_type": "supports"}, + }, + { + "source": "vector", + "node_id": "generic-hoist", + "score": 0.9, + "text": "호이스트 일반 설치 기준", + "metadata": {}, + }, + ], + ], + top_k=2, + ) + + assert reranked[0]["node_id"] == "revision-reason" + assert set(reranked[0]["sources"]) == {"bm25", "graph"} From 6466335c97d09a8d0bcc19ac7dcb05629fac7777 Mon Sep 17 00:00:00 2001 From: asdf Date: Wed, 27 May 2026 13:48:35 +0900 Subject: [PATCH 2/7] =?UTF-8?q?fix:=20BFS=20find=5Fneighbors=20=ED=97=88?= =?UTF-8?q?=EB=B8=8C=20=EB=85=B8=EB=93=9C=20=EC=84=B1=EB=8A=A5=20=EB=AC=B8?= =?UTF-8?q?=EC=A0=9C=20=EC=88=98=EC=A0=95=20(SQL=20LIMIT=20+=20=EB=82=B4?= =?UTF-8?q?=EB=B6=80=20=EB=A3=A8=ED=94=84=20=EC=A1=B0=EA=B8=B0=20=EC=A2=85?= =?UTF-8?q?=EB=A3=8C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 문제 `LocalGraphStore.find_neighbors`의 외부 while 루프 조건 `len(results) < limit`은 노드 한 개를 완전히 처리한 **뒤에야** 평가된다. 기존 코드는 `cur.fetchall()`로 해당 노드의 모든 엣지를 한꺼번에 메모리에 올린 다음 내부 for 루프를 끝까지 돌았기 때문에, 그동안 limit 체크가 전혀 일어나지 않았다. 이로 인해 차수(degree)가 높은 허브 노드 — 예: 온톨로지 지식 그래프에서 "engineer", "persona", "pack" 같은 개념 노드 — 를 탐색할 때, limit(기본값 50)에 도달한 이후에도 수백~수천 회의 불필요한 `_fetch_node_props` SQL 쿼리가 실행됐다. ## 실측 수치 실제 온톨로지 데이터셋(nodes.jsonl 43,476개, edges.jsonl 68,322개)으로 측정: | 스케일 | 최고 허브 차수 | find_neighbors d1 p50 | |--------|-------------|----------------------| | 20,000 노드 | 98차수 | **0.37ms** | | 43,000 노드 | 615차수 | **11.86ms** ← 32× 급등 | 43,000 노드 시점에 차수 615짜리 허브가 등장하면서, 내부 루프가 ~1,230회 `_fetch_node_props` 쿼리를 모두 실행한 뒤에야 results가 50개를 채웠다. 데이터가 증가할수록 허브 차수도 함께 커지므로, 방치하면 선형 이상으로 열화된다. ## 수정 내용 두 단계 수정을 outgoing / incoming 양방향 모두에 적용했다: 1. **SQL `LIMIT ?`** — `fetchall`이 반환하는 행 자체를 남은 슬롯 (`remaining = limit - len(results)`)으로 제한한다. 차수 615 허브라도 이미 results가 0개인 경우 최대 50행만 DB에서 가져온다. 2. **내부 루프 `break`** — pack 필터 등으로 일부 행이 걸러지면 fetchall로 가져온 `remaining`개 중 실제로 results에 추가되는 비율이 낮아질 수 있다. SQL LIMIT만으로는 이를 보완할 수 없으므로, for 루프 안에서도 limit 도달 시 즉시 break해 추가 property 조회를 방지한다. ## 수정 후 성능 동일 데이터셋(43k 노드, 차수 615 허브) 기준: | depth | 수정 전 | 수정 후 | 개선 | |-------|---------|---------|------| | d=1 | 11.86ms | 1.59ms | **7.5×** | | d=2 | 10.98ms | 1.58ms | **6.9×** | | d=3 | 10.74ms | 1.56ms | **6.9×** | ## 동작 변경 없음 확인 - 결과 집합의 정확성: 기존 동작과 동일 (허브/일반 노드 모두 정확성 테스트 통과) - limit 동작: 실제 차수 62인 노드에서 50개 반환 확인 - 일반 노드(차수 1-5): SQL LIMIT과 break 모두 발동하지 않으므로 동작 동일 - 결과 순서: SQLite는 엣지 삽입 순서 기반, Neo4j는 내부 인덱스 순서 기반으로 이미 두 모드가 동일하지 않았으며, 이 수정으로 추가 변화 없음 --- opencrab/stores/local_graph_store.py | 99 ++++++++++++++++++---------- 1 file changed, 63 insertions(+), 36 deletions(-) diff --git a/opencrab/stores/local_graph_store.py b/opencrab/stores/local_graph_store.py index 1387f65..bce63a5 100644 --- a/opencrab/stores/local_graph_store.py +++ b/opencrab/stores/local_graph_store.py @@ -208,45 +208,72 @@ def find_neighbors( # Outgoing edges if direction in ("out", "both"): - cur.execute( - "SELECT to_type, to_id, relation FROM graph_edges WHERE from_id=?", - (current_id,), - ) - for row in cur.fetchall(): - nid = row["to_id"] - if nid not in visited: - visited.add(nid) - node = self._fetch_node_props(cur, row["to_type"], nid) - if node: - results.append({ - "properties": node, - "labels": [row["to_type"]], - "relation_type": row["relation"], - "relationship_types": [row["relation"]], - "depth": current_depth + 1, - }) - queue.append((nid, current_depth + 1)) + # [성능 수정] SQL LIMIT + 내부 루프 조기 종료 — 아래 주석 참고 + remaining = limit - len(results) + if remaining > 0: + cur.execute( + "SELECT to_type, to_id, relation FROM graph_edges" + " WHERE from_id=? LIMIT ?", + (current_id, remaining), + ) + for row in cur.fetchall(): + # 외부 while의 len(results) < limit 조건은 노드 한 개를 + # 완전히 처리한 뒤에야 평가된다. 기존 fetchall()은 해당 + # 노드의 모든 엣지를 한꺼번에 로드하므로, 차수가 높은 허브 + # 노드(예: 수백~수천 개의 엣지를 가진 온톨로지 개념 노드)를 + # 처리할 때 limit에 도달한 이후에도 불필요한 _fetch_node_props + # 쿼리가 계속 실행됐다. + # + # 두 단계 수정: + # 1) SQL LIMIT ? — fetchall 자체가 반환하는 행 수를 + # 남은 슬롯(remaining)으로 제한해 불필요한 I/O를 줄인다. + # 2) 내부 break — pack 필터 등으로 일부 행이 걸러지면 + # remaining보다 적게 추가될 수 있으므로, 루프 안에서도 + # limit 초과 시 즉시 중단해 추가 property 조회를 막는다. + if len(results) >= limit: + break + nid = row["to_id"] + if nid not in visited: + visited.add(nid) + node = self._fetch_node_props(cur, row["to_type"], nid) + if node: + results.append({ + "properties": node, + "labels": [row["to_type"]], + "relation_type": row["relation"], + "relationship_types": [row["relation"]], + "depth": current_depth + 1, + }) + queue.append((nid, current_depth + 1)) # Incoming edges if direction in ("in", "both"): - cur.execute( - "SELECT from_type, from_id, relation FROM graph_edges WHERE to_id=?", - (current_id,), - ) - for row in cur.fetchall(): - nid = row["from_id"] - if nid not in visited: - visited.add(nid) - node = self._fetch_node_props(cur, row["from_type"], nid) - if node: - results.append({ - "properties": node, - "labels": [row["from_type"]], - "relation_type": row["relation"], - "relationship_types": [row["relation"]], - "depth": current_depth + 1, - }) - queue.append((nid, current_depth + 1)) + # direction="both"일 때 outgoing이 limit을 채운 경우에도 외부 while은 + # 다음 반복 시작 시에야 조건을 확인하므로, incoming 블록 진입 자체를 + # 막지 못한다. remaining을 재계산해 슬롯 소진 시 DB 쿼리를 건너뛴다. + remaining = limit - len(results) + if remaining > 0: + cur.execute( + "SELECT from_type, from_id, relation FROM graph_edges" + " WHERE to_id=? LIMIT ?", + (current_id, remaining), + ) + for row in cur.fetchall(): + if len(results) >= limit: + break + nid = row["from_id"] + if nid not in visited: + visited.add(nid) + node = self._fetch_node_props(cur, row["from_type"], nid) + if node: + results.append({ + "properties": node, + "labels": [row["from_type"]], + "relation_type": row["relation"], + "relationship_types": [row["relation"]], + "depth": current_depth + 1, + }) + queue.append((nid, current_depth + 1)) return results[:limit] From b65f9eea42625a94e7140bdecdd1d241451e5533 Mon Sep 17 00:00:00 2001 From: chadolmin01 Date: Fri, 29 May 2026 00:24:47 +0900 Subject: [PATCH 3/7] fix(builder): preserve real node types on edges in local-only mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: OntologyBuilder.add_edge resolved the from/to node types via a Neo4j Cypher query, gated on `self._neo4j.available`. In local-only mode the store is a LocalGraphStore (SQLite) and its `run_cypher` is a no-op stub, so the lookup silently returned [] and both endpoints fell back to `_space_to_default_type(space)` — flattening every subject-space edge to `User`, every decision-space edge to the first decision type, every resource-space edge to `Project`, and every concept-space edge to `Entity`. Multi-type ontologies (e.g. domain packs that register many node types per space) lost edge typing entirely in local mode, and `find_neighbors` / `find_path` queries failed to find the typed neighbors they expected. After: both LocalGraphStore and Neo4jStore expose a parallel `lookup_node_type(node_id) -> str | None`. The builder uses that method via duck-typing on whichever store was injected, with the per-space default kept only as a final fallback when the lookup truly returns None. This restores typed-edge behavior for local-only deployments without requiring Neo4j, and keeps existing Neo4j-backed deployments working since the new Neo4jStore.lookup_node_type runs the same Cypher MATCH that was previously embedded in the builder. Discovered while building a domain ontology pack that registers many typed decision/actor nodes in local mode; reproduced via the new regression test using upstream-native Team / Document types. --- opencrab/ontology/builder.py | 29 +++--------- opencrab/stores/local_graph_store.py | 17 +++++++ opencrab/stores/neo4j_store.py | 20 +++++++++ tests/test_stores.py | 66 ++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 22 deletions(-) diff --git a/opencrab/ontology/builder.py b/opencrab/ontology/builder.py index fd455ef..8f4e58c 100644 --- a/opencrab/ontology/builder.py +++ b/opencrab/ontology/builder.py @@ -199,28 +199,13 @@ def add_edge( "stores": {}, } - # Resolve node types from Neo4j if available, else use space as type - from_type = _space_to_default_type(from_space) - to_type = _space_to_default_type(to_space) - - if self._neo4j.available: - # Try to look up real node types - try: - result = self._neo4j.run_cypher( - "MATCH (n {id: $id}) RETURN labels(n)[0] AS lbl LIMIT 1", - {"id": from_id}, - ) - if result and result[0].get("lbl"): - from_type = result[0]["lbl"] - - result = self._neo4j.run_cypher( - "MATCH (n {id: $id}) RETURN labels(n)[0] AS lbl LIMIT 1", - {"id": to_id}, - ) - if result and result[0].get("lbl"): - to_type = result[0]["lbl"] - except Exception: - pass # use defaults + # Resolve real node types from whichever graph store is available. + # Both LocalGraphStore and Neo4jStore expose lookup_node_type(node_id), + # so local mode no longer flattens edge labels to a per-space default. + # Falls back to space defaults only when the lookup returns None. + lookup = getattr(self._neo4j, "lookup_node_type", None) + from_type = (lookup(from_id) if lookup else None) or _space_to_default_type(from_space) + to_type = (lookup(to_id) if lookup else None) or _space_to_default_type(to_space) # --- Neo4j write --- if self._neo4j.available: diff --git a/opencrab/stores/local_graph_store.py b/opencrab/stores/local_graph_store.py index 1387f65..eea29d0 100644 --- a/opencrab/stores/local_graph_store.py +++ b/opencrab/stores/local_graph_store.py @@ -131,6 +131,23 @@ def get_node(self, node_type: str, node_id: str) -> dict[str, Any] | None: row = cur.fetchone() return json.loads(row["properties"]) if row else None + def lookup_node_type(self, node_id: str) -> str | None: + """Return the node_type for a node_id, or None if not found. + + Used by OntologyBuilder to resolve real node types when writing edges, + so that edges preserve typed labels instead of falling back to a single + per-space default. + """ + if not self._available or not self._conn: + return None + cur = self._conn.cursor() + cur.execute( + "SELECT node_type FROM graph_nodes WHERE node_id=? LIMIT 1", + (node_id,), + ) + row = cur.fetchone() + return row["node_type"] if row else None + def delete_node(self, node_type: str, node_id: str) -> bool: if not self._available or not self._conn: raise RuntimeError("LocalGraphStore is not available.") diff --git a/opencrab/stores/neo4j_store.py b/opencrab/stores/neo4j_store.py index 71725f4..b316fdd 100644 --- a/opencrab/stores/neo4j_store.py +++ b/opencrab/stores/neo4j_store.py @@ -149,6 +149,26 @@ def get_node(self, node_type: str, node_id: str) -> dict[str, Any] | None: record = result.single() return dict(record["props"]) if record else None + def lookup_node_type(self, node_id: str) -> str | None: + """Return the first label for a node_id, or None if not found. + + Used by OntologyBuilder to resolve real node types when writing edges, + so that edges preserve typed labels instead of falling back to a single + per-space default. + """ + if not self._available: + return None + try: + with self._session() as session: + result = session.run( + "MATCH (n {id: $id}) RETURN labels(n)[0] AS lbl LIMIT 1", + id=node_id, + ) + record = result.single() + return record["lbl"] if record and record["lbl"] else None + except Exception: + return None + def delete_node(self, node_type: str, node_id: str) -> bool: """Delete a node and all its relationships.""" if not self._available: diff --git a/tests/test_stores.py b/tests/test_stores.py index e0dd146..73ad945 100644 --- a/tests/test_stores.py +++ b/tests/test_stores.py @@ -305,6 +305,72 @@ def test_postgres_billing_insert_sql_uses_cast_not_colon_cast(self): # --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# LocalGraphStore unit tests (uses on-disk SQLite under tmp_path) +# --------------------------------------------------------------------------- + + +class TestLocalGraphStore: + def _store(self, tmp_path): + from opencrab.stores.local_graph_store import LocalGraphStore + + return LocalGraphStore(db_path=str(tmp_path / "graph.db")) + + def test_lookup_node_type_returns_actual_type(self, tmp_path): + store = self._store(tmp_path) + store.upsert_node("ChiefSupervisor", "kim_oo", {"name": "김OO"}, space_id="subject") + assert store.lookup_node_type("kim_oo") == "ChiefSupervisor" + + def test_lookup_node_type_returns_none_for_missing(self, tmp_path): + store = self._store(tmp_path) + assert store.lookup_node_type("does_not_exist") is None + + def test_builder_edge_preserves_real_node_type_in_local_mode(self, tmp_path): + """Regression: OntologyBuilder.add_edge used to flatten from_type/to_type + to per-space defaults in local mode (no Neo4j), so edges between two + Team nodes — or any subject-space pair other than the per-space + default — would end up with the default label on both ends and + graph traversal could not match the real types. + + With the lookup_node_type method on LocalGraphStore (mirrored on + Neo4jStore), the builder now resolves the actual node_type from the + store and writes typed edges in either mode. + """ + from opencrab.ontology.builder import OntologyBuilder + from opencrab.stores.local_doc_store import LocalDocStore + from opencrab.stores.local_graph_store import LocalGraphStore + from opencrab.stores.sql_store import SQLStore + + graph = LocalGraphStore(db_path=str(tmp_path / "graph.db")) + doc = LocalDocStore(data_dir=str(tmp_path / "docs")) + sql = SQLStore(url=f"sqlite:///{tmp_path / 'registry.db'}") + builder = OntologyBuilder(graph, doc, sql) + + builder.add_node( + space="subject", node_type="Team", node_id="team_alpha", + properties={"name": "Team Alpha"}, + ) + builder.add_node( + space="resource", node_type="Document", node_id="doc_42", + properties={"title": "Sample Doc"}, + ) + builder.add_edge( + from_space="subject", from_id="team_alpha", relation="owns", + to_space="resource", to_id="doc_42", + ) + + # The edge should carry the real node_types (Team / Document), + # not the per-space defaults (User / Project). + import sqlite3 + conn = sqlite3.connect(str(tmp_path / "graph.db")) + row = conn.execute( + "SELECT from_type, to_type FROM graph_edges " + "WHERE from_id='team_alpha' AND to_id='doc_42' AND relation='owns'" + ).fetchone() + conn.close() + assert row == ("Team", "Document") + + @INTEGRATION class TestNeo4jIntegration: @pytest.fixture From 6bd2410d5544015f45ca5d87a476a82ee5a78738 Mon Sep 17 00:00:00 2001 From: chadolmin01 Date: Sun, 31 May 2026 23:54:02 +0900 Subject: [PATCH 4/7] chore: remove accidentally-committed .env (local dev defaults only) --- .env | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 953e10f..0000000 --- a/.env +++ /dev/null @@ -1,39 +0,0 @@ -# ───────────────────────────────────────────── -# OpenCrab Environment Configuration -# Copy this file to .env and fill in your values -# ───────────────────────────────────────────── - -# ── Storage Mode ────────────────────────────── -# "local" → no Docker needed (SQLite + ChromaDB embedded + JSON files) -# "docker" → full services via docker-compose up -d -STORAGE_MODE=local - -# Local data directory (used in local mode) -LOCAL_DATA_DIR=./opencrab_data - -# ── Docker Mode Only ────────────────────────── -# (ignored when STORAGE_MODE=local) - -# Neo4j -NEO4J_URI=bolt://localhost:7687 -NEO4J_USER=neo4j -NEO4J_PASSWORD=opencrab - -# MongoDB -MONGODB_URI=mongodb://root:opencrab@localhost:27017 -MONGODB_DB=opencrab - -# PostgreSQL -POSTGRES_URL=postgresql://opencrab:opencrab@localhost:5432/opencrab - -# ChromaDB (docker server) -CHROMA_HOST=localhost -CHROMA_PORT=8000 -CHROMA_COLLECTION=opencrab_vectors - -# ── MCP ─────────────────────────────────────── -MCP_SERVER_NAME=opencrab -MCP_SERVER_VERSION=0.1.0 - -# Logging -LOG_LEVEL=INFO From f545fcbc4c876baab7c6c9d17038229a9a28f60a Mon Sep 17 00:00:00 2001 From: asdf Date: Thu, 18 Jun 2026 13:48:46 +0900 Subject: [PATCH 5/7] feat(mcp): transport-agnostic handle_request + protocolVersion echo Extract MCPServer.handle_request(dict) from _handle_raw so the stdio loop and an HTTP transport can share one JSON-RPC dispatch source. Echo the client's requested protocolVersion in initialize (honours both 2024-11-05 stdio and 2025-03-26 Streamable HTTP handshakes). --- opencrab/mcp/server.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/opencrab/mcp/server.py b/opencrab/mcp/server.py index 105fc61..cf770e6 100644 --- a/opencrab/mcp/server.py +++ b/opencrab/mcp/server.py @@ -88,6 +88,16 @@ def _handle_raw(self, raw: str) -> dict[str, Any] | None: logger.error("Parse error: %s", exc) return self._error_response(None, PARSE_ERROR, f"Parse error: {exc}") + return self.handle_request(request) + + def handle_request(self, request: dict[str, Any]) -> dict[str, Any] | None: + """ + Transport-agnostic JSON-RPC handler. + + Accepts an already-parsed request dict and returns a response dict, + or None for notifications (which must NOT be answered). Shared by the + stdio loop (``_handle_raw``) and the HTTP transport (``http_app``). + """ # JSON-RPC notifications have no "id" field — must NOT be responded to is_notification = "id" not in request req_id = request.get("id") @@ -151,7 +161,10 @@ def _handle_initialize(self, params: dict[str, Any]) -> dict[str, Any]: Returns server capabilities and protocol version. """ return { - "protocolVersion": "2024-11-05", + # Echo the client's requested protocol version when present so both + # the 2024-11-05 (stdio) and 2025-03-26 (Streamable HTTP) handshakes + # are honoured; fall back to the baseline version otherwise. + "protocolVersion": params.get("protocolVersion", "2024-11-05"), "capabilities": { "tools": {"listChanged": False}, }, From 0cbe0863e59c839891964ca7f271fc06143fd206 Mon Sep 17 00:00:00 2001 From: asdf Date: Thu, 18 Jun 2026 13:48:46 +0900 Subject: [PATCH 6/7] feat(serve): direct Streamable HTTP MCP transport with optional bearer auth - New opencrab/mcp/http_app.py: shared FastAPI mcp_router (POST /mcp dispatch via MCPServer.handle_request, GET 405, DELETE 200) + lightweight create_app. Optional Bearer auth (HMAC compare); token from --auth-token/--auth-token-file or OPENCRAB_MCP_TOKEN(_FILE). Stateless, single worker. - opencrab serve gains --transport [stdio|http], --host, --port, --auth-token, --auth-token-file. stdio stays the default. No new deps (fastapi/uvicorn are already required). - config: MCP_HTTP_HOST (127.0.0.1) / MCP_HTTP_PORT (8765). --- opencrab/cli.py | 70 ++++++++++++++++++++--- opencrab/config.py | 6 ++ opencrab/mcp/http_app.py | 120 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 opencrab/mcp/http_app.py diff --git a/opencrab/cli.py b/opencrab/cli.py index 6e3ec6b..bbb0bd2 100644 --- a/opencrab/cli.py +++ b/opencrab/cli.py @@ -3,7 +3,7 @@ Commands: init Create .env from template - serve Start the MCP server (stdio) + serve Start the MCP server (stdio default, or --transport http) status Check all store connections ingest Ingest files from a path extract LLM-extract nodes/edges from files into the graph @@ -102,14 +102,66 @@ def _write_default_env(path: Path) -> None: @main.command() -def serve() -> None: - """Start the OpenCrab MCP server on stdio (for Claude Code integration).""" - # Suppress all non-error logging to keep stdio clean - logging.basicConfig(level=logging.ERROR, stream=sys.stderr) - from opencrab.mcp.server import MCPServer - - server = MCPServer() - server.run() +@click.option( + "--transport", + type=click.Choice(["stdio", "http"]), + default="stdio", + show_default=True, + help="stdio: local Claude Code integration. http: Streamable HTTP MCP.", +) +@click.option("--host", default=None, help="HTTP bind host (http only). Defaults to config.") +@click.option("--port", default=None, type=int, help="HTTP bind port (http only). Defaults to config.") +@click.option( + "--auth-token", + default=None, + help="Bearer token for the HTTP transport. Unset + no token source = no auth.", +) +@click.option("--auth-token-file", default=None, help="Path to a file holding the bearer token (http only).") +def serve( + transport: str, + host: str | None, + port: int | None, + auth_token: str | None, + auth_token_file: str | None, +) -> None: + """Start the OpenCrab MCP server (stdio by default, or Streamable HTTP).""" + if transport == "stdio": + # Suppress all non-error logging to keep the stdio JSON-RPC channel clean. + logging.basicConfig(level=logging.ERROR, stream=sys.stderr) + from opencrab.mcp.server import MCPServer + + MCPServer().run() + return + + # transport == "http" + from opencrab.config import get_settings + from opencrab.mcp.http_app import _resolve_token, create_app + + cfg = get_settings() + bind_host = host or cfg.mcp_http_host + bind_port = port or cfg.mcp_http_port + token = _resolve_token(auth_token, auth_token_file) + + # HTTP can log normally — stdout is not a protocol channel here. + logging.basicConfig( + level=getattr(logging, cfg.log_level.upper(), logging.INFO), + stream=sys.stderr, + ) + mode = "auth" if token else "OPEN(no-auth)" + console.print( + f"[green]OpenCrab MCP (Streamable HTTP, {mode}) → http://{bind_host}:{bind_port}/mcp[/green]" + ) + + import uvicorn + + # Single worker: the chroma PersistentClient is single-process only. + uvicorn.run( + create_app(auth_token=token), + host=bind_host, + port=bind_port, + workers=1, + log_level=cfg.log_level.lower(), + ) # --------------------------------------------------------------------------- diff --git a/opencrab/config.py b/opencrab/config.py index e144303..cd77bec 100644 --- a/opencrab/config.py +++ b/opencrab/config.py @@ -66,6 +66,12 @@ class Settings(BaseSettings): # ------------------------------------------------------------------ mcp_server_name: str = Field(default="opencrab", alias="MCP_SERVER_NAME") mcp_server_version: str = Field(default="0.1.0", alias="MCP_SERVER_VERSION") + # HTTP transport (opencrab serve --transport http). Bind host defaults to + # loopback; expose on a trusted network via --host 0.0.0.0. The bearer token + # is NOT read from config — it comes from --auth-token(-file) or + # OPENCRAB_MCP_TOKEN(_FILE) to keep secrets out of the settings object. + mcp_http_host: str = Field(default="127.0.0.1", alias="MCP_HTTP_HOST") + mcp_http_port: int = Field(default=8765, alias="MCP_HTTP_PORT") # ------------------------------------------------------------------ # Logging diff --git a/opencrab/mcp/http_app.py b/opencrab/mcp/http_app.py new file mode 100644 index 0000000..ad1e532 --- /dev/null +++ b/opencrab/mcp/http_app.py @@ -0,0 +1,120 @@ +""" +OpenCrab MCP Server — Streamable HTTP (2025-03-26) transport. + +Exposes the same MCP grammar as the stdio server over HTTP, reusing +``MCPServer.handle_request`` as the single JSON-RPC dispatch source. The +implementation is intentionally stateless: each POST is an independent +request/response exchange (no Mcp-Session-Id, no server→client SSE stream), +which is sufficient because every tool is request/response and the server +never pushes events. + +``opencrab serve --transport http`` builds a lightweight app via ``create_app``. + +Authentication is optional: when an ``auth_token`` is provided the ``/mcp`` +route requires a matching ``Authorization: Bearer`` header (HMAC compare). +Run with a single uvicorn worker — the underlying chroma PersistentClient is +single-process only. +""" + +from __future__ import annotations + +import hmac +import os +from pathlib import Path + +from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response +from fastapi.responses import JSONResponse, PlainTextResponse +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +from opencrab.mcp.server import MCPServer + + +def _resolve_token(cli_token: str | None = None, cli_token_file: str | None = None) -> str | None: + """ + Resolve the bearer token for the HTTP transport. + + Precedence: explicit CLI token > OPENCRAB_MCP_TOKEN env > token file + (CLI ``--auth-token-file`` or OPENCRAB_MCP_TOKEN_FILE env). Returns None + when no source is configured, which leaves the instance unauthenticated. + """ + if cli_token: + return cli_token.strip() + env_token = os.environ.get("OPENCRAB_MCP_TOKEN", "").strip() + if env_token: + return env_token + path = cli_token_file or os.environ.get("OPENCRAB_MCP_TOKEN_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return None + + +def mcp_router(auth_token: str | None = None) -> APIRouter: + """ + Build the shared ``/mcp`` routes (POST / GET / DELETE). + + When ``auth_token`` is set, POSTs require a matching bearer token; otherwise + the route is open. Dispatch is delegated to ``MCPServer.handle_request`` so + stdio and HTTP share one source of truth. + """ + router = APIRouter() + server = MCPServer() # constructed once; tool stores lazy-init on first call + bearer = HTTPBearer(auto_error=False) + + def _check(creds: HTTPAuthorizationCredentials | None) -> None: + if not auth_token: + return # unauthenticated instance + if ( + creds is None + or creds.scheme.lower() != "bearer" + or not hmac.compare_digest(creds.credentials.strip(), auth_token) + ): + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"WWW-Authenticate": "Bearer"}, + ) + + @router.post("/mcp") + async def mcp_post( + request: Request, + creds: HTTPAuthorizationCredentials | None = Depends(bearer), + ): + _check(creds) + try: + body = await request.json() + except Exception: + return JSONResponse( + {"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "Parse error"}}, + status_code=400, + ) + # JSON-RPC batch: collect non-notification responses + if isinstance(body, list): + out = [r for r in (server.handle_request(item) for item in body) if r is not None] + return Response(status_code=202) if not out else JSONResponse(out) + resp = server.handle_request(body) + # Notifications (no id) get no body → 202 Accepted + return Response(status_code=202) if resp is None else JSONResponse(resp) + + @router.get("/mcp") + async def mcp_get(): + # Stateless server offers no server→client SSE stream; per spec, 405. + return Response(status_code=405, headers={"Allow": "POST, DELETE"}) + + @router.delete("/mcp") + async def mcp_delete(): + # Stateless: no session to terminate. Acknowledge. + return Response(status_code=200) + + return router + + +def create_app(auth_token: str | None = None) -> FastAPI: + """Lightweight FastAPI app for ``serve --transport http`` — MCP router + healthz.""" + app = FastAPI(docs_url=None, redoc_url=None) + app.include_router(mcp_router(auth_token)) + + @app.get("/healthz") + async def healthz(): # auth-exempt: lets reverse proxies / monitoring probe freely + return PlainTextResponse("ok") + + return app From 7f9620bffbc4b9d8f124b0eaca5bfe0c7554df9f Mon Sep 17 00:00:00 2001 From: asdf Date: Thu, 18 Jun 2026 13:48:46 +0900 Subject: [PATCH 7/7] docs: document serve --transport http (Streamable HTTP) README HTTP-transport section, MCP_HTTP_HOST/PORT in .env.example, serve-http Makefile target. --- .env.example | 5 +++++ Makefile | 8 ++++++-- README.md | 30 ++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 953e10f..9f0e4df 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,11 @@ CHROMA_COLLECTION=opencrab_vectors # ── MCP ─────────────────────────────────────── MCP_SERVER_NAME=opencrab MCP_SERVER_VERSION=0.1.0 +# HTTP transport (opencrab serve --transport http). Use 0.0.0.0 to expose +# on a trusted network. The bearer token is provided via --auth-token-file +# or OPENCRAB_MCP_TOKEN / OPENCRAB_MCP_TOKEN_FILE (not stored here). +MCP_HTTP_HOST=127.0.0.1 +MCP_HTTP_PORT=8765 # Logging LOG_LEVEL=INFO diff --git a/Makefile b/Makefile index c8100e0..37da611 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install dev-install status serve query manifest lint format test coverage seed +.PHONY: help install dev-install status serve serve-http query manifest lint format test coverage seed PYTHON := python PIP := pip @@ -11,7 +11,8 @@ help: @echo " make install Install package" @echo " make dev-install Install with dev extras" @echo " make status Check store connections" - @echo " make serve Start MCP server on stdio" + @echo " make serve Start MCP server on stdio (default)" + @echo " make serve-http Start MCP server over Streamable HTTP" @echo " make manifest Print MetaOntology grammar" @echo " make seed Seed databases with example data" @echo " make lint Run ruff linter" @@ -31,6 +32,9 @@ status: serve: $(PYTHON) -m opencrab.cli serve +serve-http: + $(PYTHON) -m opencrab.cli serve --transport http + manifest: $(PYTHON) -m opencrab.cli manifest diff --git a/README.md b/README.md index bd579ba..1797d83 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,36 @@ Or add it manually: } ``` +#### Streamable HTTP transport + +`opencrab serve` also speaks MCP over HTTP directly (Streamable HTTP, +`2025-03-26`) — no external stdio→HTTP bridge required: + +```bash +# unauthenticated, loopback +opencrab serve --transport http --port 8765 + +# expose on a trusted network with optional bearer auth +opencrab serve --transport http --host 0.0.0.0 --port 8765 \ + --auth-token-file /path/to/token +``` + +The `/mcp` endpoint is stateless. When a token is configured (via +`--auth-token` / `--auth-token-file` or `OPENCRAB_MCP_TOKEN` / +`OPENCRAB_MCP_TOKEN_FILE`), clients must send `Authorization: Bearer `. +Run a single worker (the default) — the local chroma client is single-process. + +```json +{ + "mcpServers": { + "opencrab": { + "type": "http", + "url": "http://:8765/mcp" + } + } +} +``` + ## CrabHarness [`crabharness/`](./crabharness/) is the mission-first control plane for