From f39f418b397269cb9725db6fc78db76701ba92da Mon Sep 17 00:00:00 2001 From: Eysn0130 Date: Thu, 16 Jul 2026 01:11:34 +0800 Subject: [PATCH] feat(library): separate official and private legal sources --- .agents/plugins/marketplace.json | 4 +- AGENTS.md | 18 +- README.md | 144 ++- README_EN.md | 145 ++- benchmarks/core-v3-candidate-2026-07-16.json | 58 ++ catalogs/deeplaw-official-cn.json | 305 ++++++ .../corpus-release-manifest.v2.schema.json | 7 +- contracts/law-support.input.v2.schema.json | 267 ++++++ contracts/official-catalog.v1.schema.json | 82 ++ .../private-library-state.v1.schema.json | 74 ++ docs/AGENT_ADAPTERS.md | 44 +- docs/ANALYTIX_INTEGRATION.md | 33 +- docs/ARCHITECTURE.md | 131 ++- docs/BENCHMARKS.md | 27 +- docs/CORPUS_GOVERNANCE.md | 50 +- docs/DEEPLAW_2.md | 43 +- evals/core-2026-07-14.jsonl | 12 +- plugins/deeplaw/.claude-plugin/plugin.json | 4 +- plugins/deeplaw/.codex-plugin/plugin.json | 6 +- .../skills/research-chinese-law/SKILL.md | 36 +- pyproject.toml | 6 +- src/deeplaw/__init__.py | 2 +- src/deeplaw/admin_lock.py | 71 ++ src/deeplaw/cli.py | 182 +++- src/deeplaw/evaluate.py | 38 +- src/deeplaw/extract.py | 33 + src/deeplaw/ingest.py | 76 +- src/deeplaw/mcp_server.py | 143 ++- src/deeplaw/official.py | 892 ++++++++++++++++++ src/deeplaw/private_library.py | 560 +++++++++++ src/deeplaw/search.py | 100 +- src/deeplaw/store.py | 43 +- tests/test_library_scopes.py | 603 ++++++++++++ tests/test_search_execution.py | 97 +- uv.lock | 2 +- 35 files changed, 4092 insertions(+), 246 deletions(-) create mode 100644 benchmarks/core-v3-candidate-2026-07-16.json create mode 100644 catalogs/deeplaw-official-cn.json create mode 100644 contracts/law-support.input.v2.schema.json create mode 100644 contracts/official-catalog.v1.schema.json create mode 100644 contracts/private-library-state.v1.schema.json create mode 100644 src/deeplaw/admin_lock.py create mode 100644 src/deeplaw/official.py create mode 100644 src/deeplaw/private_library.py create mode 100644 tests/test_library_scopes.py diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 19c1e78..ac57762 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -1,7 +1,7 @@ { - "name": "personal", + "name": "deeplaw", "interface": { - "displayName": "Personal" + "displayName": "DeepLaw" }, "plugins": [ { diff --git a/AGENTS.md b/AGENTS.md index ca51dbe..80fdaa3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,9 @@ # DeepLaw Agent Guide -DeepLaw is a read-only, version-aware Agent Knowledge Base for Chinese legal -sources, used by Codex, Claude Code, OpenCode, and future Analytix integration. +DeepLaw is a version-aware Agent Knowledge Base for Chinese legal sources, used +by Codex, Claude Code, OpenCode, and future Analytix integration. Its Agent/MCP +surface is read-only; offline CLI administration owns official updates and +per-OS-user private legal-reference imports. ## Source Of Truth @@ -9,6 +11,9 @@ sources, used by Codex, Claude Code, OpenCode, and future Analytix integration. dependency lockfile. - Legal text is authoritative only when it belongs to an immutable release and retains its official source URL, source SHA-256, locator, and release ID. +- User-private legal references are never authoritative DeepLaw sources. Keep + them under the owner-only private root, mark them unverified, and never merge + their ranking, receipts, or lifecycle with the official catalog. - Generated topic pages, summaries, tags, graphs, embeddings, model output, and search rankings are derived data. They never replace source text or determine legal validity. @@ -17,15 +22,18 @@ sources, used by Codex, Claude Code, OpenCode, and future Analytix integration. ## Safety Boundaries - Keep the MCP surface read-only. Do not add corpus, memory, or case write tools. -- Never mix case-private documents, facts, chats, or identifiers into a public - DeepLaw release, cache, log, benchmark, or query corpus. +- Never mix case-private documents, facts, chats, or identifiers into either + DeepLaw scope, cache, log, benchmark, or query corpus. Analytix case projects + remain outside DeepLaw. - Do not claim that a retrieved rule applies to a case merely because its effective date matches. Temporal applicability can require legal review. - Do not silently fall back to model memory or web search when a release is missing or verification fails. - Keep provider-visible output bounded. Search returns at most five evidence cards; full text is fetched by exact segment ID. -- Runtime database access is SQLite read-only and immutable. +- Runtime database access is SQLite read-only and immutable. User-private add + and delete operations remain local CLI administration and must not become MCP + write tools. ## Engineering Discipline diff --git a/README.md b/README.md index 8e3873f..a144c1b 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,12 @@

- Architecture 2.0 是目标架构 · 当前可运行版本为 0.2.0 alpha + Architecture 2.0 是目标架构 · 当前可运行版本为 0.3.0 alpha

CI - Runtime 0.2.0 alpha + Runtime 0.3.0 alpha Python 3.11 and 3.13 Read-only MCP Apache 2.0 @@ -40,12 +40,12 @@ DeepLaw Architecture 2.0 目标架构：文件进入知识库，经定位、连接和解释形成证据包并交给 Agent

-上图是 Architecture 2.0 目标，不是当前运行截图。当前 `0.2.0` 能把 DOCX 和 PDF 构建成 +上图是 Architecture 2.0 目标，不是当前运行截图。当前 `0.3.0` 能把 DOCX 和 PDF 构建成 只读、版本化、可追溯的 Agent Knowledge Base，在知识库内部完成有界定位、连接和核验，再向 Agent 交付小型 **Evidence Pack**：主证据、不确定证据、显式缺口和可复核回执。2.0 将补齐 -source-bound Explain、TXT 输入和最小充分证据选择。 +source-bound Explain、官方治理的 TXT 输入和最小充分证据选择。 -`DeepLaw` 是产品名，Architecture 2.0 是下一代架构方向；当前可运行包仍为 `0.2.0` alpha。 +`DeepLaw` 是产品名，Architecture 2.0 是下一代架构方向；当前可运行包仍为 `0.3.0` alpha。 主页不会把研究目标伪装成已完成能力。 ## 快速开始 @@ -55,11 +55,39 @@ source-bound Explain、TXT 输入和最小充分证据选择。 ```bash git clone https://github.com/Eysn0130/DeepLaw.git cd DeepLaw -uv sync --extra dev -uv run deeplaw --version +uv tool install . +deeplaw --version ``` -构建操作者自己合法取得、且有权处理并保留的 source package： +`uv tool install .` 把 `deeplaw` 放入 Agent 宿主可见的用户级命令环境；仅参与源码开发时再运行 +`uv sync --extra dev`。代码升级后使用 `uv tool install --force .` 更新运行时。 + +安装随当前版本发布的 DeepLaw 官方团队目录。目录记录来源 URL、字节数和 SHA-256；客户端从 +原官方下载地址取得 DOCX/PDF 并在本地构建，不由 GitHub 仓库重新分发原件： + +```bash +deeplaw official install +deeplaw official status +deeplaw doctor +``` + +已经持有与目录完全一致的 source package 时，可跳过网络下载并复用本地原件： + +```bash +deeplaw official install --source-root "/path/to/legal-source-package" +``` + +团队发布递增目录后，用户显式更新。正在执行中的一次读取仍固定旧 release；后续读取检测到 +官方 epoch 已变化会失败关闭，重启 Agent/MCP 后才读取新 release： + +```bash +deeplaw official update +``` + +官方下载遇到 `429/5xx` 或超时时会执行有上限的指数退避；已经通过大小与 SHA-256 校验的原件 +保留在本机缓存，重新运行可继续构建。DeepLaw 不会为追求成功率静默切换到非官方镜像。 + +也可以继续构建操作者自己合法取得、且有权处理并保留的 source package： ```bash export DEEPLAW_SOURCE_ROOT="/path/to/legal-source-package" @@ -75,8 +103,29 @@ uv run deeplaw doctor uv run deeplaw search --query "刑法第二百六十六条" --as-of 2024-07-01 ``` -仓库不分发受限法源、案件材料或生成的 SQLite release。`build` 成功只表示产物通过了机器 -闸门，不表示资料已经完成人工法律审核。 +仓库不分发受限法源、案件材料或生成的 SQLite release。`official install` / `build` 成功只 +表示来源字节和机器闸门通过，不表示资料已经完成人工法律审核。 + +官方团队目录是可选能力。停用不改写 release，卸载只删除它登记的本机 official release、 +目录快照和下载缓存，不触碰用户私有库： + +```bash +deeplaw official disable +deeplaw official enable +deeplaw official uninstall +``` + +用户自己的法律参考资料进入独立的本机私有库。导入要求人工确认它不是 Analytix 案件材料； +Agent 只能通过只读 `private_*` 操作检索，不能通过 MCP 上传或删除： + +```bash +deeplaw private add \ + --source "/path/to/user-legal-reference.docx" \ + --confirm-no-case-data +deeplaw private list +deeplaw private search --query "资料题名 第一条" +deeplaw private delete --document-id "doc_..." +``` ## DeepLaw 如何工作 @@ -84,12 +133,12 @@ uv run deeplaw search --query "刑法第二百六十六条" --as-of 2024-07-01 DeepLaw Architecture 2.0 目标知识闭环：Ingest、Organize、Locate、Connect、Explain、Verify 围绕 Evidence Core，并交付 Evidence、Gaps 与 Receipts

-Architecture 2.0 用六个核心知识动作定义从文件到 Agent 的闭环；下表同时标明 `0.2.0` 与 +Architecture 2.0 用六个核心知识动作定义从文件到 Agent 的闭环；下表同时标明 `0.3.0` 与 2.0 目标的边界： | 动作 | 作用 | 当前状态与边界 | | --- | --- | --- | -| **Ingest** | 校验文件、解析内容、保留页码/段落/hash | 当前支持 DOCX/PDF；处理成功不等于人工审核通过 | +| **Ingest** | 校验文件、解析内容、保留页码/段落/hash | 官方目录支持 DOCX/PDF；用户私有库另支持 UTF-8 TXT；处理成功不等于人工审核通过 | | **Organize** | 构建层级、版本、关系和 Knowledge Map | 当前保留标题/条文分段和顺序；完整法律层级属于 2.0 目标 | | **Locate** | 精确定位题名、文号、条款、关键词和相关片段 | 泛词不会展开成无限候选 | | **Connect** | 连接引用、修订、废止、替代、实施和例外关系 | 当前仅 provenance-bound 单跳文档关系；定义、范围和 challenge closure 属于 2.0 目标 | @@ -107,7 +156,7 @@ Architecture 2.0 用六个核心知识动作定义从文件到 Agent 的闭环 DeepLaw Architecture 2.0 目标 Evidence Core：来源与版本、知识地图、证据义务、限制与缺口、回执与重放

-Evidence Core 是 2.0 目标架构。`0.2.0` 已实现其中的不可变来源、基础 Knowledge Map、 +Evidence Core 是 2.0 目标架构。`0.3.0` 已实现其中的不可变来源、基础 Knowledge Map、 Evidence Duties、有界结果、gaps 与 receipts；coverage witness、挑战结果和 replay trace 尚未 实现。 @@ -178,35 +227,46 @@ temporal、extraction 和 provenance 是正交能力；只有确定性校验或 这是 2.0 目标。系统主动检查时效变化、例外、定义、范围、交叉引用、抽取风险和冲突。每项只能是 `satisfied`、`unresolved` 或 `not_applicable`；`unresolved` 必须保留为 gap。 -### 5. Public knowledge, private cases +### 5. Official catalog, user-private references, private cases + +DeepLaw `0.3.0` 固定三层边界： + +| 层 | 位置与权限 | 更新规则 | Agent 访问 | +| --- | --- | --- | --- | +| 官方团队目录 | `~/.deeplaw/official` 管理目录；release 在 `~/.deeplaw/releases` | 团队递增 catalog sequence；新建不可变 release；用户可停用或卸载 | 默认 `search/get/verify/release_info`，只读 | +| 用户私有法律资料 | `~/.deeplaw/private`，目录 `0700`、源文件/状态 `0600`、数据库 `0400` | 本机用户显式增删；每次变更重建独立快照；绝不提升为官方法源 | 仅显式 `private_*`，只读且不与官方结果混排 | +| Analytix 案件项目 | Analytix 每案件自己的 SQLite/DuckDB/附件与会话存储 | 由 Analytix 管理 | 不进入 DeepLaw，也不由 DeepLaw 读取 | -DeepLaw 只承载公用、只读知识。案件项目上传、事实、聊天、身份、交易与 Agent memory 必须留 -在宿主私有项目中，不得写入公共 release、sidecar、日志或公开 benchmark。这是强制集成 -边界；`0.2.0` 没有内容级 DLP 或私有资料分类器，宿主必须在调用 DeepLaw 之前完成隔离和拒绝。 +“仅该用户访问”在当前本地版中指**操作系统账户 + owner-only 文件权限**，不是面向共享服务的 +多租户身份认证。把 `DEEPLAW_HOME` 放在共享目录、容器共享卷或多人服务中不能获得用户隔离； +此类部署必须另加宿主认证与每用户物理根目录。DeepLaw 没有内容级 DLP，导入 CLI 因此要求 +`--confirm-no-case-data`，但该确认不能替代宿主的案件数据隔离。 完整 2.0 设计、形式化不变量、实现阶段和“不能做”的事项见 [`docs/DEEPLAW_2.md`](docs/DEEPLAW_2.md)。 ## 当前已经实现什么 -| 能力 | `0.2.0` 状态 | +| 能力 | `0.3.0` 状态 | | --- | --- | -| 文件处理 | DOCX 直接解析；文本 PDF 原生层优先；低质量页可进入本地视觉共识管线 | +| 文件处理 | 官方 DOCX/PDF；私有库另支持 UTF-8 TXT；文本 PDF 原生层优先；低质量页可进入本地视觉共识管线 | +| 官方目录生命周期 | bundled catalog 安装、HTTPS 更新、sequence 防回滚/改写、停用/启用/卸载、逐来源 hash 校验 | +| 用户私有法律资料 | owner-only 物理根目录、显式导入/列出/删除、独立不可变快照、删除后旧快照清理 | | 不可变发布 | source/segment/release hash、原子发布、只读 SQLite、receipt | | 精确定位 | 题名、别名、文号、条款与中文 FTS | | QueryPlan | 8 类封闭 Evidence Duty、稳定 plan ID 和硬边界 | | 时效 | 目标日期、已验证区间、不确定元数据和区间外候选分离 | | Knowledge Map | provenance-bound 单跳关系与有限路径 | -| Agent 接口 | 一个只读 MCP leaf tool，四个 operation | +| Agent 接口 | 一个只读 MCP leaf tool，官方与私有共八个显式 operation；没有写工具 | | 宿主 | Codex、Claude Code、OpenCode 适配；Analytix 仅有未来设计，尚未修改 | -当前还没有实现：签名 release 审批/撤销、完整法律双时态事件账本、coverage-first 最小集、 +当前还没有实现：**加密签名的 catalog/release 审批与撤销**、完整法律双时态事件账本、coverage-first 最小集、 coverage witness、replay trace、外部 held-out 中文法律 benchmark，以及 Analytix 的 schema 前 激活门。这些不会被当前主页表述成已完成。 ## 一个小型 Agent 接口 -DeepLaw 只暴露一个 MCP leaf tool：`law_support`。内部四个 operation 全部只读： +DeepLaw 只暴露一个 MCP leaf tool：`law_support`。内部八个 operation 全部只读： | Operation | 作用 | | --- | --- | @@ -214,6 +274,14 @@ DeepLaw 只暴露一个 MCP leaf tool：`law_support`。内部四个 operation | `get` | 按精确 `segment_id` 读取规范化抽取文本，并显式返回 `truncated` | | `verify` | 验证 receipt 与当前 release 中的 segment hash | | `release_info` | 检查固定 release、schema、审核与再分发状态 | +| `private_search` | 只检索用户私有法律资料；不与官方候选混排 | +| `private_get` | 按精确 ID 读取一个私有 segment | +| `private_verify` | 验证私有快照中的 receipt；快照变更后旧 receipt 失效 | +| `private_info` | 检查当前私有快照；结果始终是用户提供、未经官方审核 | + +私有库增删只存在于本机 CLI 管理面，不存在 `upload`、`delete`、`remember` 或任何 MCP 写操作。 +官方与私有资料需要同时研究时，Agent 必须分别执行两组有界查询并分栏呈现，不能合并排名、 +权威等级或时效结论。 响应骨架： @@ -240,10 +308,20 @@ DeepLaw 只暴露一个 MCP leaf tool：`law_support`。内部四个 operation | OpenCode | [`adapters/opencode`](adapters/opencode) | 默认 deny，由专用 agent 显式授权 | | Analytix | [`docs/ANALYTIX_INTEGRATION.md`](docs/ANALYTIX_INTEGRATION.md) | 未来 turn-scoped 接入；当前未改代码 | +Codex 本地安装（将路径替换为实际 checkout 的绝对路径）： + +```bash +codex plugin marketplace add /absolute/path/to/DeepLaw +codex plugin add deeplaw@deeplaw +``` + 安装插件不等于每轮自动调用。普通代码、数据、SQL 或文档任务不应进入 DeepLaw。未来 Analytix 接入必须在 provider tool schema 物化前完成法律意图门禁，并用 inactive A/B 证明 普通任务的 route、stable prefix、request body、Token 和延迟零退化。 +插件只负责启动只读 MCP；它不会在后台下载或修改资料。首次使用前由用户运行 +`deeplaw official install`，后续更新也只能由用户显式运行 `deeplaw official update`。 + ## 文件与抽取质量 - DOCX：直接解析 OOXML，保留段落、表格行和脚注引用； @@ -263,8 +341,9 @@ uv run deeplaw eval --cases evals/core-2026-07-14.jsonl --limit 5 git diff --check ``` -当前本地 candidate 在 32 项已知语料白盒 smoke case 上为 32/32，109/109 张返回证据的 -receipt 往返核验通过。它绑定特定 release、database、case、源码与环境，并且不是盲测、 +当前本地 candidate 在 32 项已知语料白盒 smoke case 上为 32/32；其中 6 项明确断言抽取风险 +材料只能命中 `uncertain_evidence`，109/109 张返回证据的 receipt 往返核验通过。它绑定特定 +release、database、case、源码与环境，并且不是盲测、 留出集、人工法律金标或跨系统冠军榜。完整 hash、指标与限制见 [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md)。 @@ -278,7 +357,8 @@ DeepLaw 只有在外部 held-out 集、专家中文法律集、mutation suite - 查询时不自动把公网内容加入主证据； - 模型不能决定修订、废止、冲突或优先级； - 不预测有罪、量刑、责任或案件结果； -- 宿主和操作者不得把案件私有材料写入或发送到公共知识库； +- 用户私有库只接收法律参考资料；案件证据、事实、聊天和项目附件仍不得进入 DeepLaw； +- 用户私有候选不得冒充官方法源，也不得影响官方目录的 release、排序、审核或更新； - 不在 issue、PR、日志、截图或 benchmark 中发布受限法源与案件信息。 语料治理见 [`docs/CORPUS_GOVERNANCE.md`](docs/CORPUS_GOVERNANCE.md)，安全报告流程见 @@ -289,7 +369,7 @@ DeepLaw 只有在外部 held-out 集、专家中文法律集、mutation suite | 文档 | 内容 | | --- | --- | | [`docs/DEEPLAW_2.md`](docs/DEEPLAW_2.md) | DeepLaw Architecture 2.0 完整技术设计与研究门禁 | -| [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | 当前 `0.2.0` 实现、存储与运行时事实 | +| [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | 当前 `0.3.0` 实现、存储与运行时事实 | | [`docs/CORPUS_GOVERNANCE.md`](docs/CORPUS_GOVERNANCE.md) | 法源、审核、许可、发布和更新治理 | | [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) | 可复现 smoke 结果与下一阶段评价协议 | | [`docs/AGENT_ADAPTERS.md`](docs/AGENT_ADAPTERS.md) | Codex、Claude Code 与 OpenCode 适配 | @@ -303,9 +383,11 @@ DeepLaw 只有在外部 held-out 集、专家中文法律集、mutation suite - [x] provenance-bound Knowledge Map 与单跳有限导航 - [x] 页级 PDF 证据和人工审校 attestation - [x] Codex、Claude Code、OpenCode 适配 +- [x] 官方团队 catalog 安装、递增更新、停用/启用与本机卸载 +- [x] 用户私有法律资料的物理隔离、显式增删和只读 Agent 检索 - [ ] coverage witness、challenge result 和 replay trace - [ ] coverage-first 最小充分证据集 -- [ ] TXT 输入、完整法律层级与 source-bound Explain +- [ ] 官方 TXT 输入、完整法律层级与 source-bound Explain（私有 UTF-8 TXT 已支持） - [ ] Corpus Coverage Manifest 与法律双时态事件账本 - [ ] 签名发布、撤销、supersession feed 和安全更新 - [ ] 外部 held-out 中文法律证据 benchmark @@ -315,7 +397,8 @@ DeepLaw 只有在外部 held-out 集、专家中文法律集、mutation suite DeepLaw 面向通用法律知识库建设。下表仅展示截至 **2026-07-14** 当前已收录的资料：共 **28** 份二进制原件，含 **10 DOCX**、**18 PDF**、**0 HTML**，不构成未来收录范围的限制。下列数量 -来自操作方维护的下载清单；DeepLaw 仓库不再分发这些原件。 +来自官方团队维护的 bundled catalog；DeepLaw 仓库只分发目录元数据、来源 URL、大小和 hash， +不重新分发这些原件。安装后运行 `deeplaw official update` 才会获取团队后来发布的递增目录。 | 经侦法源分组 | 数量 | 覆盖内容 | | --- | ---: | --- | @@ -340,6 +423,11 @@ DeepLaw 面向通用法律知识库建设。下表仅展示截至 **2026-07-14** | [证监会](https://www.csrc.gov.cn/)、[国家移民管理局](https://www.nia.gov.cn/)、[深交所](https://www.szse.cn/)官方托管页 | 3 | PDF：发布机关文件的官方托管原件 | | **合计** | **28** | **每份均记录 URL、格式、字节数与 SHA-256** | +“官方团队目录”表示该目录、来源声明、hash 和版本生命周期由 DeepLaw 团队维护，不表示每一页 +已经完成人工法律审核。当前 28 份资料的可复现构建中有 **5 份 PDF、15 页**仍带抽取复核标记； +这些片段只进入 `uncertain_evidence`，不会作为已核验主证据。完整逐页记录保存在本机 release 的 +`build-report.json`，团队后续校正必须发布新的递增目录和不可变 release，不能覆盖旧版本。 + 精简更新方法： 1. 以题名、文号、发布机关、公布/施行日期和时效状态定位文件； diff --git a/README_EN.md b/README_EN.md index e962599..080f97a 100644 --- a/README_EN.md +++ b/README_EN.md @@ -14,12 +14,12 @@

- Architecture 2.0 is the target · The current runnable version is 0.2.0 alpha + Architecture 2.0 is the target · The current runnable version is 0.3.0 alpha

CI - Runtime 0.2.0 alpha + Runtime 0.3.0 alpha Python 3.11 and 3.13 Read-only MCP Apache 2.0 @@ -41,14 +41,14 @@

The figure above is a target architecture, not a screenshot of the current runtime. The -`0.2.0` baseline turns DOCX and PDF material into a read-only, versioned, traceable Agent +`0.3.0` baseline turns DOCX and PDF material into a read-only, versioned, traceable Agent Knowledge Base. It performs bounded location, connection, and verification outside the model, then delivers a small **Evidence Pack** containing admitted evidence, uncertain evidence, explicit gaps, and verifiable receipts. Architecture 2.0 adds source-bound -Explain, TXT input, and minimal sufficient evidence selection. +Explain, governed official TXT input, and minimal sufficient evidence selection. `DeepLaw` is the product name; Architecture 2.0 is the next architecture direction. The -runnable package is still the `0.2.0` alpha baseline. This README keeps implemented +runnable package is still the `0.3.0` alpha baseline. This README keeps implemented capabilities separate from research targets. ## Quick Start @@ -58,11 +58,60 @@ DeepLaw requires Python 3.11+ and [`uv`](https://docs.astral.sh/uv/): ```bash git clone https://github.com/Eysn0130/DeepLaw.git cd DeepLaw -uv sync --extra dev -uv run deeplaw --version +uv tool install . +deeplaw --version ``` -Build an operator-owned source package that you are authorized to process and retain: +`uv tool install .` makes the `deeplaw` executable visible to Agent hosts. Run +`uv sync --extra dev` only for source development, and use `uv tool install --force .` +after upgrading runtime code from a checkout. + +Install the bundled DeepLaw team catalog. The client fetches DOCX/PDF files from their +recorded official download URLs, verifies byte sizes and SHA-256 values, then builds locally; +the GitHub repository does not redistribute those source binaries: + +```bash +deeplaw official install +deeplaw official status +deeplaw doctor +``` + +Use `--source-root "/path/to/legal-source-package"` to reuse an already downloaded package +that exactly matches the catalog. Fetch a later monotonically sequenced team catalog with: + +```bash +deeplaw official update +``` + +Transient `429/5xx` and timeout failures use bounded exponential backoff. Sources that already +passed byte-size and SHA-256 verification remain in the local cache, so rerunning can resume the +build. DeepLaw never silently falls back to a non-official mirror just to make an install succeed. + +An in-flight read remains pinned to its release. Later calls from an older MCP process fail +closed after the official epoch changes; restart the Agent/MCP process to use the new release. + +The official catalog is optional. `disable` preserves immutable files, `enable` restores the +pointer, and `uninstall` removes only team-catalog releases and caches: + +```bash +deeplaw official disable +deeplaw official enable +deeplaw official uninstall +``` + +User-supplied legal references use a physically separate local library. MCP access remains +read-only; add and delete are explicit local CLI administration: + +```bash +deeplaw private add \ + --source "/path/to/user-legal-reference.docx" \ + --confirm-no-case-data +deeplaw private list +deeplaw private search --query "document title article 1" +deeplaw private delete --document-id "doc_..." +``` + +You can also build an operator-owned source package that you are authorized to process and retain: ```bash export DEEPLAW_SOURCE_ROOT="/path/to/legal-source-package" @@ -88,12 +137,12 @@ human legal approval. DeepLaw Architecture 2.0 target knowledge cycle: Ingest, Organize, Locate, Connect, Explain, and Verify around the Evidence Core, delivering Evidence, Gaps, and Receipts

-Architecture 2.0 defines six core knowledge actions. The table separates the `0.2.0` +Architecture 2.0 defines six core knowledge actions. The table separates the `0.3.0` baseline from the target: | Action | Responsibility | Current status and boundary | | --- | --- | --- | -| **Ingest** | Verify files, extract content, preserve locators and hashes | DOCX/PDF today; processing success is not human approval | +| **Ingest** | Verify files, extract content, preserve locators and hashes | Official DOCX/PDF; the private library also accepts UTF-8 TXT; processing success is not human approval | | **Organize** | Build hierarchy, versions, relations, and a Knowledge Map | Heading/article segments and order today; full legal hierarchy is a 2.0 target | | **Locate** | Find titles, citations, articles, terms, and related segments | Broad terms do not expand into unbounded output | | **Connect** | Link citations, amendments, repeal, replacement, implementation, and exceptions | Provenance-bound one-hop document relations today; definitions, scope, and challenge closure are 2.0 targets | @@ -113,7 +162,7 @@ before it can become citable. DeepLaw Architecture 2.0 target Evidence Core: Sources and Versions, Knowledge Map, Evidence Duties, Limits and Gaps, Receipts and Replay

-The Evidence Core is the Architecture 2.0 target. The `0.2.0` baseline already implements +The Evidence Core is the Architecture 2.0 target. The `0.3.0` baseline already implements immutable sources, a basic Knowledge Map, Evidence Duties, bounded output, gaps, and receipts. Coverage witnesses, challenge results, and replay traces are not implemented. @@ -191,37 +240,46 @@ This is an Architecture 2.0 target. DeepLaw will actively check temporal change, extraction risk, and conflicts. Each challenge is `satisfied`, `unresolved`, or `not_applicable`; unresolved checks remain gaps. -### Public knowledge, private cases +### Official catalog, user-private references, and private cases + +DeepLaw keeps three physical scopes: + +| Scope | Storage and mutation | Agent access | +| --- | --- | --- | +| Team-maintained official catalog | Immutable releases under `~/.deeplaw/releases`; monotonic catalog updates; optional disable/uninstall | `search/get/verify/release_info` | +| User-private legal references | Separate `~/.deeplaw/private` root; owner-only files; explicit local add/delete and snapshot rebuild | Explicit `private_*` operations only; never blended with official ranking | +| Analytix case projects | Analytix-owned per-case SQLite/DuckDB, attachments, and sessions | Outside DeepLaw; DeepLaw never reads or owns it | -DeepLaw stores shared read-only knowledge. Case uploads, facts, chats, identities, -transactions, and agent memory stay inside the host's private case project. They must -never enter a public release, sidecar, log, or public benchmark. This is a mandatory host -integration boundary: `0.2.0` does not provide content-level DLP or a private-data -classifier, so the host must isolate and reject private material before invoking DeepLaw. +“Only this user” currently means the local OS account plus owner-only filesystem permissions, +not multi-tenant authentication for a shared server or volume. Case evidence, facts, chats, +identities, transactions, and agent memory remain outside both DeepLaw scopes. The CLI requires +`--confirm-no-case-data`, but that acknowledgement is not content-level DLP. The full design, invariants, implementation phases, and non-goals are documented in [`docs/DEEPLAW_2.md`](docs/DEEPLAW_2.md). ## What Is Implemented Today -| Capability | `0.2.0` status | +| Capability | `0.3.0` status | | --- | --- | -| File processing | Direct DOCX parsing; native-text PDF first; local vision consensus for poor pages | +| File processing | Official DOCX/PDF; private UTF-8 TXT; native-text PDF first; local vision consensus for poor pages | +| Official lifecycle | Bundled install, HTTPS catalog update, sequence anti-rollback/rewrite checks, enable/disable/uninstall, per-source hash verification | +| Private legal references | Owner-only physical root, explicit add/list/delete, separate immutable snapshot, old-snapshot cleanup on deletion | | Immutable releases | Source/segment/release hashes, atomic publication, read-only SQLite, receipts | | Precise location | Titles, aliases, document numbers, articles, and Chinese FTS | | QueryPlan | Eight closed Evidence Duties, stable plan IDs, and hard bounds | | Time | Target dates, verified intervals, uncertain metadata, and out-of-interval separation | | Knowledge Map | Provenance-bound one-hop relations and bounded paths | -| Agent interface | One read-only MCP leaf tool with four operations | +| Agent interface | One read-only MCP leaf tool with eight explicit official/private operations and no write operation | | Hosts | Codex, Claude Code, and OpenCode adapters; Analytix integration is design-only | -Not implemented yet: signed release approval and revocation, a complete bitemporal legal +Not implemented yet: cryptographically signed catalog/release approval and revocation, a complete bitemporal legal event ledger, coverage-first selection, coverage witnesses, replay traces, an external held-out Chinese legal benchmark, and the Analytix pre-schema activation gate. ## One Small Agent Interface -DeepLaw exposes one MCP leaf tool, `law_support`, with four read-only operations: +DeepLaw exposes one MCP leaf tool, `law_support`, with eight read-only operations: | Operation | Purpose | | --- | --- | @@ -229,6 +287,14 @@ DeepLaw exposes one MCP leaf tool, `law_support`, with four read-only operations | `get` | Read normalized extracted text by exact `segment_id`, with explicit `truncated` state | | `verify` | Verify a receipt and segment hash in the fixed release | | `release_info` | Inspect the fixed release, schema, review, and redistribution status | +| `private_search` | Search only the user-private legal-reference snapshot | +| `private_get` | Read one exact private segment | +| `private_verify` | Verify a private receipt; old receipts expire after a private snapshot change | +| `private_info` | Inspect the current user-private snapshot | + +Private `add` and `delete` exist only in the local administrative CLI. They are not MCP +operations. When both scopes matter, an Agent must query and present them separately; it may +not merge rankings, authority, or temporal conclusions. Response skeleton: @@ -256,11 +322,21 @@ valid runtime identifiers. | OpenCode | [`adapters/opencode`](adapters/opencode) | Default deny; explicit dedicated-agent grant | | Analytix | [`docs/ANALYTIX_INTEGRATION.md`](docs/ANALYTIX_INTEGRATION.md) | Future turn-scoped integration; no code change yet | +Local Codex install (replace the path with the checkout's absolute path): + +```bash +codex plugin marketplace add /absolute/path/to/DeepLaw +codex plugin add deeplaw@deeplaw +``` + Installation does not mean automatic invocation. Ordinary code, data, SQL, and document tasks should not enter DeepLaw. Future Analytix integration must gate legal intent before provider tool-schema materialization and prove inactive zero impact on routing, the stable prefix, request bodies, tokens, and latency. +The plugin never downloads or mutates legal data in the background. A user explicitly runs +`deeplaw official install` and later `deeplaw official update` outside the Agent MCP surface. + ## File and Extraction Quality - DOCX: direct OOXML parsing with paragraph order, table rows, and footnote references; @@ -282,8 +358,9 @@ uv run deeplaw eval --cases evals/core-2026-07-14.jsonl --limit 5 git diff --check ``` -The current local candidate passes 32/32 known-corpus white-box smoke cases, and 109/109 -returned receipts complete a round-trip verification. This result is pinned to one release, +The current local candidate passes 32/32 known-corpus white-box smoke cases; six cases explicitly +require extraction-risk material to appear only in `uncertain_evidence`, and 109/109 returned +receipts complete a round-trip verification. This result is pinned to one release, database, case set, source tree, and environment. It is not a blind test, a held-out set, a human legal gold standard, or a cross-system leaderboard. See [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) for hashes and limitations. @@ -299,7 +376,8 @@ architecture is welcome; performance claims require evidence. - live web content never enters primary evidence at query time; - a model may not determine amendment, repeal, conflict, or priority on its own; - DeepLaw does not predict guilt, sentence, liability, or case outcome; -- hosts and operators must not write or send private case material to the public knowledge base; +- the private library accepts legal references, not case evidence, facts, chats, or attachments; +- private material never changes official releases, review status, ranking, or update state; - restricted sources and case data must not appear in issues, PRs, logs, screenshots, or benchmarks. See [`docs/CORPUS_GOVERNANCE.md`](docs/CORPUS_GOVERNANCE.md) and @@ -310,7 +388,7 @@ See [`docs/CORPUS_GOVERNANCE.md`](docs/CORPUS_GOVERNANCE.md) and | Document | Scope | | --- | --- | | [`docs/DEEPLAW_2.md`](docs/DEEPLAW_2.md) | Complete 2.0 design and research gates | -| [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Current `0.2.0` implementation and runtime facts | +| [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | Current `0.3.0` implementation and runtime facts | | [`docs/CORPUS_GOVERNANCE.md`](docs/CORPUS_GOVERNANCE.md) | Source, review, license, release, and update governance | | [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) | Reproducible smoke evidence and the next evaluation protocol | | [`docs/AGENT_ADAPTERS.md`](docs/AGENT_ADAPTERS.md) | Codex, Claude Code, and OpenCode adapters | @@ -324,9 +402,11 @@ See [`docs/CORPUS_GOVERNANCE.md`](docs/CORPUS_GOVERNANCE.md) and - [x] Provenance-bound Knowledge Map with bounded one-hop navigation - [x] Page-level PDF evidence and human-review attestations - [x] Codex, Claude Code, and OpenCode adapters +- [x] Team catalog install/update/enable/disable/uninstall lifecycle +- [x] Physically separate user-private legal references with explicit add/delete and read-only Agent access - [ ] Coverage witnesses, challenge results, and replay traces - [ ] Coverage-first minimal sufficient evidence selection -- [ ] TXT input, complete legal hierarchy, and source-bound Explain +- [ ] Official TXT input, complete legal hierarchy, and source-bound Explain (private UTF-8 TXT is implemented) - [ ] Corpus Coverage Manifest and bitemporal legal event ledger - [ ] Signed publication, revocation, supersession feed, and secure updates - [ ] External held-out Chinese legal evidence benchmark @@ -337,7 +417,8 @@ See [`docs/CORPUS_GOVERNANCE.md`](docs/CORPUS_GOVERNANCE.md) and DeepLaw is being built as a general legal knowledge base. The tables below show only the material currently recorded as of **2026-07-14**: **28** binary source files — **10 DOCX**, **18 PDF**, and **0 HTML**. They do not limit future coverage. Counts come from the -operator-maintained download manifest; this repository does not redistribute the files. +bundled team catalog; this repository distributes only catalog metadata, source URLs, sizes, +and hashes, not the source binaries. `deeplaw official update` fetches later monotonic catalogs. | Source-package group | Count | Coverage | | --- | ---: | --- | @@ -362,6 +443,14 @@ the former determines authority; the latter records where the source binary was | Official hosts of the [CSRC](https://www.csrc.gov.cn/), [NIA](https://www.nia.gov.cn/), and [SZSE](https://www.szse.cn/) | 3 | PDF: officially hosted originals issued by the relevant authority | | **Total** | **28** | **Each file records URL, format, byte size, and SHA-256** | +“Team-maintained official catalog” means that the DeepLaw team maintains catalog identity, +source declarations, hashes, and version lifecycle. It does not mean that every page has +completed human legal review. In the reproducible build of the current 28 items, **5 PDFs +and 15 pages** still carry extraction-review flags. Their segments are returned only as +`uncertain_evidence`, never as admitted primary evidence. Full page-level records remain in +the local release's `build-report.json`; corrections require a later catalog sequence and a +new immutable release rather than an overwrite. + Concise update method: 1. Identify the title, document number, issuing authority, promulgation/effective date, and status. diff --git a/benchmarks/core-v3-candidate-2026-07-16.json b/benchmarks/core-v3-candidate-2026-07-16.json new file mode 100644 index 0000000..f0f5dde --- /dev/null +++ b/benchmarks/core-v3-candidate-2026-07-16.json @@ -0,0 +1,58 @@ +{ + "schema_version": "deeplaw.benchmark-summary/v2", + "status": "candidate_smoke_not_held_out", + "run_date": "2026-07-16", + "package_version": "0.3.0", + "release_id": "lawrel_d900543c826fbc4a19ee4f51adc24e1c", + "release_schema": "deeplaw.release/v2", + "storage_schema": "deeplaw.sqlite/v4", + "database_sha256": "7fecc8d151e0e59555adfcc312355f54861debf4dfcae52829381f40db480787", + "source_manifest_sha256": "f25d117a1db575d51fed22114b8ecb2f9ada8f6c3ea9d02fdb2024dc57aa7d5c", + "review_overlay_sha256": "9e9a67ca0e12282d610e192c984b89f2ade63a394b52324e37a404aa84288af2", + "temporal_status": "partially_verified", + "redistribution_status": "restricted", + "cases_path": "evals/core-2026-07-14.jsonl", + "cases_sha256": "95f52e14b11589850a3a7ecc57fb2bf4614a6be85a979a73869dd37973453625", + "eval_report_schema": "deeplaw.eval-report/v2", + "case_count": 32, + "ranked_case_count": 30, + "expected_bucket_counts": { + "evidence": 26, + "uncertain_evidence": 6 + }, + "limit": 5, + "retrieval_pass_rate": 1.0, + "constraint_pass_rate": 1.0, + "overall_pass_rate": 1.0, + "receipt_count": 109, + "verified_receipt_count": 109, + "receipt_verification_pass_rate": 1.0, + "hit_at_1": 1.0, + "mrr": 1.0, + "average_excerpt_chars": 679.719, + "p50_latency_ms": 12.0, + "p95_latency_ms": 17.25, + "latency_scope": "search_only_receipt_verification_excluded", + "implementation": { + "python_source_tree_sha256": "cbbf2bab274259d5ef79971d335cc6a4d42a329f6d0205b165a2b2a81fbf4983", + "search_sha256": "4becaab7ba9e807c04f2277e2c26fe6e36420f09e068172a3e1752ce7c242df6", + "evaluator_sha256": "b9537addf77107024facbb9ff598ca2c4bd30cb02705ccba95db4b1b37f10825", + "query_plan_sha256": "0a042312c774854267e06d95854caad772e441326bf1dec02a18aabe8096742d", + "store_sha256": "4022baaa4d5c0a18520c18ed5416ea07173f4438fe7cc1fe434a1b1ba6a1ae0d", + "mcp_server_sha256": "59ef1d11a57540c1f2843681450cae78698c0973e0495d24a3b56c461f94d105" + }, + "environment": { + "os": "macOS 26.5.2 arm64", + "cpu": "Apple M4 Pro", + "memory_bytes": 25769803776, + "python": "3.12.13", + "sqlite": "3.50.4" + }, + "limitations": [ + "The query set was designed against the known 28-document corpus.", + "This is not a held-out or independently adjudicated legal benchmark.", + "No external retrieval system was compared in this run.", + "Six assertions require extraction-risk documents to appear only in uncertain_evidence; they do not count those documents as admitted primary evidence.", + "The local corpus release remains AI-prechecked, temporally partial, and redistribution-restricted." + ] +} diff --git a/catalogs/deeplaw-official-cn.json b/catalogs/deeplaw-official-cn.json new file mode 100644 index 0000000..00a7f18 --- /dev/null +++ b/catalogs/deeplaw-official-cn.json @@ -0,0 +1,305 @@ +{ + "schemaVersion": "deeplaw.official-catalog/v1", + "catalogId": "deeplaw-cn-official", + "sequence": 1, + "version": "2026.07.14.1", + "publishedOn": "2026-07-14", + "reviewOverlay": { + "resource": "core-2026-07-14.ai-review.json", + "url": "https://raw.githubusercontent.com/Eysn0130/DeepLaw/main/governance/core-2026-07-14.ai-review.json", + "sha256": "9e9a67ca0e12282d610e192c984b89f2ade63a394b52324e37a404aa84288af2" + }, + "buildPolicy": { + "pdfFallback": "vision-consensus", + "allowNeedsOcr": true + }, + "package": { + "name": "DeepLaw 官方法律资料目录", + "retrievedOn": "2026-07-14", + "reviewedOn": "2026-07-14", + "documentCount": 28 + }, + "documents": [ + { + "path": "01-核心法源/中华人民共和国刑法（2020年修正）.docx", + "title": "中华人民共和国刑法（2020年修正）", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff808181796a636a0179822a19640c92", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "byteSize": 114407, + "sha256": "282fee49f12b6420a1a5ecde4ea2aa58b0e496e566cf959378bd1f8edb53f5e6", + "note": "须与刑法修正案（十二）配套使用；不是已合并修正案（十二）的单一合订本。" + }, + { + "path": "01-核心法源/中华人民共和国刑法修正案（十二）.docx", + "title": "中华人民共和国刑法修正案（十二）", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff8081818c3ce31f018cb6a6bc412f55", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2024-03-01", + "byteSize": 16198, + "sha256": "ee51fc72c880c8ed4456ec50196498da6b16bab2430ad63229d408b247ce20c4" + }, + { + "path": "01-核心法源/中华人民共和国刑事诉讼法（2018年修正）.docx", + "title": "中华人民共和国刑事诉讼法（2018年修正）", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff8080816f135f46016f1d1b81b01351", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2018-10-26", + "byteSize": 78913, + "sha256": "6810ffec97e23afcf5e5e0605f6f15c38299e0acb70c55d77c3c5a1882d954a7" + }, + { + "path": "01-核心法源/最高检_公安部_公安机关管辖刑事案件立案追诉标准（二）.pdf", + "title": "最高人民检察院、公安部关于公安机关管辖的刑事案件立案追诉标准的规定（二）", + "format": "PDF", + "officialSource": "https://www.csrc.gov.cn/anhui/c106365/c3007662/3007662/files/%E6%9C%80%E9%AB%98%E4%BA%BA%E6%B0%91%E6%A3%80%E5%AF%9F%E9%99%A2%E2%80%82%E5%85%AC%E5%AE%89%E9%83%A8%E5%85%B3%E4%BA%8E%E5%85%AC%E5%AE%89%E6%9C%BA%E5%85%B3%E7%AE%A1%E8%BE%96%E7%9A%84%E5%88%91%E4%BA%8B%E6%A1%88%E4%BB%B6%E7%AB%8B%E6%A1%88%E8%BF%BD%E8%AF%89%E6%A0%87%E5%87%86%E7%9A%84%E8%A7%84%E5%AE%9A%EF%BC%88%E4%BA%8C%EF%BC%89.pdf", + "downloadMethod": "中国证监会安徽监管局官网公开 PDF", + "byteSize": 277353, + "sha256": "26e82081c441c0237a3b43581550f0fd77db4e817e291b40793499aa03df6954" + }, + { + "path": "02-金融与非法集资/两高_办理洗钱刑事案件适用法律若干问题的解释.docx", + "title": "最高人民法院、最高人民检察院关于办理洗钱刑事案件适用法律若干问题的解释", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff808181923a4d0c019240c6a60609b5", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2024-08-20", + "byteSize": 17838, + "sha256": "ee2b353b241645588d93f974f37d7467132b73a98f5e9d6367de4f60333dd102" + }, + { + "path": "02-金融与非法集资/中华人民共和国反洗钱法（2024年修订）.docx", + "title": "中华人民共和国反洗钱法（2024年修订）", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff808181927b2d1a01930c84b1c96d8b", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2025-01-01", + "byteSize": 27425, + "sha256": "19e760e7c1cf5ffb9806a4063a5dbb641cc5aad1c196bcd1c5b905e5a88948fd" + }, + { + "path": "02-金融与非法集资/防范和处置非法集资条例.pdf", + "title": "防范和处置非法集资条例", + "format": "PDF", + "officialSource": "https://xzfg.moj.gov.cn/law/download?LawID=675&type=pdf", + "downloadMethod": "司法部行政法规库正式 PDF", + "byteSize": 319308, + "sha256": "02cefeec306c9d55fe5dd7ef69c91fbde84ef25061e6433dbbedcfd7aac04b3c" + }, + { + "path": "02-金融与非法集资/非法金融机构和非法金融业务活动取缔办法.pdf", + "title": "非法金融机构和非法金融业务活动取缔办法", + "format": "PDF", + "officialSource": "https://xzfg.moj.gov.cn/law/download?LawID=506&type=pdf", + "downloadMethod": "司法部行政法规库正式 PDF", + "byteSize": 289681, + "sha256": "d58dc8bcdcef4111146641734d7c485083a5d28edf567f24f78123decc288111" + }, + { + "path": "03-数据与网络/中华人民共和国个人信息保护法.docx", + "title": "中华人民共和国个人信息保护法", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff8081817b6472a3017b656cc2040044", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2021-11-01", + "byteSize": 30818, + "sha256": "1815f84f43dd5011442c817a197d745c9aef631f5db3b6c7a75be1acc2b5cd8b" + }, + { + "path": "03-数据与网络/中华人民共和国反电信网络诈骗法.docx", + "title": "中华人民共和国反电信网络诈骗法", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff80818182cf5c220182fd54401023d6", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2022-12-01", + "byteSize": 24566, + "sha256": "f4d47081a729a401b65dd2292159d888c445e51e39b43528a7a49c8e2d68578b" + }, + { + "path": "03-数据与网络/中华人民共和国数据安全法.docx", + "title": "中华人民共和国数据安全法", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff80818179f5e0800179f885c7e70392", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2021-09-01", + "byteSize": 21467, + "sha256": "722d29084fef73485e45485309daef570cee2cdee5c1ed19dcac95b0a28a065b" + }, + { + "path": "04-案例参考/人民法院案例库_何某林非法吸收公众存款案.pdf", + "title": "何某林非法吸收公众存款案", + "caseId": "2024-04-1-113-001", + "format": "PDF", + "officialSource": "https://www.sdcourt.gov.cn/dyzy/resource/cms/article/372898/33902698/2025021808234899218.pdf", + "downloadMethod": "山东省高级人民法院官网公开的人民法院案例库 PDF", + "byteSize": 203917, + "sha256": "810467739eaebe231aba8c49369188ee3d67345ccfb98b41b4f1f1f3c5b95e72" + }, + { + "path": "04-案例参考/人民法院案例库_古某某洗钱案.pdf", + "title": "古某某贩卖、运输毒品、洗钱案", + "caseId": "2023-04-1-356-003", + "format": "PDF", + "officialSource": "https://sdcourt.gov.cn/dyzy/resource/cms/article/372898/44519013/2026020509360576410.pdf", + "downloadMethod": "山东省高级人民法院官网公开的人民法院案例库 PDF", + "byteSize": 346228, + "sha256": "0025f6ab6f48c6055cf9692ae88b46cc4800ff9e02f29dacbc9fab988e23b2e6", + "note": "用于自洗钱与资金掩饰、隐瞒的研判参考；上游犯罪为毒品犯罪。" + }, + { + "path": "04-案例参考/人民法院案例库_徐某_王某掩饰隐瞒犯罪所得案.pdf", + "title": "徐某盗窃、王某掩饰、隐瞒犯罪所得案", + "caseId": "2024-18-1-221-001", + "format": "PDF", + "officialSource": "https://www.sdcourt.gov.cn/dyzy/resource/cms/article/372898/40618344/2025061109221731733.pdf", + "downloadMethod": "山东省高级人民法院官网公开的人民法院案例库 PDF", + "byteSize": 337117, + "sha256": "76ce473953c210c76df7239f638335187dc3e200cd37c9fea2e1dd144c2fbb23" + }, + { + "path": "04-案例参考/人民法院案例库_邱某平_谭某等骗取贷款案.pdf", + "title": "邱某平、谭某等骗取贷款案", + "caseId": "2025-04-1-112-001", + "format": "PDF", + "officialSource": "https://sdcourt.gov.cn/dyzy/resource/cms/article/372898/43361792/2025082009050764215.pdf", + "downloadMethod": "山东省高级人民法院官网公开的人民法院案例库 PDF", + "byteSize": 372809, + "sha256": "ff9983bb190581066c850ca642920c2801f04d7f5abf49f42ce9725c027cefcd" + }, + { + "path": "05-办案程序与证据/最高人民检察院_公安部_关于公安机关办理经济犯罪案件的若干规定.pdf", + "title": "最高人民检察院、公安部关于公安机关办理经济犯罪案件的若干规定", + "format": "PDF", + "officialSource": "https://www.szse.cn/lawrules/rules/judicial/P020180529522156813339.pdf", + "downloadMethod": "深圳证券交易所官网公开 PDF", + "effectiveDate": "2018-01-01", + "byteSize": 291609, + "sha256": "55f16443ff508902e2c820676fcf6d680fe518776182906d5f7685835bf4e00d", + "note": "公通字〔2017〕25 号；首页 OCR 已核验。" + }, + { + "path": "05-办案程序与证据/公安机关办理刑事案件程序规定（公安部令第159号）.pdf", + "title": "公安机关办理刑事案件程序规定", + "format": "PDF", + "officialSource": "https://www.nia.gov.cn/News/files/c1460628/1506344.pdf", + "downloadMethod": "国家移民管理局官网公开 PDF", + "effectiveDate": "2020-09-01", + "byteSize": 530328, + "sha256": "1393c6ac1f94704c6a2995e4def1b485184070bdb0062bf47f430ddbbac3ceeb", + "note": "2012 年公安部令第 127 号修订发布，根据 2020 年公安部令第 159 号修正。" + }, + { + "path": "05-办案程序与证据/关于进一步规范刑事诉讼涉案财物处置工作的意见.pdf", + "title": "关于进一步规范刑事诉讼涉案财物处置工作的意见", + "format": "PDF", + "officialSource": "https://www.sdcourt.gov.cn/zztzfy/388051/388072/388101/1034295/2015031811150726778.pdf", + "downloadMethod": "山东省高级人民法院官网公开 PDF", + "byteSize": 5156550, + "sha256": "1a65e80fe998af0729b87bcf5474d6d4827bace7eeb799c2bd94fb9fd58f0459", + "note": "中办发〔2015〕7 号；首页 OCR 已核验。" + }, + { + "path": "05-办案程序与证据/最高人民法院关于适用〈中华人民共和国刑事诉讼法〉的解释.docx", + "title": "最高人民法院关于适用《中华人民共和国刑事诉讼法》的解释", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff8081817b969375017b9b912cd009dc", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2021-03-01", + "byteSize": 134295, + "sha256": "3f2a61fe440bef7ff186714c40a50526bdea9a041d5bbd1035a93f6962f63a39" + }, + { + "path": "06-反洗钱、支付与主体穿透/中华人民共和国外汇管理条例.pdf", + "title": "中华人民共和国外汇管理条例", + "format": "PDF", + "officialSource": "https://xzfg.moj.gov.cn/law/download?LawID=458&type=pdf", + "downloadMethod": "司法部行政法规库正式 PDF", + "byteSize": 373853, + "sha256": "87ee8eb151daa3b45b0e2c2f0a31ed9257e1ca59f8ec79ebc06c3df7df1715c1" + }, + { + "path": "06-反洗钱、支付与主体穿透/中国人民银行关于修改和废止部分规章的决定（2025）.pdf", + "title": "中国人民银行关于修改和废止部分规章的决定", + "format": "PDF", + "officialSource": "https://www.pbc.gov.cn/zhengwugongkai/4081330/4406346/4406348/5885116/2025111915332696325.pdf", + "downloadMethod": "中国人民银行官网公开 PDF", + "effectiveDate": "2025-12-01", + "byteSize": 163788, + "sha256": "830768edc19e64eaa7491dcb9940d09b33a6e22418af72a01dde5ce0db218996", + "note": "涉及金融机构大额交易和可疑交易报告、金融机构反洗钱和反恐怖融资监督管理办法等。" + }, + { + "path": "06-反洗钱、支付与主体穿透/受益所有人信息管理办法.pdf", + "title": "受益所有人信息管理办法", + "format": "PDF", + "officialSource": "https://www.pbc.gov.cn/zhengwugongkai/attachDir/2025/11/2025111915185929550.pdf", + "downloadMethod": "中国人民银行官网公开 PDF", + "effectiveDate": "2024-11-01", + "byteSize": 133834, + "sha256": "c12a102ff3bc3ad942d0af08f9030669f3189f8bcdc9687c13d5d307a576c0b5" + }, + { + "path": "06-反洗钱、支付与主体穿透/金融机构反洗钱和反恐怖融资监督管理办法（2021）.pdf", + "title": "金融机构反洗钱和反恐怖融资监督管理办法", + "format": "PDF", + "officialSource": "https://xining.pbc.gov.cn/zhengwugongkai/attachDir/2025/11/2025111914391365975.pdf", + "downloadMethod": "中国人民银行西宁中心支行官网公开 PDF", + "effectiveDate": "2021-08-01", + "byteSize": 264585, + "sha256": "d7b9322aff9da739f5167ad0befb34ad5cc4b47ebee6ff658bbccb0656bba5ce", + "note": "须与本包中的 2025 年修改决定配套阅读。" + }, + { + "path": "06-反洗钱、支付与主体穿透/金融机构大额交易和可疑交易报告管理办法（现行整合文本）.pdf", + "title": "金融机构大额交易和可疑交易报告管理办法（现行整合文本）", + "format": "PDF", + "officialSource": "https://www.pbc.gov.cn/zhengwugongkai/attachDir/2025/11/2025111914324982441.pdf", + "downloadMethod": "中国人民银行官网公开 PDF", + "byteSize": 301472, + "sha256": "1f0e210f5d01b5c3029d8d6d71db3adad16385c0b022510bb2a3c6f42c6bb5b0", + "note": "2016 年第 5 号令文本；同时适用本包中的 2025 年修改决定。" + }, + { + "path": "06-反洗钱、支付与主体穿透/金融机构客户尽职调查和客户身份资料及交易记录保存管理办法（2025）.pdf", + "title": "金融机构客户尽职调查和客户身份资料及交易记录保存管理办法", + "format": "PDF", + "officialSource": "https://www.pbc.gov.cn/zhengwugongkai/attachDir/2025/12/2025122909264498909.pdf", + "downloadMethod": "中国人民银行官网公开 PDF", + "effectiveDate": "2026-01-01", + "byteSize": 250905, + "sha256": "b304b78dca6ccf44fe076866cda884fcee16a9dc687394ecdc7d7b11a3b84f85" + }, + { + "path": "06-反洗钱、支付与主体穿透/非银行支付机构监督管理条例.pdf", + "title": "非银行支付机构监督管理条例", + "format": "PDF", + "officialSource": "https://xzfg.moj.gov.cn/law/download?LawID=1696&type=pdf", + "downloadMethod": "司法部行政法规库正式 PDF", + "effectiveDate": "2024-05-01", + "byteSize": 371064, + "sha256": "2ee521dff7070bf0471b041fb302b8b0f46b6b137b13157b7e627203aa6907c8" + }, + { + "path": "06-反洗钱、支付与主体穿透/非银行支付机构监督管理条例实施细则.pdf", + "title": "非银行支付机构监督管理条例实施细则", + "format": "PDF", + "officialSource": "https://www.pbc.gov.cn/zhengwugongkai/attachDir/2025/11/2025111915174857578.pdf", + "downloadMethod": "中国人民银行官网公开 PDF", + "effectiveDate": "2024-07-09", + "byteSize": 269856, + "sha256": "7e99645e83429ef4e073dec77ff4e2ce31558a41c13c1910fafbf8f22dc64afb" + }, + { + "path": "07-罪名专题/最高人民法院、最高人民检察院关于办理危害税收征管刑事案件适用法律若干问题的解释.docx", + "title": "最高人民法院、最高人民检察院关于办理危害税收征管刑事案件适用法律若干问题的解释", + "format": "DOCX", + "officialSource": "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=ff8081818faab35d018fc240e2795985", + "downloadMethod": "国家法律法规数据库页面下载按钮", + "effectiveDate": "2024-03-20", + "byteSize": 23365, + "sha256": "99ccd53a5245c7aa4a1777c24eeadedf4a38a307320309831325549e90154f05" + } + ] +} diff --git a/contracts/corpus-release-manifest.v2.schema.json b/contracts/corpus-release-manifest.v2.schema.json index 8ae8be2..53f8601 100644 --- a/contracts/corpus-release-manifest.v2.schema.json +++ b/contracts/corpus-release-manifest.v2.schema.json @@ -56,6 +56,11 @@ }, "redistribution_status": { "enum": ["not_assessed", "approved", "restricted"] }, "vector_index": { "type": "boolean" }, - "derived_wiki": { "type": "boolean" } + "derived_wiki": { "type": "boolean" }, + "collection_scope": { "enum": ["official", "user_private"] }, + "library_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$" + } } } diff --git a/contracts/law-support.input.v2.schema.json b/contracts/law-support.input.v2.schema.json new file mode 100644 index 0000000..eb1469f --- /dev/null +++ b/contracts/law-support.input.v2.schema.json @@ -0,0 +1,267 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:deeplaw:schema:law-support-input:v2", + "title": "DeepLawSupportInputV2", + "oneOf": [ + { + "title": "search", + "type": "object", + "additionalProperties": false, + "required": [ + "query" + ], + "properties": { + "operation": { + "const": "search", + "default": "search" + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "purpose": { + "enum": [ + "auto", + "exact_citation", + "as_of_version", + "elements", + "legal_issue_screen", + "citation_verify", + "broad_topic" + ], + "default": "auto" + }, + "as_of": { + "type": [ + "string", + "null" + ], + "format": "date", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 5, + "default": 5 + }, + "max_chars": { + "type": "integer", + "minimum": 500, + "maximum": 6000, + "default": 3500 + }, + "document_types": { + "type": [ + "array", + "null" + ], + "items": { + "enum": [ + "law", + "administrative_regulation", + "judicial_interpretation", + "prosecution_standard", + "departmental_rule", + "normative_document", + "case_reference" + ] + }, + "uniqueItems": true, + "maxItems": 7 + } + } + }, + { + "title": "get", + "type": "object", + "additionalProperties": false, + "required": [ + "operation", + "segment_id" + ], + "properties": { + "operation": { + "const": "get" + }, + "segment_id": { + "type": "string", + "pattern": "^seg_[0-9a-f]{24}$" + }, + "max_chars": { + "type": "integer", + "minimum": 500, + "maximum": 6000, + "default": 3500 + } + } + }, + { + "title": "verify", + "type": "object", + "additionalProperties": false, + "required": [ + "operation", + "segment_id", + "receipt_id" + ], + "properties": { + "operation": { + "const": "verify" + }, + "segment_id": { + "type": "string", + "pattern": "^seg_[0-9a-f]{24}$" + }, + "receipt_id": { + "type": "string", + "pattern": "^lawrcpt_[0-9a-f]{32}$" + } + } + }, + { + "title": "release_info", + "type": "object", + "additionalProperties": false, + "required": [ + "operation" + ], + "properties": { + "operation": { + "const": "release_info" + } + } + }, + { + "title": "private_search", + "type": "object", + "additionalProperties": false, + "required": [ + "operation", + "query" + ], + "properties": { + "operation": { + "const": "private_search" + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "purpose": { + "enum": [ + "auto", + "exact_citation", + "as_of_version", + "elements", + "legal_issue_screen", + "citation_verify", + "broad_topic" + ], + "default": "auto" + }, + "as_of": { + "type": [ + "string", + "null" + ], + "format": "date", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 5, + "default": 5 + }, + "max_chars": { + "type": "integer", + "minimum": 500, + "maximum": 6000, + "default": 3500 + }, + "document_types": { + "type": [ + "array", + "null" + ], + "items": { + "enum": [ + "law", + "administrative_regulation", + "judicial_interpretation", + "prosecution_standard", + "departmental_rule", + "normative_document", + "case_reference" + ] + }, + "uniqueItems": true, + "maxItems": 7 + } + } + }, + { + "title": "private_get", + "type": "object", + "additionalProperties": false, + "required": [ + "operation", + "segment_id" + ], + "properties": { + "operation": { + "const": "private_get" + }, + "segment_id": { + "type": "string", + "pattern": "^seg_[0-9a-f]{24}$" + }, + "max_chars": { + "type": "integer", + "minimum": 500, + "maximum": 6000, + "default": 3500 + } + } + }, + { + "title": "private_verify", + "type": "object", + "additionalProperties": false, + "required": [ + "operation", + "segment_id", + "receipt_id" + ], + "properties": { + "operation": { + "const": "private_verify" + }, + "segment_id": { + "type": "string", + "pattern": "^seg_[0-9a-f]{24}$" + }, + "receipt_id": { + "type": "string", + "pattern": "^lawrcpt_[0-9a-f]{32}$" + } + } + }, + { + "title": "private_info", + "type": "object", + "additionalProperties": false, + "required": [ + "operation" + ], + "properties": { + "operation": { + "const": "private_info" + } + } + } + ] +} diff --git a/contracts/official-catalog.v1.schema.json b/contracts/official-catalog.v1.schema.json new file mode 100644 index 0000000..7d6acfa --- /dev/null +++ b/contracts/official-catalog.v1.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:deeplaw:schema:official-catalog:v1", + "title": "DeepLawOfficialCatalogV1", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "catalogId", + "sequence", + "version", + "publishedOn", + "package", + "documents" + ], + "properties": { + "schemaVersion": { "const": "deeplaw.official-catalog/v1" }, + "catalogId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$" + }, + "sequence": { "type": "integer", "minimum": 1 }, + "version": { "type": "string", "minLength": 1, "maxLength": 100 }, + "publishedOn": { "type": "string", "format": "date" }, + "reviewOverlay": { + "type": "object", + "additionalProperties": false, + "required": ["resource", "url", "sha256"], + "properties": { + "resource": { + "type": "string", + "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}\\.json$" + }, + "url": { "type": "string", "format": "uri", "maxLength": 2048 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, + "buildPolicy": { + "type": "object", + "additionalProperties": false, + "required": ["pdfFallback", "allowNeedsOcr"], + "properties": { + "pdfFallback": { "enum": ["off", "vision-consensus"] }, + "allowNeedsOcr": { "type": "boolean" } + } + }, + "package": { + "type": "object", + "required": ["name", "documentCount"], + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 500 }, + "retrievedOn": { "type": "string", "format": "date" }, + "reviewedOn": { "type": "string", "format": "date" }, + "documentCount": { "type": "integer", "minimum": 1 } + } + }, + "documents": { + "type": "array", + "minItems": 1, + "maxItems": 10000, + "items": { + "type": "object", + "required": [ + "path", + "title", + "format", + "officialSource", + "byteSize", + "sha256" + ], + "properties": { + "path": { "type": "string", "minLength": 1, "maxLength": 1024 }, + "title": { "type": "string", "minLength": 1, "maxLength": 500 }, + "format": { "enum": ["DOCX", "PDF"] }, + "officialSource": { "type": "string", "format": "uri", "maxLength": 2048 }, + "byteSize": { "type": "integer", "minimum": 1, "maximum": 536870912 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + } + } + } +} diff --git a/contracts/private-library-state.v1.schema.json b/contracts/private-library-state.v1.schema.json new file mode 100644 index 0000000..bb825ec --- /dev/null +++ b/contracts/private-library-state.v1.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:deeplaw:schema:private-library-state:v1", + "title": "DeepLawPrivateLibraryStateV1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "library_id", + "revision", + "active_release_id", + "documents", + "pdf_fallback", + "allow_needs_ocr" + ], + "properties": { + "schema_version": { "const": "deeplaw.private-library/v1" }, + "library_id": { "const": "local-user" }, + "revision": { "type": "integer", "minimum": 0 }, + "active_release_id": { + "type": ["string", "null"], + "pattern": "^lawrel_[0-9a-f]{32}$" + }, + "pdf_fallback": { "enum": ["off", "vision-consensus"] }, + "allow_needs_ocr": { "type": "boolean" }, + "documents": { + "type": "array", + "maxItems": 10000, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "document_id", + "title", + "format", + "source_sha256", + "byte_size", + "storage_name", + "document_type", + "issuer", + "effective_from", + "effective_to", + "added_at" + ], + "properties": { + "document_id": { "type": "string", "pattern": "^doc_[0-9a-f]{24}$" }, + "title": { "type": "string", "minLength": 1, "maxLength": 500 }, + "format": { "enum": ["DOCX", "PDF", "TXT"] }, + "source_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "byte_size": { "type": "integer", "minimum": 1, "maximum": 536870912 }, + "storage_name": { + "type": "string", + "pattern": "^[0-9a-f]{64}\\.(docx|pdf|txt)$" + }, + "document_type": { + "enum": [ + "law", + "administrative_regulation", + "judicial_interpretation", + "prosecution_standard", + "departmental_rule", + "normative_document", + "case_reference" + ] + }, + "issuer": { "type": "string", "minLength": 1, "maxLength": 200 }, + "effective_from": { "type": ["string", "null"], "format": "date" }, + "effective_to": { "type": ["string", "null"], "format": "date" }, + "added_at": { "type": "string", "format": "date-time", "maxLength": 40 } + } + } + } + } +} diff --git a/docs/AGENT_ADAPTERS.md b/docs/AGENT_ADAPTERS.md index e04c944..8309546 100644 --- a/docs/AGENT_ADAPTERS.md +++ b/docs/AGENT_ADAPTERS.md @@ -21,7 +21,7 @@ so the visible name can differ, but the server-level leaf name must remain exactly `law_support`. For example, OpenCode renders it as `deeplaw_law_support`. Host namespacing does not create a second public tool. -`law_support` routes four operations: +`law_support` routes eight operations: | Operation | Purpose | Required selector | | --- | --- | --- | @@ -29,10 +29,16 @@ exactly `law_support`. For example, OpenCode renders it as | `get` | Read one selected exact segment | `segment_id` | | `verify` | Verify one evidence receipt | `segment_id`, `receipt_id` | | `release_info` | Inspect the active immutable release | none | +| `private_search` | Search the separate user-private legal-reference snapshot | `query` | +| `private_get` | Read one selected private segment | `segment_id` | +| `private_verify` | Verify one private snapshot receipt | `segment_id`, `receipt_id` | +| `private_info` | Inspect the private snapshot | none | -No host adapter may expose a separate search, get, write, upload, memory, +No host adapter may expose a separate write, upload, memory, reindex, delete, activation, administration, case, or chat tool. Build and -activation remain offline CLI administration, outside the Agent surface. +activation remain offline CLI administration, outside the Agent surface. The +private operations are read-only routes on the same leaf; private add/delete +remain local CLI administration. The MCP handshake is a release gate: `tools/list` must contain one item, and its leaf name must be `law_support`. Treat zero tools, a renamed tool, or a second @@ -41,13 +47,21 @@ tool as a deployment failure rather than silently continuing. ## Runtime prerequisite Install the `deeplaw` executable into the environment used to launch the Agent -host. From a DeepLaw checkout, one development option is: +host. From a DeepLaw checkout, use a user-level tool install for normal use: ```bash -uv tool install --editable . +uv tool install . deeplaw --version +deeplaw official install +deeplaw official status ``` +Contributors who intentionally want live source edits can instead use +`uv tool install --editable .`; a normal user updates checkout code with +`uv tool install --force .`. A project-only `uv sync` is insufficient when the +Agent launches the plugin from another working directory unless its environment +also exposes that project's `.venv/bin`. + The MCP process inherits its environment. Point it at an immutable database with `DEEPLAW_DB`, or point `DEEPLAW_HOME` at a directory containing `ACTIVE` and the corresponding `releases//deeplaw.sqlite3`. Do not put a machine-local @@ -65,6 +79,13 @@ deeplaw mcp --help Running `deeplaw mcp --stdio` directly waits for MCP messages on standard input; that is expected, not a startup hang. +The plugin never downloads or mutates a corpus in the background. Official +updates require an explicit `deeplaw official update`; private legal-reference +imports require `deeplaw private add --confirm-no-case-data`. An MCP process +pins both available scopes at startup. Restart it after an official update or a +private mutation; after either managed epoch changes, the old process rejects +later reads in that scope when its pinned epoch no longer matches. + ## Explicit invocation policy DeepLaw is an optional capability, not a permanent legal persona. Ordinary @@ -81,7 +102,8 @@ These are negative triggers and must not invoke DeepLaw by themselves: - coding, SQL, statistics, dashboards, and ordinary data analysis; - summarizing, extracting, OCRing, translating, or rewriting DOCX/PDF/text; - UI, session, attachment, SQLite, DuckDB, or project-management work; -- searching or storing private case evidence and chat history; +- searching or storing private case evidence and chat history; the DeepLaw + private scope contains legal references only; - filenames, columns, labels, or prose containing `诈骗`, `案件`, `法务`, `fraud`, `risk`, or another isolated domain word. @@ -99,10 +121,15 @@ The Codex plugin is rooted at `plugins/deeplaw`: - `skills/research-chinese-law/agents/openai.yaml` sets `policy.allow_implicit_invocation: false`. -The repository's personal marketplace entry can be used from the Codex plugin +The repository's DeepLaw marketplace entry can be used from the Codex plugin browser during development. Install DeepLaw, start a new task, and invoke it explicitly: +```bash +codex plugin marketplace add /absolute/path/to/DeepLaw +codex plugin add deeplaw@deeplaw +``` + ```text $research-chinese-law 核验《中华人民共和国刑法》某条在 2020-06-01 的有效版本。 ``` @@ -214,7 +241,8 @@ in a later, scoped Analytix change with these invariants: stable system prefix and ordinary data tools for every other turn. 5. Keep private case documents, facts, chats, embeddings, SQLite state, and DuckDB data inside the case project. Send only a minimal de-identified legal - issue to DeepLaw; never ingest private material into a public release. + issue to the official DeepLaw scope. The optional DeepLaw user-private scope + is for legal references, not an Analytix case store. 6. Keep the two-stage pattern: bounded `search`, then exact `get` and selective `verify`. Do not place broad retrieval results in the main context. 7. Fail closed. If the process, release, receipt, or version check fails, report diff --git a/docs/ANALYTIX_INTEGRATION.md b/docs/ANALYTIX_INTEGRATION.md index e1ca66b..88acb3d 100644 --- a/docs/ANALYTIX_INTEGRATION.md +++ b/docs/ANALYTIX_INTEGRATION.md @@ -7,7 +7,8 @@ 已接受规范和测试为准重新复核。 目标不是把 DeepLaw 写进 Analytix 核心，而是让 Analytix 的唯一生产 Go Runtime 安全消费 -一个外部、只读、版本化的公共法律能力： +一个外部、只读、版本化的官方法律能力。DeepLaw `0.3.0` 另有用户私有法律参考资料范围， +但它不是 Analytix 案件项目库，首期 Analytix 集成不开放 `private_*` operation： ```text 公共 DeepLaw release @@ -29,17 +30,19 @@ SQLite/DuckDB，不决定案件事实，也不拥有最终发布权。 | 数据 | 所有者 | 权限 | 可否进入 DeepLaw | |---|---|---|---| | 公共法律 release | DeepLaw | 全局只读、版本化 | 是 | +| 用户私有法律参考资料 | DeepLaw，本机 OS 用户 | owner-only、可由用户离线增删；Agent 只读 | 是，但与公共 release 物理分离；不是案件库 | | 案件文档/对话/事实 | 每个 Analytix 案件项目 | 案件独占、可写 | 否 | | 交易和统计数据 | 每个案件的 DuckDB | 案件独占、可写 | 否 | “全局只读”表示所有被授权 Agent 可以查询同一公共 release，不表示法律 Skill、正文或 MCP -schema 对所有 provider turn 永久可见。 +schema 对所有 provider turn 永久可见。用户私有法律资料也不能通过 `case_id` filter 与案件库 +或官方库合并；三者分别拥有物理根目录和生命周期。 ## Provider 可见接口 DeepLaw 当前只暴露一个紧凑工具：`law_support`。公开 contract 位于： -- `contracts/law-support.input.v1.schema.json` +- `contracts/law-support.input.v2.schema.json`（当前八 operation；v1 保留官方四 operation） - `contracts/law-support.output.v2.schema.json` - `contracts/law-search-response.v2.schema.json` - `contracts/law-segment.v2.schema.json` @@ -48,10 +51,11 @@ DeepLaw 当前只暴露一个紧凑工具：`law_support`。公开 contract 位 - `contracts/legal-evidence-card.v2.schema.json` - `contracts/corpus-release-manifest.v2.schema.json` -`law-support.output` 是四种只读操作结果的 closed union；宿主必须按 operation 验证对应分支， +`law-support.output` 是四种只读结果形状的 closed union；`law-support.input` 当前路由八种只读 +operation（官方四种、私有四种）。宿主必须按 operation 验证对应分支， 拒绝未知字段，不能只验证 MCP transport success。 -DeepLaw `0.2.0` 使用 MCP SDK 的低层 `Server`，只支持本地 stdio transport；它不是 HTTP +DeepLaw `0.3.0` 使用 MCP SDK 的低层 `Server`，只支持本地 stdio transport；它不是 HTTP 服务。握手时发布的是已内联本地 `$ref` 的 closed output schema，宿主不应在运行时依赖网络 解析 GitHub schema URL。 @@ -67,6 +71,15 @@ DeepLaw `0.2.0` 使用 MCP SDK 的低层 `Server`，只支持本地 stdio transp | `get` | 按稳定 `segment_id` 获取规范化抽取文本 | 只能读取已选定 segment；显式返回 `truncated`，必要时提高 `max_chars`；仍须按 source 与 locator 核对 | | `verify` | 校验 segment/receipt | 重算 segment 文本 hash，并校验 receipt 对 release、document、segment 和已记录 source/segment hash 的绑定；不重哈希原件，不判断案件适用 | | `release_info` | 获取当前 release 元数据 | 用于宿主预检和版本绑定 | +| `private_search` | 检索用户私有法律参考资料 | 首期 Analytix 必须拒绝；不得用于案件事实检索 | +| `private_get` | 读取一个私有 segment | 首期 Analytix 必须拒绝 | +| `private_verify` | 校验私有快照 receipt | 首期 Analytix 必须拒绝 | +| `private_info` | 获取私有快照元数据 | 首期 Analytix 必须拒绝 | + +由于八种 operation 共用一个 `law_support` leaf，只按 leaf allowlist 不足以完成首期隔离。 +Analytix 必须在 provider schema 物化前投影出只含官方四个分支的输入 schema，并在执行适配器 +再次拒绝 `private_*`，不能只依赖模型“不去调用”。未来若要开放用户私有法律参考，应建立另一个 +明确授权的能力 scope；仍不得把它接到案件附件、会话或案件 SQLite/DuckDB。 Provider 输入不得包含： @@ -115,7 +128,8 @@ Skill 文本、MCP description 或模型自行选择都不能成为激活权威 DeepLaw 当前在 MCP 进程 lifespan 启动时只解析一次 `ACTIVE` 或 `DEEPLAW_DB`，校验数据库 artifact，并在该进程内复用同一个只读连接；运行中改变 `ACTIVE` 不会切换既有进程的 -release。未来 Analytix 仍必须把这一进程/连接生命周期纳入宿主权威： +release。由本机官方目录管理的进程会在后续调用前检测 ACTIVE epoch，漂移后失败关闭；显式 +`DEEPLAW_DB` 则由宿主负责 epoch。未来 Analytix 仍必须把这一进程/连接生命周期纳入宿主权威： - 在 provider 调用前完成 MCP 初始化和 `release_info` 预检； - 将已验证的 `release_id`、数据库 hash、MCP server identity 和 connection epoch 绑定到 @@ -233,12 +247,14 @@ verify current case authority - `packages/runtime-go/internal/adapters/outbound/mcp` - 通过本地 stdio 消费 DeepLaw 低层 MCP server，保留 closed input/output schema、MCP error、server identity、connection epoch 和只读提示； - - 对超限、未知字段、release 漂移和 receipt mismatch 失败关闭。 + - 对超限、未知字段、release 漂移和 receipt mismatch 失败关闭；首期执行层硬拒绝 + `private_*` operation。 - `packages/runtime-go/internal/runtimeapp/app.go` - 组装 capability gate、release binding 和 receipt verifier； - 法律服务不可用不影响非法律 use case。 - `packages/runtime-go/internal/server/tool_catalog.go` - - 在 MCP schema 物化之前应用 host-owned DeepLaw advertisement filter。 + - 在 MCP schema 物化之前应用 host-owned DeepLaw advertisement filter，并把 + `law-support.input` 投影为官方四个 operation；不能原样广告私有分支。 - `packages/runtime-go/internal/server/agent_loop.go` - 使用过滤后的 schema 计算 route、tool manifest hash 和 provider request； - 记录激活与 release 绑定证据，但不记录案件事实或法律正文。 @@ -356,6 +372,7 @@ DeepLaw 在 Analytix 中应遵循已接受的 Context Epoch 规则： - 不把法律 Skill 写入全局 stable system prefix； - 不把当前案件绑定当作法律能力自动激活信号； - 不把公共法律库与案件 SQLite/DuckDB 或全局向量库合并； +- 不把 DeepLaw 用户私有法律参考库当作案件附件/会话/事实存储； - 不让 provider 指定 case/path/release 或直接签发权威 receipt； - 不让模型、派生专题页、相似关系或相似案例决定效力和案件结论； - 不因法律服务故障阻断普通数据、代码、文件或会话能力； diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 43bfad9..60f1bf4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,12 +1,15 @@ # DeepLaw Architecture -Status: architecture baseline for DeepLaw `0.2.0`, reviewed against the current +Status: architecture baseline for DeepLaw `0.3.0`, reviewed against the current implementation on 2026-07-15. -DeepLaw is a read-only, version-aware Agent Knowledge Base for Chinese legal sources. It is a -separate local service and release format used by Codex, Claude Code, and -OpenCode, and designed for a later Analytix integration. It is not an agent -memory store, a case workspace, or an LLM-authored legal authority. +DeepLaw is a version-aware Agent Knowledge Base for Chinese legal sources. Its +Agent/MCP surface is read-only. Offline CLI administration manages a +team-maintained official catalog and a physically separate per-OS-user legal +reference library. It is a separate local service and release format used by +Codex, Claude Code, and OpenCode, and designed for a later Analytix integration. +It is not an agent memory store, a case workspace, or an LLM-authored legal +authority. ## Decision Summary @@ -21,8 +24,11 @@ memory store, a case workspace, or an LLM-authored legal authority. explanations optional, derived, replaceable, and outside the authority decision. - Expose one compact, read-only MCP tool instead of injecting a legal corpus or a large tool catalogue into every agent session. -- Never place case-private documents, conversations, facts, or identifiers in a - public DeepLaw release. +- Keep official releases and user-private legal-reference snapshots in separate + roots, ACTIVE pointers, source caches, and operation namespaces; never blend + their candidate ranking or authority. +- Never place case-private documents, conversations, facts, or identifiers in + either DeepLaw scope. This is a constrained Agent Knowledge Base. Local semantic discovery, relationship discovery, and source-bound explanations may be added as @@ -51,7 +57,9 @@ establish legal validity. - Treat a date match as proof that a rule governs a case. - Let an LLM decide whether a law was amended, repealed, or superseded. - Automatically crawl and publish an unreviewed comprehensive legal corpus. -- Store case projects, uploads, chats, user memory, or agent state. +- Store case projects, case uploads, chats, user memory, or agent state. A + user-private DeepLaw upload is limited to legal references and is not an + Analytix case library. - Make every non-legal Analytix task pass through a legal classifier or legal retrieval pipeline. - Claim that DeepLaw exceeds every external knowledge system without a fair, @@ -87,22 +95,26 @@ flowchart LR Q --> G["Stable legal segments"] G --> R["Content-addressed SQLite release"] R --> E["Read-only DeepLaw engine"] + U["Explicit user legal-reference upload"] --> P["Owner-only private snapshot"] + P --> E E --> M["Single MCP leaf tool\nlaw_support"] M --> C["Codex"] M --> H["Claude Code"] M --> O["OpenCode"] M -. "future scoped integration" .-> A["Analytix"] - P["Case-private documents, chats, facts"] -. "must remain in host case project" .-> A - P -. "never enters public release" .-> R + C["Case-private documents, chats, facts"] -. "must remain in host case project" .-> A + C -. "never enters either DeepLaw scope" .-> R + C -. "never enters either DeepLaw scope" .-> P ``` -The builder is an offline administrative surface. The `0.2.0` MCP runtime uses +The builder, official updater, and private add/delete commands are offline +administrative surfaces. The `0.3.0` MCP runtime uses the SDK's low-level `Server` over local stdio only; it has no HTTP listener and no corpus-write operation. Its process lifespan resolves and verifies one -release, computes the database hash once during startup, keeps that read-only -release fixed for the lifetime of the process, and serializes access to the -shared SQLite connection. By default, installed hosts share `~/.deeplaw`; +available release, computes database hashes once during startup, keeps each +read-only release fixed for the lifetime of the process, and serializes access +to the SQLite connections. By default, installed hosts share `~/.deeplaw`; deployments can override that root with `DEEPLAW_HOME` or select one artifact before startup with `DEEPLAW_DB`. Host-specific configuration and skills are thin adapters; they do not contain a second copy of retrieval logic. @@ -143,7 +155,7 @@ bytes, OCR output, parser identity, or normalized legal text creates a different release. SQLite's binary hash remains a separate artifact integrity value. -### Known `0.2.0` release limitations +### Known `0.3.0` release limitations The current implementation is an alpha baseline, not yet a production release authority. The following gaps must remain visible until code and tests close @@ -153,7 +165,10 @@ them: heuristics; they require explicit review before a production release. - The builder validates an HTTPS source declaration but does not yet enforce an approved official-domain policy or independently refetch the source. -- Release metadata carries a database SHA-256 but no release signature. +- Release metadata carries a database SHA-256 but no release signature. The + current update channel trusts HTTPS plus the GitHub team account and catalog + sequence, then verifies every source byte size and SHA-256; it is not yet a + cryptographically signed feed. - The CLI has a local candidate activation pointer but does not yet implement signed release approval, revocation, or supersession workflows. Human PDF page-review files record reviewer identity, role, time, and attestation, but @@ -218,6 +233,52 @@ DOCX/PDF bytes. Source packages remain separately controlled inputs. Runtime connections use `query_only`, foreign keys, and SQLite's immutable read-only URI. No WAL or write-capable cache belongs beside a published database. +### Storage scopes and lifecycle + +```text +~/.deeplaw/ +├── ACTIVE # enabled official release only +├── releases// # immutable official releases +├── official/ +│ ├── state.json # installed IDs + monotonic catalog state +│ ├── catalogs/ # exact catalog snapshots +│ └── sources/ # hash-addressed official downloads +└── private/ + ├── ACTIVE # active user-private snapshot + ├── library.json # private document inventory, mode 0600 + ├── sources/ # hash-addressed private files, mode 0600 + └── releases// # owner-only snapshot, database mode 0400 +``` + +`official install` builds the bundled catalog; `official update` accepts only +the same catalog ID and a non-decreasing sequence, rejects a same-sequence +content rewrite, verifies every source size/hash, creates a new release, and +atomically moves the official pointer. `disable` removes only that pointer; +`uninstall` deletes only release IDs registered by official state plus its +catalog/source cache. + +Some official download endpoints return a JSON envelope rather than the source +binary. The current adapter recognizes only the National Laws and Regulations +Database endpoint, caps the envelope at 64 KiB, requires its exact HTTPS object +storage host and matching file suffix, then still verifies the catalog byte size +and SHA-256. Transient 429/5xx failures receive three bounded retries; integrity +failures are never retried into acceptance. + +Private add/delete never touches the official pointer, state, source cache, or +release directories. A mutation rebuilds a new private snapshot and removes +obsolete private snapshots/sources. The MCP process checks the private ACTIVE +epoch before each private read, so a process pinned before deletion fails +closed until restart instead of continuing to expose the deleted snapshot. +Private receipts are snapshot-scoped and intentionally cease to verify after a +mutation removes the prior snapshot. + +For a managed official catalog, the MCP process performs the same epoch check +before every official read. Update, disable, or uninstall can finish an already +executing read on its pinned connection, but later calls from that old process +fail closed until restart. An explicit `DEEPLAW_DB` is host-pinned and therefore +uses the host's own release/connection epoch policy instead of the local ACTIVE +guard. + ### Stable identifiers - `lawrel_*`: derived from source/metadata, normalized segment hashes, @@ -258,9 +319,16 @@ enter primary evidence; unknown or unreviewed dates are isolated in `uncertain_evidence`, and known out-of-scope material is excluded. This is research assistance, not an applicability ruling. +Extraction admission is independent of temporal admission. A document carrying +`extraction_review_required=true` is isolated in `uncertain_evidence` even when +its temporal classification would otherwise pass. Its evidence card retains the +extractor, configuration, warnings, locator, and source hash for source comparison. +The current v4 storage flag is document-level, so this safety gate can be broader +than the affected pages; page-scoped admission remains a later schema refinement. + The current model accepts document numbers, aliases, promulgation dates, jurisdiction, effective intervals, issuer/status fields, and hash-bound review -metadata. SQLite v4 includes `legal_edges`, but the `0.2.0` runtime produces only +metadata. SQLite v4 includes `legal_edges`, but the `0.3.0` runtime produces only `deterministic_exact` edges derived from an exact known-document-name reference in a source segment. Review-overlay relations are hash-bound governance proposals; they are not inserted into the runtime graph, and no current producer @@ -296,9 +364,10 @@ review accepts it. The provider-facing schemas live in [`contracts`](../contracts): -- DeepLaw `0.2.0` keeps `law-support.input` and verification at v1; the advertised - output union, search response, segment, release-info, evidence-card, and corpus - release-manifest contracts are v2. +- DeepLaw `0.3.0` advertises `law-support.input` v2 for the eight official/private + operations and retains input v1 as the official-only compatibility contract. + Verification remains v1; the output union, search response, segment, + release-info, evidence-card, and corpus release-manifest contracts are v2. - `LegalEvidenceCardV2` returns the release, receipt, stable IDs, title, issuer, source URL/hash, segment hash, locator, effective interval, status, extraction method/configuration/review warnings, score, hit reason, and bounded excerpt. @@ -309,7 +378,9 @@ The provider-facing schemas live in [`contracts`](../contracts): release, document, segment, stored source hash, and stored segment hash. It does not reopen or rehash the original DOCX/PDF. - `law-support.output` is a closed union of search, segment, verification, and - release-info schemas; hosts must reject unknown output fields. + release-info schemas. Official and private operations reuse these bounded + result shapes, while private release metadata binds `collection_scope` and + `library_id`; hosts must reject unknown output fields. This two-stage pattern prevents broad queries such as `诈骗` from inserting dozens of semantically related provisions into the model context. Search is a @@ -319,7 +390,7 @@ selection operation; `get` is a deliberate evidence-read operation. [`src/deeplaw/mcp_server.py`](../src/deeplaw/mcp_server.py) uses the MCP SDK's low-level `Server` and local stdio transport to expose one tool, `law_support`, -with four read-only operations. Its advertised output schema is a bundled, +with eight read-only operations. Its advertised output schema is a bundled, closed copy of the repository contracts, so clients do not need to resolve remote schema URLs during the handshake: @@ -329,6 +400,10 @@ remote schema URLs during the handshake: | `get` | Fetch one exact segment | `segment_id` | | `verify` | Verify a returned receipt | `segment_id` and `receipt_id` | | `release_info` | Inspect the active release | None | +| `private_search` | Search only the user-private legal-reference snapshot | Query and optional purpose/date filters | +| `private_get` | Fetch one exact private segment | `segment_id` | +| `private_verify` | Verify one private snapshot receipt | `segment_id` and `receipt_id` | +| `private_info` | Inspect the active private snapshot | None | Some hosts display a transport-qualified name such as `mcp__deeplaw__law_support`. The `mcp__deeplaw__` portion is host routing @@ -347,7 +422,10 @@ Host integration must follow these rules: - Do not invoke it merely because an unrelated dataset contains words such as `诈骗`, `合同`, or `法院`. - Send a de-identified legal issue, citation, or date filter; do not send a full - case record or personal identifiers to the public service. + case record or personal identifiers to either DeepLaw scope. +- Use `private_*` only after an explicit request for the user's private legal + references. Query official and private scopes separately and label them; + never merge their ranking or authority. - Keep case-private attachments, conversation state, and analysis in the host's case project storage. - Treat an unavailable or unverified release as a visible failure; do not fall @@ -371,7 +449,7 @@ Derived layers are permitted only when all of the following hold: segment; - evaluated against exact/lexical-only retrieval before activation. -The `0.2.0` deterministic graph supports only `cites`, `amends`, `repeals`, +The `0.3.0` deterministic graph supports only `cites`, `amends`, `repeals`, `replaces`, `implements`, and `exception_to`. Each runtime edge is `deterministic_exact` and retains the source segment and evidence hash. Relations declared in a review overlay remain governance proposals; the current runtime @@ -404,8 +482,9 @@ DeepLaw must not: - infer repeal or legal effect solely from an embedding or LLM output; - return unlimited vector top-k context; - use generated summaries as authoritative citations; -- accept case-private writes through MCP; -- share one mutable database between public law and case projects; +- accept any writes through MCP; +- share one mutable database between official law, private legal references, + or case projects; - hide missing evidence by answering from model memory; - market benchmark superiority without reproducible evidence. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 145cbb9..6320898 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -1,29 +1,32 @@ # DeepLaw 评测说明 -## 当前 0.2 候选结果 +## 当前 0.3 候选结果 -2026-07-15 使用当前 `deeplaw.release/v2` / `deeplaw.sqlite/v4` 本地候选运行 32 项白盒 +2026-07-16 使用当前 `deeplaw.release/v2` / `deeplaw.sqlite/v4` 本地候选运行 32 项白盒 smoke set，结果记录在 -[`benchmarks/core-v2-candidate-2026-07-15.json`](../benchmarks/core-v2-candidate-2026-07-15.json)： +[`benchmarks/core-v3-candidate-2026-07-16.json`](../benchmarks/core-v3-candidate-2026-07-16.json)： - 32/32 同时通过检索目标与噪声/上下文约束； +- 其中 6 项明确要求命中的 5 份抽取风险 PDF 只能出现在 `uncertain_evidence`，不能进入主证据； - 30 个有排名目标的 case 为 Hit@1 1.0、MRR 1.0； - 平均 evidence excerpt 679.719 字符； - 109/109 张返回证据的 receipt 往返核验通过率为 1.0； -- 已打开本地数据库后的 `law.search()` 本机延迟为 p50 12.337 ms、p95 18.322 ms；数据库 +- 已打开本地数据库后的 `law.search()` 本机延迟为 p50 12.000 ms、p95 17.250 ms；数据库 打开、receipt 往返断言、JSON 序列化和 MCP transport 均不包含在该延迟中； - 精确题名聚焦、法名 + 条号、未来时点负例、历史标题纠偏、OCR review flag 和泛词预算均被 固定断言覆盖； -- 两个 expected title 随 hash-bound overlay 的安全标题纠偏同步更新，cases SHA-256 因此变为 - `f60d88ab4b90e421b965e16ea85040edfab9dedb5cb387a1f92e20dd4ecc8d50`。 +- `expected_bucket` 将“检索到风险候选”和“准入主证据”拆开断言，cases SHA-256 为 + `95f52e14b11589850a3a7ecc57fb2bf4614a6be85a979a73869dd37973453625`。 报告绑定 release、database、source manifest、review overlay、case 文件、Python source tree、 关键实现文件和本机环境。语料二进制及 SQLite 不进入 Git；release 仍是 `partially_verified`、`restricted`、`ai_precheck`。成功只能证明这组已知语料白盒断言，不能 证明法律内容已获人工批准，也不能证明 DeepLaw 超过任何外部系统。 -`benchmarks/core-candidate-2026-07-15.json` 保留的是 0.1 / SQLite v3 历史快照，不代表当前 -实现。两份报告均为 `candidate_smoke_not_held_out`，不是盲测、留出集或独立专家金标。 +`benchmarks/core-v2-candidate-2026-07-15.json` 和 +`benchmarks/core-candidate-2026-07-15.json` 分别保留 0.2 / SQLite v4 与 0.1 / SQLite v3 +历史快照，不代表当前实现。三份报告均为 `candidate_smoke_not_held_out`，不是盲测、留出集或 +独立专家金标；旧报告绑定的是旧 case hash，不能用当前 case 文件重放后声称结果相同。 外部复现需要调用者合法取得确切 candidate 数据库，或用同一 source package、overlay 和 匹配构建实现重新生成。不同 release 必须作为新快照评测，不能沿用这里的数字。 @@ -43,11 +46,11 @@ deeplaw eval \ 评测器检查： -- expected title/article 是否出现及其 rank； -- `expected_empty` case 是否确实没有返回证据； -- forbidden title 是否被错误返回； +- expected title/article 是否出现在明确指定的 `evidence` 或 `uncertain_evidence` 分桶及其桶内 rank； +- `expected_empty` case 是否在两个分桶都没有返回候选； +- forbidden title 是否在任一分桶被错误返回； - expected route； -- evidence 数量和 excerpt 字符预算（不是完整序列化响应的字节预算）； +- 两个分桶合计数量和 excerpt 字符预算（不是完整序列化响应的字节预算）； - 指定 case 的 `extraction_review_required` 标记； - 每张主证据和不确定证据的 receipt、release、source hash 与 segment hash 往返核验； - retrieval、constraint 和 overall pass rate； diff --git a/docs/CORPUS_GOVERNANCE.md b/docs/CORPUS_GOVERNANCE.md index 8bb6db9..9d4a958 100644 --- a/docs/CORPUS_GOVERNANCE.md +++ b/docs/CORPUS_GOVERNANCE.md @@ -2,8 +2,10 @@ ## 目的与适用范围 -DeepLaw 是跨 Agent 使用的公共中国法律研究底座。它提供的是可定位、可校验的法律 -研究材料，不替代法律效力审查、案件事实证明、法律适用判断或人工复核。 +DeepLaw 是跨 Agent 使用的中国法律研究底座。它包含团队维护的官方目录，以及物理分离、 +仅本机操作系统用户可管理的私有法律参考资料库。两者都提供可定位、可校验的研究材料， +但只有完成官方发布治理的资料才可能成为 DeepLaw 官方法源；用户私有资料始终是未经官方 +审核的参考候选。DeepLaw 不替代法律效力审查、案件事实证明、法律适用判断或人工复核。 本政策适用于： @@ -13,8 +15,9 @@ DeepLaw 是跨 Agent 使用的公共中国法律研究底座。它提供的是 - SQLite release、搜索索引、证据卡和 release 元数据； - 本地构建、分发、激活、撤销和更新流程。 -案件证据、案件聊天、工作笔记、身份信息、交易数据和案件项目私有文档不属于 -DeepLaw 公共语料，任何情况下都不得进入公共 release、缓存、日志、评测集或派生专题页。 +案件证据、案件聊天、工作笔记、身份信息、交易数据和案件项目私有文档不属于 DeepLaw +任何语料，任何情况下都不得进入官方或用户私有 release、缓存、日志、评测集或派生专题页。 +用户私有库只接收法律参考资料，不能作为 Analytix 案件项目库的替代品。 ## 权威层级 @@ -41,6 +44,25 @@ DeepLaw 按以下顺序判断资料权威性： MCP 运行时不得拥有构建和发布权限。`build`、OCR、人工复核和激活属于离线管理路径， 不能作为 Agent 可调用工具暴露。 +## 三个不可合并的所有权域 + +| 范围 | 所有者 | 可写面 | 权威与更新 | +| --- | --- | --- | --- | +| 官方团队目录 | DeepLaw 发布团队 | 离线 `official install/update/enable/disable/uninstall` | catalog sequence 单调递增；每次更新新建不可变 release；用户可停用/卸载本机副本 | +| 用户私有法律资料 | 当前本机 OS 用户 | 离线 `private add/delete` | owner-only 根目录；始终 `authorityRank=0`、`status=unknown`；不得进入官方目录 | +| Analytix 案件项目 | 对应案件项目与 Analytix 宿主 | Analytix 自己的项目 API/SQLite/DuckDB | 完全不属于 DeepLaw；DeepLaw 不读取、不索引、不更新 | + +官方与私有范围分别使用 ACTIVE 指针、source cache、release 目录和 MCP operation。不得通过 +同库 `scope` 字段、`case_id` filter 或统一候选排序模拟隔离。官方更新不能读取、删除、提升或 +重排用户私有资料；用户私有增删也不能修改官方状态或 release。 + +当前“仅该用户访问”由本地 OS 账户和 owner-only 权限实现：私有目录 `0700`，状态/源文件 +`0600`，只读数据库 `0400`。它不是共享服务的多租户认证。共享目录、共享容器卷或多人服务 +必须由宿主提供身份认证，并为每个用户配置独立物理 `DEEPLAW_HOME`；否则不满足该承诺。 + +私有导入要求 `--confirm-no-case-data`，禁止目录扫描、后台学习和 Agent MCP 写入。该确认只是 +防误操作门禁，不是内容级 DLP；宿主仍须在导入前阻止案件事实、附件、会话和身份数据。 + ## Release 生命周期 每个 release 必须处于下列状态之一： @@ -60,7 +82,7 @@ active -> superseded | revoked - `revoked`：发现完整性、来源、许可或重大内容错误；停止新查询并保留审计记录。 - `rejected`：未通过发布闸门，不能通过 `--allow-needs-ocr` 等开发选项绕过。 -当前 v0.2.0 构建器会校验路径、大小、SHA-256、HTTPS 来源和 PDF 文本质量，但尚未机械 +当前 v0.3.0 构建器会校验路径、大小、SHA-256、HTTPS 来源和 PDF 文本质量，但尚未机械 执行本政策的全部人工闸门。`deeplaw build --activate` 只改变本地 ACTIVE 指针，不是 许可、隐私、时效或法律有效性证明。生产发布流程必须在其外层实施本政策。 @@ -102,6 +124,8 @@ active -> superseded | revoked `pdftoppm` 版本、`dpi=300`、`language=chi_sim+eng`、`psm=3`，并为每页记录图像、原生 文本、OCR 文本和最终文本 hash、字符数、OCR confidence、native/OCR consistency、风险 标志和复核状态。任何低质量或分歧页面在人工审校关闭前仍未完整通过 G2。 +当前运行时把所属文档标为 `extraction_review_required`，并将其检索结果放入 +`uncertain_evidence`；构建时显式允许保留候选文本，不等于允许它进入已核验主证据。 ### G3：结构和引文可复核性 @@ -125,7 +149,7 @@ active -> superseded | revoked 修改决定不能由 LLM 自动合并为现行文本；如生成合并视图，必须保存确定性变换记录、 逐条来源和人工批准。日期过滤只说明候选时间相符，不自动证明案件适用。 -review overlay 中的关系是绑定来源 hash 的治理提案。v0.2.0 运行时不会把这些提案写入 +review overlay 中的关系是绑定来源 hash 的治理提案。v0.3.0 运行时不会把这些提案写入 `legal_edges`，也没有生成 `reviewed` 边的生产路径；当前运行时图只包含从来源 segment 中 精确文件名引用派生并保留 segment/hash provenance 的 `deterministic_exact` 导航边。 @@ -171,14 +195,18 @@ review overlay 中的关系是绑定来源 hash 的治理提案。v0.2.0 运行 - 当前 stdio MCP 进程在 lifespan 启动时解析并校验一次 `ACTIVE` 或 `DEEPLAW_DB`，随后在 整个进程生命周期固定并复用该只读 release；切换 `ACTIVE` 只影响新进程，不能热切换 已运行进程。 +- 私有库删除具有更严格的撤销语义：每次 `private_*` 读取前比较当前私有 ACTIVE 与进程固定 + release；发生变化即拒绝继续读取并要求重启，避免删除后旧进程继续暴露旧快照。 - 回滚通过切换到既有已批准 release 完成，不修改旧 release。 ## 只读运行时要求 - SQLite 以只读、immutable 模式打开；低层 MCP server 只通过本地 stdio 提供 - `search`、`get`、`verify` 和 `release_info`，不监听 HTTP。 + 官方 `search/get/verify/release_info` 与私有 `private_search/private_get/private_verify/private_info`， + 不监听 HTTP。 - 不提供 `add`、`update`、`remember`、`feedback`、`topic_write` 或后台学习工具。 -- 查询、模型输出和用户反馈不能写回公共 corpus。 +- 查询、模型输出和用户反馈不能写回官方或私有 corpus。`official update` 与 `private add/delete` + 只存在于人工显式运行的本机 CLI 管理面。 - 运行时不得下载模型、抓取网页或在校验失败时回退到模型记忆。 - 法律 release 不得直接打开案件 SQLite/DuckDB，也不得接受案件路径、`caseId` 或完整事实。 @@ -213,9 +241,15 @@ Tesseract/Poppler 只由显式选择的离线 `vision-consensus` 构建路径调 写入案件材料或敏感正文。客户端必须显式显示 release 不可用，不能静默切换到网络搜索或 模型记忆后继续声称是现行法。 +官方更新目录使用固定 catalog ID、递增 sequence 和整份 catalog SHA-256；拒绝降序更新和 +“同 sequence 改内容”。客户端经 HTTPS 取得目录与原件，并逐文件校验字节数/SHA-256。 +当前 v0.3.0 尚无公钥签名，因此 GitHub 团队账号与 HTTPS 仍属于信任根；在签名 catalog、 +撤销列表和密钥轮换落地前，不得把它描述为抗仓库接管的安全更新通道。 + ## 禁止事项 - 把公共法源与案件私有库合并到同一数据库或全局向量库； +- 把官方目录与用户私有法律资料合并到同一数据库、候选池或排序； - 用 `case_id` filter 代替物理隔离； - 把 Obsidian Vault、Downloads 或整个工作区作为自动摄取根目录； - 覆盖、修补或重新保存官方原件后仍沿用旧 hash； diff --git a/docs/DEEPLAW_2.md b/docs/DEEPLAW_2.md index 0217597..2de031b 100644 --- a/docs/DEEPLAW_2.md +++ b/docs/DEEPLAW_2.md @@ -6,15 +6,16 @@ Status: architecture target and research plan, 2026-07-15. > 文件进入，Agent 得到可验证的知识。 `DeepLaw` 是产品名，Architecture 2.0 是下一代架构方向，不是当前 Python 包版本号。当前可 -运行实现仍是 `0.2.0` alpha；本文严格区分已经存在的能力与后续研究目标。 +运行实现仍是 `0.3.0` alpha；本文严格区分已经存在的能力与后续研究目标。 ## DeepLaw 是什么 -DeepLaw 是面向 Agent 的公共知识库。Architecture 2.0 目标支持 DOCX、PDF、TXT 等文件，处理为 +DeepLaw 是面向 Agent 的法律知识库。Architecture 2.0 目标支持 DOCX、PDF、TXT 等文件，处理为 只读、版本化、可追溯的 Knowledge Release，并向 Agent 交付小型 Evidence Pack；当前 -`0.2.0` 输入仅为 DOCX 与 PDF。 +`0.3.0` 官方团队目录输入为 DOCX/PDF，物理分离的用户私有法律参考库另支持 UTF-8 TXT。 -DeepLaw 不是聊天记忆，不保存案件私有资料，也不把整座知识库塞进模型上下文。它解决四个 +DeepLaw 不是聊天记忆，不保存案件项目私有资料，也不把整座知识库塞进模型上下文。用户私有 +DeepLaw 范围只保存法律参考资料，始终标记为用户提供且未经官方审核。它解决四个 问题： 1. 文件中的知识如何稳定进入知识库； @@ -22,8 +23,9 @@ DeepLaw 不是聊天记忆，不保存案件私有资料，也不把整座知识 3. 哪些内容满足当前问题的证据要求； 4. 哪些地方仍然缺失、不确定或不能使用。 -公共/私有隔离是宿主集成必须执行的边界。当前 `0.2.0` 不提供内容级 DLP 或案件私有资料 -分类器；宿主必须在调用 DeepLaw 前完成隔离和拒绝，不能依赖 DeepLaw 自动识别误传内容。 +官方目录、用户私有法律参考与 Analytix 案件项目必须物理隔离。当前 `0.3.0` 不提供内容级 +DLP 或案件私有资料分类器；宿主必须在调用或导入 DeepLaw 前完成隔离和拒绝，不能依赖 +DeepLaw 自动识别误传内容。 ## 五秒架构 @@ -101,7 +103,7 @@ Agent 的问题先被编译成一个封闭的证据任务，而不是直接取 ### 4. Evidence Pack -Evidence Pack 是 Agent 实际看到的有界结果。下列是 Architecture 2.0 目标合同；`0.2.0` 已有 +Evidence Pack 是 Agent 实际看到的有界结果。下列是 Architecture 2.0 目标合同；`0.3.0` 已有 `query_plan`、`evidence`、`uncertain_evidence`、`obligation_coverage` 与 `gaps`，`receipt_id` 位于每张 evidence card 内。顶层 `receipt_ids`、`trace_hash`，以及选择前 coverage witness 均为 2.0 目标： @@ -178,7 +180,7 @@ Question → evidence + uncertain + gaps + receipts ``` -当前 `0.2.0` 已实现 QueryPlan、八类 Duty、严格时效分桶、事后 coverage、gaps 和 receipt。 +当前 `0.3.0` 已实现 QueryPlan、八类 Duty、严格时效分桶、事后 coverage、gaps 和 receipt。 2.0 的关键增量是把选择顺序改为 coverage-first，并使每个 `covered` 状态携带机器可检查的 witness。 @@ -275,28 +277,29 @@ redistribution_status 1. **Provenance integrity**：所有主证据解析到当前 release 的唯一 segment，hash 全部匹配。 2. **No authority escalation**：派生索引、模型和排序器不能提高权威、时效或审核状态。 3. **Temporal admission**：`as_of=t` 时，未知时效只能进入 uncertain，区间外不得进入主证据。 -4. **Coverage witness**：没有通过 capability predicate 的 witness 就不能标记 `covered`。 -5. **Map non-entailment**：关系路径只能提出候选或 challenge，不能单独覆盖法律 Duty。 -6. **Bounded context**：任何插件都不能突破卡片、字符、路径和 hop 上限。 -7. **Deterministic replay**：相同输入和版本产生相同 IDs、gaps 和 trace hash。 -8. **Exact stability**：添加无关文档或派生索引不能挤掉有效精确匹配。 -9. **Gap monotonicity**：低信任或 derived 材料不能消除 blocking gap。 -10. **Scope honesty**：无结果只描述声明范围，不能断言知识不存在。 -11. **Public/private isolation**：公共 release、sidecar、日志和 benchmark 不持久化案件事实。 +4. **Extraction admission**：未关闭抽取复核标记的候选只能进入 uncertain，不能成为主证据。 +5. **Coverage witness**：没有通过 capability predicate 的 witness 就不能标记 `covered`。 +6. **Map non-entailment**：关系路径只能提出候选或 challenge，不能单独覆盖法律 Duty。 +7. **Bounded context**：任何插件都不能突破卡片、字符、路径和 hop 上限。 +8. **Deterministic replay**：相同输入和版本产生相同 IDs、gaps 和 trace hash。 +9. **Exact stability**：添加无关文档或派生索引不能挤掉有效精确匹配。 +10. **Gap monotonicity**：低信任或 derived 材料不能消除 blocking gap。 +11. **Scope honesty**：无结果只描述声明范围，不能断言知识不存在。 +11. **Scope isolation**：官方、用户私有法律参考、案件项目物理分离；日志和 benchmark 不持久化案件事实。 12. **Model non-interference**：替换模型可改变发现顺序，不能改变权威、时效、hash 或 receipt。 ## 当前实现与 2.0 目标 -| 能力 | `0.2.0` 当前实现 | Architecture 2.0 目标 | +| 能力 | `0.3.0` 当前实现 | Architecture 2.0 目标 | | --- | --- | --- | -| Ingest | DOCX/PDF、页级证据、content-addressed SQLite | 完整 Corpus Coverage Manifest | +| Ingest | 官方 DOCX/PDF、私有 UTF-8 TXT、页级证据、content-addressed SQLite | 官方 TXT 与完整 Corpus Coverage Manifest | | Organize | 标题/条文分段、顺序、稳定 segment、来源关系 | 完整法律层级、双时态事件账本和双 Map 隔离 | | Locate | 精确题名/文号/条款、中文 FTS | 可插拔本地发现 sidecar | | Connect | provenance-bound 单跳关系 | 有限 challenge closure | | Explain | 有界 excerpt 与 next questions | release-pinned、source-bound navigation | | Verify | 时效分桶、gaps、receipt | capability types、witness、replay trace | | Select | 排序后截取，再计算 coverage | coverage-first 最小充分证据集 | -| Agent | Skill 与只读 MCP 边界 | host schema 物化前显式激活 | +| Agent | Skill 与单一只读 MCP；官方/私有 operation 分离 | host schema 物化前显式激活 | ## 如何证明技术领先 @@ -353,7 +356,7 @@ DeepLaw 不用口号证明自己。外部主张需要在相同语料、问题、 ### Phase 4：Input, Hierarchy And Discovery Sidecars -- 增加确定性 TXT extractor，并保留 locator、hash 和抽取证据； +- 把已用于私有库的确定性 TXT extractor 纳入官方治理流程，并保留 locator、hash 和抽取证据； - 把当前标题/条文分段扩展为可校验的完整法律层级； - 按实验逐一增加本地语义发现、Discovery Map 和 source-bound Explain； - 每个 sidecar 必须固定到 release、可删除重建，并有 ablation；没有净增益就不进入默认路径。 diff --git a/evals/core-2026-07-14.jsonl b/evals/core-2026-07-14.jsonl index 94814d9..6b23b54 100644 --- a/evals/core-2026-07-14.jsonl +++ b/evals/core-2026-07-14.jsonl @@ -16,17 +16,17 @@ {"id":"personal-information-law-article","query":"中华人民共和国个人信息保护法 第十三条","purpose":"exact_citation","expected_titles":["中华人民共和国个人信息保护法"],"expected_articles":["第十三条"],"max_evidence":3,"max_excerpt_chars":3500} {"id":"data-security-law-article","query":"中华人民共和国数据安全法 第二十一条","purpose":"exact_citation","expected_titles":["中华人民共和国数据安全法"],"expected_articles":["第二十一条"],"max_evidence":3,"max_excerpt_chars":3500} {"id":"economic-crime-procedure","query":"公安机关办理经济犯罪案件的若干规定 管辖","purpose":"legal_issue_screen","expected_titles":["最高人民检察院、公安部关于公安机关办理经济犯罪案件的若干规定"],"max_evidence":5,"max_excerpt_chars":3500} -{"id":"criminal-case-procedure","query":"公安机关办理刑事案件程序规定 冻结","purpose":"legal_issue_screen","expected_titles":["公安机关办理刑事案件程序规定"],"max_evidence":5,"max_excerpt_chars":3500} -{"id":"seized-property-ocr","query":"涉案财物 查封 扣押 冻结 三日内解除","purpose":"legal_issue_screen","expected_titles":["关于进一步规范刑事诉讼涉案财物处置工作的意见"],"expected_extraction_review_required":true,"max_evidence":5,"max_excerpt_chars":3500} +{"id":"criminal-case-procedure","query":"公安机关办理刑事案件程序规定 冻结","purpose":"legal_issue_screen","expected_titles":["公安机关办理刑事案件程序规定"],"expected_bucket":"uncertain_evidence","expected_extraction_review_required":true,"max_evidence":5,"max_excerpt_chars":3500} +{"id":"seized-property-ocr","query":"涉案财物 查封 扣押 冻结 三日内解除","purpose":"legal_issue_screen","expected_titles":["关于进一步规范刑事诉讼涉案财物处置工作的意见"],"expected_bucket":"uncertain_evidence","expected_extraction_review_required":true,"max_evidence":5,"max_excerpt_chars":3500} {"id":"criminal-procedure-interpretation","query":"最高人民法院 刑事诉讼法解释 涉案财物","purpose":"legal_issue_screen","expected_titles":["最高人民法院关于适用《中华人民共和国刑事诉讼法》的解释"],"max_evidence":5,"max_excerpt_chars":3500} -{"id":"foreign-exchange-regulation","query":"外汇管理条例 逃汇","purpose":"legal_issue_screen","expected_titles":["中华人民共和国外汇管理条例"],"max_evidence":5,"max_excerpt_chars":3500} +{"id":"foreign-exchange-regulation","query":"外汇管理条例 逃汇","purpose":"legal_issue_screen","expected_titles":["中华人民共和国外汇管理条例"],"expected_bucket":"uncertain_evidence","expected_extraction_review_required":true,"max_evidence":5,"max_excerpt_chars":3500} {"id":"pboc-amend-repeal","query":"中国人民银行 修改和废止部分规章 2025","purpose":"exact_citation","expected_titles":["中国人民银行关于修改和废止部分规章的决定"],"max_evidence":5,"max_excerpt_chars":3500} {"id":"beneficial-owner","query":"受益所有人信息管理办法 备案","purpose":"legal_issue_screen","expected_titles":["受益所有人信息管理办法"],"max_evidence":5,"max_excerpt_chars":3500} -{"id":"aml-supervision","query":"金融机构反洗钱和反恐怖融资监督管理办法","purpose":"exact_citation","expected_titles":["金融机构反洗钱和反恐怖融资监督管理办法（2021年发布文本）"],"max_evidence":5,"max_excerpt_chars":3500} +{"id":"aml-supervision","query":"金融机构反洗钱和反恐怖融资监督管理办法","purpose":"exact_citation","expected_titles":["金融机构反洗钱和反恐怖融资监督管理办法（2021年发布文本）"],"expected_bucket":"uncertain_evidence","expected_extraction_review_required":true,"max_evidence":5,"max_excerpt_chars":3500} {"id":"large-suspicious-transactions","query":"金融机构大额交易和可疑交易报告管理办法","purpose":"exact_citation","expected_titles":["金融机构大额交易和可疑交易报告管理办法（2016年发布文本，未整合后续修改）"],"max_evidence":5,"max_excerpt_chars":3500} {"id":"customer-record-retention","query":"金融机构客户尽职调查 客户身份资料 交易记录保存","purpose":"legal_issue_screen","expected_titles":["金融机构客户尽职调查和客户身份资料及交易记录保存管理办法"],"max_evidence":5,"max_excerpt_chars":3500} -{"id":"nonbank-payment-regulation","query":"非银行支付机构监督管理条例 备付金","purpose":"legal_issue_screen","expected_titles":["非银行支付机构监督管理条例"],"max_evidence":5,"max_excerpt_chars":3500} -{"id":"nonbank-payment-regulation-article","query":"非银行支付机构监督管理条例 第二十七条","purpose":"exact_citation","expected_titles":["非银行支付机构监督管理条例"],"expected_articles":["第二十七条"],"forbidden_titles":["非银行支付机构监督管理条例实施细则"],"max_evidence":3,"max_excerpt_chars":3500} +{"id":"nonbank-payment-regulation","query":"非银行支付机构监督管理条例 备付金","purpose":"legal_issue_screen","expected_titles":["非银行支付机构监督管理条例"],"expected_bucket":"uncertain_evidence","expected_extraction_review_required":true,"max_evidence":5,"max_excerpt_chars":3500} +{"id":"nonbank-payment-regulation-article","query":"非银行支付机构监督管理条例 第二十七条","purpose":"exact_citation","expected_titles":["非银行支付机构监督管理条例"],"expected_articles":["第二十七条"],"expected_bucket":"uncertain_evidence","expected_extraction_review_required":true,"forbidden_titles":["非银行支付机构监督管理条例实施细则"],"max_evidence":3,"max_excerpt_chars":3500} {"id":"nonbank-payment-rules","query":"非银行支付机构监督管理条例实施细则","purpose":"exact_citation","expected_titles":["非银行支付机构监督管理条例实施细则"],"max_evidence":5,"max_excerpt_chars":3500} {"id":"tax-crime-interpretation","query":"危害税收征管刑事案件 司法解释 虚开","purpose":"legal_issue_screen","expected_titles":["最高人民法院、最高人民检察院关于办理危害税收征管刑事案件适用法律若干问题的解释"],"max_evidence":5,"max_excerpt_chars":3500} {"id":"future-rule-as-of-negative","query":"金融机构客户尽职调查和客户身份资料及交易记录保存管理办法","purpose":"as_of_version","as_of":"2025-07-01","forbidden_titles":["金融机构客户尽职调查和客户身份资料及交易记录保存管理办法"],"expected_empty":true,"max_evidence":0,"max_excerpt_chars":0} diff --git a/plugins/deeplaw/.claude-plugin/plugin.json b/plugins/deeplaw/.claude-plugin/plugin.json index 492033e..160529d 100644 --- a/plugins/deeplaw/.claude-plugin/plugin.json +++ b/plugins/deeplaw/.claude-plugin/plugin.json @@ -2,8 +2,8 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "deeplaw", "displayName": "DeepLaw", - "version": "0.2.0", - "description": "Explicit-use, read-only Agent Knowledge Base for versioned Chinese legal sources.", + "version": "0.3.0", + "description": "Explicit-use, read-only Agent Knowledge Base for official and user-private Chinese legal sources.", "author": { "name": "Eysn0130", "url": "https://github.com/Eysn0130" diff --git a/plugins/deeplaw/.codex-plugin/plugin.json b/plugins/deeplaw/.codex-plugin/plugin.json index a95f8ed..1d0ff1b 100644 --- a/plugins/deeplaw/.codex-plugin/plugin.json +++ b/plugins/deeplaw/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "deeplaw", - "version": "0.2.0", - "description": "Read-only Agent Knowledge Base for versioned Chinese legal sources.", + "version": "0.3.0", + "description": "Read-only Agent Knowledge Base for official and user-private Chinese legal sources.", "author": { "name": "Eysn0130", "url": "https://github.com/Eysn0130" @@ -21,7 +21,7 @@ "interface": { "displayName": "DeepLaw", "shortDescription": "Verify Chinese legal sources, versions, and citations", - "longDescription": "DeepLaw turns versioned Chinese legal sources into a bounded, read-only Agent Knowledge Base. It returns source-backed evidence cards, explicit gaps, receipts, and exact segments without adding case-private memory to ordinary agent work.", + "longDescription": "DeepLaw turns versioned Chinese legal sources into a bounded, read-only Agent Knowledge Base. A team-maintained official catalog and a physically separate per-user legal-reference library return evidence cards, explicit gaps, receipts, and exact segments without taking ownership of case projects.", "developerName": "Eysn0130", "category": "Productivity", "capabilities": [ diff --git a/plugins/deeplaw/skills/research-chinese-law/SKILL.md b/plugins/deeplaw/skills/research-chinese-law/SKILL.md index 4b4bb92..19b9699 100644 --- a/plugins/deeplaw/skills/research-chinese-law/SKILL.md +++ b/plugins/deeplaw/skills/research-chinese-law/SKILL.md @@ -5,9 +5,12 @@ description: "Use only after the user explicitly invokes this skill to retrieve # DeepLaw Chinese-Law Research -DeepLaw is a read-only legal-source substrate. Use it to collect bounded, -version-aware evidence. Do not use it to decide case facts, replace legal review, -or inject a general legal corpus into every conversation. +DeepLaw is a read-only Agent legal-knowledge substrate. Use its official catalog +to collect bounded, version-aware evidence. It may also read a physically +separate user-private legal-reference library, but those results are +user-provided and never official DeepLaw sources. Do not use either scope to +decide case facts, replace legal review, or inject a general legal corpus into +every conversation. ## Enforce the invocation gate @@ -37,20 +40,32 @@ their own server or plugin prefix to that name. The prefix is not a second tool. Do not use any other DeepLaw tool. If the server advertises a different leaf name or more than one tool, stop and report an adapter/runtime contract mismatch. -`law_support` routes four read-only operations: +`law_support` routes eight read-only operations: - `search`: return a bounded evidence-card set; - `get`: fetch one exact segment selected by `segment_id`; - `verify`: verify one `segment_id` and `receipt_id` pair; - `release_info`: inspect the active immutable release. +- `private_search`: search only the user-private legal-reference snapshot; +- `private_get`: fetch one exact private segment; +- `private_verify`: verify one private snapshot receipt; +- `private_info`: inspect the current private snapshot. -Never ask for or invent a corpus-write, memory-write, upload, delete, reindex, or -administration operation. +Use `private_*` only when the user explicitly asks to use their DeepLaw private +legal-reference library. Never infer private scope from a workspace, filename, +or case project. Do not combine official and private candidates into one ranking +or authority conclusion; query and label the two scopes separately when both +are explicitly requested. + +Never ask for or invent an MCP corpus-write, memory-write, upload, delete, +reindex, or administration operation. Private add/delete and official updates +are out-of-band local CLI administration, not Agent tools. ## Minimize private facts before retrieval -DeepLaw is a public legal-source library, not case storage. Convert case facts -into the smallest abstract legal issue that can retrieve the rule. Remove names, +DeepLaw is not case storage. Its private scope is for the user's legal reference +materials, not case evidence, facts, attachments, chats, or memory. Convert case +facts into the smallest abstract legal issue that can retrieve the rule. Remove names, identity numbers, account numbers, phone numbers, addresses, filenames, internal case IDs, quoted chats, and attachment contents. Do not send a whole case summary. @@ -79,7 +94,7 @@ persist private facts in DeepLaw. text for every hit. 5. Call `verify` for each segment that will support a material citation. Do not verify unused candidates. -6. Call `release_info` only when corpus provenance is requested or release state +6. Call `release_info` or `private_info` only when corpus provenance is requested or release state itself is material. Do not add it as a routine token cost. For an explicit one-word topic, keep `purpose: auto`, return a short navigation @@ -90,6 +105,9 @@ wide semantic top-k. - Separate retrieved normalized segment text from the agent's interpretation, and retain the official-source locator for comparison. +- For every `private_*` result, state that it is user-provided and unreviewed by + the DeepLaw team. A `private://source/...` locator is an integrity binding, not + an official download URL or authority claim. - Identify the release, source title, document number when present, exact article or locator, status, effective interval, and official source URL. - State when temporal applicability or authority relationships require human diff --git a/pyproject.toml b/pyproject.toml index 6119f7d..75818d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "deeplaw" -version = "0.2.0" -description = "Verifiable, read-only agent knowledge base for Chinese legal sources" +version = "0.3.0" +description = "Verifiable agent knowledge base for official and user-private Chinese legal sources" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" @@ -45,6 +45,8 @@ packages = ["src/deeplaw"] [tool.hatch.build.targets.wheel.force-include] "contracts" = "deeplaw/contracts" +"catalogs" = "deeplaw/catalogs" +"governance" = "deeplaw/governance" [tool.pytest.ini_options] addopts = "-q" diff --git a/src/deeplaw/__init__.py b/src/deeplaw/__init__.py index 9431f94..1eaeba5 100644 --- a/src/deeplaw/__init__.py +++ b/src/deeplaw/__init__.py @@ -4,4 +4,4 @@ from .search import DeepLaw __all__ = ["DeepLaw", "SearchRequest"] -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/src/deeplaw/admin_lock.py b/src/deeplaw/admin_lock.py new file mode 100644 index 0000000..e641d34 --- /dev/null +++ b/src/deeplaw/admin_lock.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import os +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from functools import wraps +from pathlib import Path +from typing import Any, TypeVar, cast + +from .store import default_home + +_Result = TypeVar("_Result") + + +@contextmanager +def _admin_lock(root: Path, name: str) -> Iterator[None]: + if root.is_symlink(): + raise RuntimeError(f"DeepLaw home must not be a symbolic link: {root}") + root.mkdir(parents=True, exist_ok=True, mode=0o700) + if not root.is_dir(): + raise RuntimeError(f"DeepLaw home is not a directory: {root}") + lock_path = root / name + if lock_path.is_symlink(): + raise RuntimeError(f"DeepLaw administration lock must not be a symbolic link: {lock_path}") + flags = os.O_RDWR | os.O_CREAT + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(lock_path, flags, 0o600) + try: + os.chmod(lock_path, 0o600) + if os.name == "nt": + import msvcrt + + if os.fstat(descriptor).st_size == 0: + os.write(descriptor, b"\0") + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_LOCK, 1) + else: + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_EX) + yield + finally: + if os.name == "nt": + import msvcrt + + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + + +def administration_locked(name: str) -> Callable[[Callable[..., _Result]], Callable[..., _Result]]: + def decorate(function: Callable[..., _Result]) -> Callable[..., _Result]: + @wraps(function) + def wrapped(*args: Any, **kwargs: Any) -> _Result: + configured_home = kwargs.get("home") + root = ( + Path(configured_home).expanduser().absolute() + if configured_home is not None + else default_home().absolute() + ) + with _admin_lock(root, name): + return function(*args, **kwargs) + + return cast(Callable[..., _Result], wrapped) + + return decorate diff --git a/src/deeplaw/cli.py b/src/deeplaw/cli.py index 4eba36f..7877913 100644 --- a/src/deeplaw/cli.py +++ b/src/deeplaw/cli.py @@ -13,6 +13,19 @@ from .ingest import build_release from .mcp_server import run_mcp from .models import SearchRequest +from .official import ( + disable_official, + enable_official, + official_status, + sync_official, + uninstall_official, +) +from .private_library import ( + add_private_document, + delete_private_document, + list_private_documents, + resolve_private_database, +) from .search import DeepLaw, response_json from .store import database_sha256, default_home, resolve_active_database from .vision import ( @@ -86,6 +99,100 @@ def _parser() -> argparse.ArgumentParser: help="Use stdio transport (explicit alias for host plugin manifests)", ) + official = commands.add_parser( + "official", + help="Install, update, disable, or uninstall the team-maintained official catalog", + ) + official_commands = official.add_subparsers(dest="official_command", required=True) + for name, help_text in ( + ("install", "Install the bundled official catalog and activate its release"), + ("update", "Fetch a newer official catalog and build its release"), + ): + sync = official_commands.add_parser(name, help=help_text) + sync.add_argument("--catalog", help="Local catalog path or HTTPS catalog URL") + sync.add_argument( + "--source-root", + type=Path, + help="Use an existing verified source package instead of downloading source files", + ) + sync.add_argument( + "--pdf-fallback", + choices=("off", "vision-consensus"), + default=None, + ) + official_commands.add_parser("status", help="Show official catalog installation state") + official_commands.add_parser("enable", help="Enable the installed official release") + official_commands.add_parser("disable", help="Disable without modifying the release") + official_commands.add_parser("uninstall", help="Delete locally installed official data") + + private = commands.add_parser( + "private", + help="Manage a per-OS-user legal-reference library outside the official catalog", + ) + private_commands = private.add_subparsers(dest="private_command", required=True) + private_add = private_commands.add_parser("add", help="Add one private legal-reference file") + private_add.add_argument("--source", type=Path, required=True) + private_add.add_argument("--title") + private_add.add_argument( + "--document-type", + choices=( + "law", + "administrative_regulation", + "judicial_interpretation", + "prosecution_standard", + "departmental_rule", + "normative_document", + "case_reference", + ), + default="normative_document", + ) + private_add.add_argument("--issuer", default="用户提供（未经 DeepLaw 官方审核）") + private_add.add_argument("--effective-from") + private_add.add_argument("--effective-to") + private_add.add_argument( + "--confirm-no-case-data", + action="store_true", + help="Confirm this is a legal reference, not Analytix case material", + ) + private_add.add_argument( + "--pdf-fallback", + choices=("off", "vision-consensus"), + default="off", + ) + private_add.add_argument("--allow-needs-ocr", action="store_true") + + private_delete = private_commands.add_parser("delete", help="Delete one private document") + private_delete.add_argument("--document-id", required=True) + private_delete.add_argument( + "--pdf-fallback", + choices=("off", "vision-consensus"), + default="off", + ) + private_delete.add_argument("--allow-needs-ocr", action="store_true") + private_commands.add_parser("list", help="List private legal-reference documents") + private_commands.add_parser("status", help="Show private legal-reference library state") + + private_search = private_commands.add_parser( + "search", help="Search only the user-private legal-reference library" + ) + private_search.add_argument("--query", required=True) + private_search.add_argument("--purpose", default="auto") + private_search.add_argument("--as-of") + private_search.add_argument("--limit", type=int, default=5) + private_search.add_argument("--max-chars", type=int, default=3500) + private_search.add_argument("--document-type", action="append", default=[]) + private_search.add_argument("--db", type=Path) + private_get = private_commands.add_parser("get", help="Fetch one private segment by ID") + private_get.add_argument("--segment-id", required=True) + private_get.add_argument("--max-chars", type=int, default=6000) + private_get.add_argument("--db", type=Path) + private_verify = private_commands.add_parser( + "verify", help="Verify a private-library receipt" + ) + private_verify.add_argument("--segment-id", required=True) + private_verify.add_argument("--receipt-id", required=True) + private_verify.add_argument("--db", type=Path) + doctor = commands.add_parser("doctor", help="Inspect the active release without changing it") doctor.add_argument("--db", type=Path) return parser @@ -134,6 +241,77 @@ def main(argv: list[str] | None = None) -> None: if args.command == "mcp": run_mcp(transport="stdio" if args.stdio else args.transport) return + if args.command == "official": + if args.official_command in {"install", "update"}: + _print_json( + sync_official( + catalog_source=args.catalog, + source_root=args.source_root, + update=args.official_command == "update", + pdf_fallback=args.pdf_fallback, + ) + ) + elif args.official_command == "status": + _print_json(official_status()) + elif args.official_command == "enable": + _print_json(enable_official()) + elif args.official_command == "disable": + _print_json(disable_official()) + elif args.official_command == "uninstall": + _print_json(uninstall_official()) + else: + raise RuntimeError(f"unhandled official command: {args.official_command}") + return + if args.command == "private": + if args.private_command == "add": + _print_json( + add_private_document( + args.source, + title=args.title, + document_type=args.document_type, + issuer=args.issuer, + effective_from=args.effective_from, + effective_to=args.effective_to, + confirm_no_case_data=args.confirm_no_case_data, + pdf_fallback=args.pdf_fallback, + allow_needs_ocr=args.allow_needs_ocr, + ) + ) + return + if args.private_command == "delete": + _print_json( + delete_private_document( + args.document_id, + pdf_fallback=args.pdf_fallback, + allow_needs_ocr=args.allow_needs_ocr, + ) + ) + return + if args.private_command in {"list", "status"}: + _print_json(list_private_documents()) + return + database = resolve_private_database(explicit_db=getattr(args, "db", None)) + with DeepLaw(database, expected_scope="user_private") as law: + if args.private_command == "search": + response = law.search( + SearchRequest( + query=args.query, + purpose=args.purpose, + as_of=args.as_of, + limit=args.limit, + max_chars=args.max_chars, + document_types=tuple(args.document_type), + ) + ) + _print_json(response.to_dict()) + return + if args.private_command == "get": + _print_json(law.get(args.segment_id, max_chars=args.max_chars)) + return + if args.private_command == "verify": + _print_json(law.verify(args.segment_id, args.receipt_id)) + return + raise RuntimeError(f"unhandled private command: {args.private_command}") database = resolve_active_database(explicit_db=getattr(args, "db", None)) if args.command == "eval": @@ -144,12 +322,12 @@ def main(argv: list[str] | None = None) -> None: _print_json(report) return if args.command == "doctor": - with DeepLaw(database) as law: + with DeepLaw(database, expected_scope="official") as law: info = law.release_info() info["database"] = str(database) _print_json(info) return - with DeepLaw(database) as law: + with DeepLaw(database, expected_scope="official") as law: if args.command == "search": response = law.search( SearchRequest( diff --git a/src/deeplaw/evaluate.py b/src/deeplaw/evaluate.py index b05edce..a6f7c92 100644 --- a/src/deeplaw/evaluate.py +++ b/src/deeplaw/evaluate.py @@ -48,12 +48,23 @@ def evaluate_file(database: Path, cases_path: Path, *, limit: int = 5) -> dict[s ) latency_ms = (perf_counter() - started) * 1000 latencies.append(latency_ms) - titles = [card.title for card in response.evidence] - articles = [card.article_label for card in response.evidence] + returned_cards = (*response.evidence, *response.uncertain_evidence) + expected_bucket = case.get("expected_bucket", "evidence") + if expected_bucket not in {"evidence", "uncertain_evidence"}: + raise ValueError( + f"unsupported expected_bucket in case {case.get('id')}: {expected_bucket}" + ) + target_cards = ( + response.evidence + if expected_bucket == "evidence" + else response.uncertain_evidence + ) + titles = [card.title for card in target_cards] + articles = [card.article_label for card in target_cards] extraction_review_flags = [ - card.extraction_review_required for card in response.evidence + card.extraction_review_required for card in target_cards ] - returned_cards = (*response.evidence, *response.uncertain_evidence) + all_titles = [card.title for card in returned_cards] receipt_checks: list[bool] = [] for card in returned_cards: verification = law.verify(card.segment_id, card.receipt_id) @@ -85,7 +96,7 @@ def evaluate_file(database: Path, cases_path: Path, *, limit: int = 5) -> dict[s is_ranked_case = bool(expected_titles or expected_articles) ranked_cases += int(is_ranked_case) if expected_empty: - retrieval_passed = not response.evidence + retrieval_passed = not returned_cards else: title_passed = not expected_titles or title_rank is not None article_passed = not expected_articles or article_rank is not None @@ -101,11 +112,11 @@ def evaluate_file(database: Path, cases_path: Path, *, limit: int = 5) -> dict[s and extraction_review_flags[0] is bool(expected_extraction_review) ) constraints_passed = ( - not (forbidden_titles & set(titles)) + not (forbidden_titles & set(all_titles)) and mode_passed and extraction_review_passed and receipt_verification_passed - and len(response.evidence) <= evidence_bound + and len(returned_cards) <= evidence_bound and response.total_excerpt_chars <= excerpt_bound ) passed = retrieval_passed and constraints_passed @@ -126,13 +137,20 @@ def evaluate_file(database: Path, cases_path: Path, *, limit: int = 5) -> dict[s "retrieval_passed": retrieval_passed, "constraints_passed": constraints_passed, "rank": rank, - "returned_titles": titles, - "returned_articles": articles, + "expected_bucket": expected_bucket, + "returned_titles": all_titles, + "returned_articles": [card.article_label for card in returned_cards], + "returned_primary_titles": [card.title for card in response.evidence], + "returned_uncertain_titles": [ + card.title for card in response.uncertain_evidence + ], "returned_extraction_review_required": extraction_review_flags, "receipt_count": len(receipt_checks), "receipt_verification_passed": receipt_verification_passed, "mode": response.mode, "evidence_count": len(response.evidence), + "uncertain_evidence_count": len(response.uncertain_evidence), + "returned_count": len(returned_cards), "excerpt_chars": response.total_excerpt_chars, "latency_ms": round(latency_ms, 3), } @@ -141,7 +159,7 @@ def evaluate_file(database: Path, cases_path: Path, *, limit: int = 5) -> dict[s latencies.sort() p95_index = max(0, min(len(latencies) - 1, int(len(latencies) * 0.95) - 1)) return { - "schema_version": "deeplaw.eval-report/v1", + "schema_version": "deeplaw.eval-report/v2", "release_id": release_id, "database_sha256": database_sha256(database), "source_manifest_sha256": source_manifest_sha256, diff --git a/src/deeplaw/extract.py b/src/deeplaw/extract.py index e8a731b..9c4fd0d 100644 --- a/src/deeplaw/extract.py +++ b/src/deeplaw/extract.py @@ -212,6 +212,37 @@ def extract_pdf(path: Path) -> ExtractionResult: ) +def extract_text(path: Path) -> ExtractionResult: + try: + raw_text = path.read_text(encoding="utf-8-sig") + except UnicodeDecodeError as error: + raise ExtractionError(f"TXT must be UTF-8 encoded: {path.name}") from error + except OSError as error: + raise ExtractionError(f"TXT cannot be read: {path.name}") from error + + blocks: list[TextBlock] = [] + paragraph = 0 + for raw_line in raw_text.splitlines(): + text = normalize_text(raw_line) + if not text: + continue + paragraph += 1 + blocks.append(TextBlock(text=text, paragraph=paragraph)) + character_count = sum(len(block.text) for block in blocks) + if character_count < 20: + raise ExtractionError(f"TXT contains too little text: {path.name}") + return ExtractionResult( + blocks=tuple(blocks), + quality=ExtractionQuality( + extractor="utf8-text", + extractor_version="deeplaw-text/v1", + block_count=len(blocks), + page_count=None, + character_count=character_count, + ), + ) + + def extract_document( path: Path, format_name: str, @@ -228,6 +259,8 @@ def extract_document( ) if format_name == "DOCX": return extract_docx(path) + if format_name == "TXT": + return extract_text(path) if format_name != "PDF": raise ExtractionError(f"unsupported source format: {format_name}") if pdf_fallback not in {"off", "vision-consensus"}: diff --git a/src/deeplaw/ingest.py b/src/deeplaw/ingest.py index cbfab42..5b5b2c1 100644 --- a/src/deeplaw/ingest.py +++ b/src/deeplaw/ingest.py @@ -9,7 +9,7 @@ import tempfile from dataclasses import asdict, replace from pathlib import Path -from typing import Any +from typing import Any, Literal from urllib.parse import urlparse from .evidence_graph import derive_relations @@ -199,7 +199,23 @@ def build_release( allow_needs_ocr: bool = False, review_overlay_path: Path | None = None, reviewed_pages_root: Path | None = None, + source_scope: Literal["official", "user_private"] = "official", + library_id: str | None = None, + artifact_mode: int = 0o444, ) -> tuple[Path, BuildReport]: + if source_scope not in {"official", "user_private"}: + raise ValueError(f"unsupported source scope: {source_scope}") + if source_scope == "official" and library_id is not None: + raise ValueError("library_id is valid only for user-private releases") + if source_scope == "user_private": + if not library_id or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", library_id): + raise ValueError("user-private releases require a safe library_id") + if review_overlay_path is not None or reviewed_pages_root is not None: + raise ValueError("user-private releases cannot use official review overlays") + if artifact_mode not in {0o400, 0o600}: + raise ValueError("user-private release artifacts must be owner-only") + elif artifact_mode != 0o444: + raise ValueError("official release artifacts must remain read-only") source_root = source_root.expanduser().resolve(strict=True) manifest_path = manifest_path.expanduser().resolve(strict=True) output_root = output_root.expanduser().resolve() @@ -301,7 +317,10 @@ def build_release( if len(title) > _MAX_TITLE_CHARACTERS: raise ValueError(f"manifest title exceeds 500 characters for {relative_path}") format_name = str(raw.get("format", source_path.suffix.lstrip("."))).upper() - expected_suffix = {"DOCX": ".docx", "PDF": ".pdf"}.get(format_name) + supported_formats = {"DOCX": ".docx", "PDF": ".pdf"} + if source_scope == "user_private": + supported_formats["TXT"] = ".txt" + expected_suffix = supported_formats.get(format_name) if expected_suffix is None or source_path.suffix.lower() != expected_suffix: raise ValueError(f"format/path mismatch for {relative_path}: {format_name}") official_source = str(raw.get("officialSource", "")).strip() @@ -311,16 +330,31 @@ def build_release( _ = parsed_source.port except ValueError as error: raise ValueError(f"officialSource is malformed for {relative_path}") from error - if ( + unsafe_source = ( len(official_source) > _MAX_SOURCE_URL_CHARACTERS or any(ord(character) < 32 for character in official_source) - or parsed_source.scheme != "https" - or not parsed_source.netloc - or not source_hostname or parsed_source.username is not None or parsed_source.password is not None + ) + if source_scope == "official": + if ( + unsafe_source + or parsed_source.scheme != "https" + or not parsed_source.netloc + or not source_hostname + ): + raise ValueError(f"officialSource must be HTTPS for {relative_path}") + elif ( + unsafe_source + or parsed_source.scheme != "private" + or parsed_source.netloc != "source" + or parsed_source.path != f"/{actual_hash}" + or parsed_source.query + or parsed_source.fragment ): - raise ValueError(f"officialSource must be HTTPS for {relative_path}") + raise ValueError( + f"user-private source locator must bind the source hash for {relative_path}" + ) inferred_type, inferred_issuer, inferred_authority = classify_document(relative_path, title) document_type = str(raw.get("documentType") or inferred_type) @@ -332,6 +366,8 @@ def build_release( authority_rank = int(raw.get("authorityRank", inferred_authority)) if not 0 <= authority_rank <= 100: raise ValueError(f"invalid authorityRank for {relative_path}: {authority_rank}") + if source_scope == "user_private" and authority_rank != 0: + raise ValueError(f"user-private authorityRank must be 0 for {relative_path}") normalized_dates: dict[str, str] = {} for field_name in ("effectiveDate", "effectiveTo", "promulgatedOn"): if raw.get(field_name) is not None: @@ -361,6 +397,8 @@ def build_release( status = str(raw.get("status") or "unverified_current").strip() if status not in _DOCUMENT_STATUSES: raise ValueError(f"unsupported status for {relative_path}: {status}") + if source_scope == "user_private" and status != "unknown": + raise ValueError(f"user-private status must be unknown for {relative_path}") if status.startswith("verified_") and not effective_from: raise ValueError(f"verified status requires effectiveDate for {relative_path}") raw_note = raw.get("note") @@ -513,6 +551,13 @@ def build_release( for item in report.documents ], } + if source_scope == "user_private": + derivation_payload.update( + { + "collection_scope": source_scope, + "library_id": library_id, + } + ) derivation_sha256 = sha256_bytes(canonical_json(derivation_payload).encode("utf-8")) release_id = stable_id("lawrel", derivation_sha256, length=32) report.release_id = release_id @@ -539,6 +584,13 @@ def build_release( "vector_index": False, "derived_wiki": False, } + if source_scope == "user_private": + release_metadata.update( + { + "collection_scope": source_scope, + "library_id": library_id, + } + ) if applied_review is not None: release_metadata.update( { @@ -553,6 +605,8 @@ def build_release( if payload["package_qa_reviewed_on"] is not None: release_metadata["package_qa_reviewed_on"] = payload["package_qa_reviewed_on"] output_root.mkdir(parents=True, exist_ok=True) + if source_scope == "user_private": + os.chmod(output_root, 0o700) staging_dir = Path(tempfile.mkdtemp(prefix=".deeplaw-build-", dir=output_root)) try: database_path = staging_dir / "deeplaw.sqlite3" @@ -565,7 +619,7 @@ def build_release( relations=relations, ) release_metadata["database_sha256"] = database_sha256(database_path) - os.chmod(database_path, 0o444) + os.chmod(database_path, artifact_mode) (staging_dir / "release.json").write_text( json.dumps(release_metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", @@ -574,8 +628,8 @@ def build_release( json.dumps(report.to_dict(), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) - os.chmod(staging_dir / "release.json", 0o444) - os.chmod(staging_dir / "build-report.json", 0o444) + os.chmod(staging_dir / "release.json", artifact_mode) + os.chmod(staging_dir / "build-report.json", artifact_mode) if release_dir.exists(): existing_manifest_path = release_dir / "release.json" existing_database_path = release_dir / "deeplaw.sqlite3" @@ -597,6 +651,8 @@ def build_release( raise RuntimeError(f"existing immutable release failed verification: {release_dir}") else: os.replace(staging_dir, release_dir) + if source_scope == "user_private": + os.chmod(release_dir, 0o700) finally: if staging_dir.exists(): shutil.rmtree(staging_dir) diff --git a/src/deeplaw/mcp_server.py b/src/deeplaw/mcp_server.py index 80b9a22..9d4375b 100644 --- a/src/deeplaw/mcp_server.py +++ b/src/deeplaw/mcp_server.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os from collections.abc import AsyncIterator from contextlib import asynccontextmanager from copy import deepcopy @@ -17,20 +18,34 @@ from . import __version__ from .models import Purpose, SearchRequest +from .official import active_official_release_id +from .private_library import active_private_release_id, resolve_private_database from .search import DeepLaw -Operation = Literal["search", "get", "verify", "release_info"] +Operation = Literal[ + "search", + "get", + "verify", + "release_info", + "private_search", + "private_get", + "private_verify", + "private_info", +] _DESCRIPTION = ( "Read-only Chinese-law research gateway. Call only after an explicit Chinese-law " "research request; never call for ordinary code, data, document, or analytics work, " "and never activate from a lone legal-looking keyword. Operations: search bounded " - "evidence, get an exact segment, verify a receipt, or inspect the active immutable release." + "official evidence, get an exact segment, verify a receipt, or inspect the active immutable " + "release. Explicit private_* operations search a separate local user-private legal-reference " + "library; they never blend its ranking or authority with the official catalog." ) _INSTRUCTIONS = ( "Read-only, version-aware Chinese legal research. Use only for explicit legal questions. " "Search returns at most five evidence cards; fetch full text only by selected segment_id. " - "Never treat retrieval as proof of case facts or applicability." + "Never treat retrieval as proof of case facts or applicability. Private results are " + "user-provided and never official DeepLaw sources." ) _OUTPUT_CONTRACTS = { "search": "law-search-response.v2.schema.json", @@ -44,7 +59,10 @@ @dataclass(frozen=True) class _RuntimeContext: - law: DeepLaw + official: DeepLaw | None + private: DeepLaw | None + guard_official_epoch: bool + guard_private_epoch: bool lock: RLock @@ -104,7 +122,7 @@ def tool_definition() -> types.Tool: return types.Tool( name="law_support", description=_DESCRIPTION, - inputSchema=deepcopy(_load_contract("law-support.input.v1.schema.json")), + inputSchema=deepcopy(_load_contract("law-support.input.v2.schema.json")), outputSchema=deepcopy(bundled_output_schema()), annotations=types.ToolAnnotations( readOnlyHint=True, @@ -115,10 +133,10 @@ def tool_definition() -> types.Tool: ) -def _execute_support( +def _execute_law_operation( law: DeepLaw, *, - operation: Operation, + operation: Literal["search", "get", "verify", "release_info"], query: str = "", purpose: Purpose = "auto", as_of: str | None = None, @@ -151,6 +169,60 @@ def _execute_support( return law.search(request).to_dict() +def _execute_support( + runtime: _RuntimeContext, + *, + operation: Operation, + query: str = "", + purpose: Purpose = "auto", + as_of: str | None = None, + limit: int = 5, + max_chars: int = 3500, + document_types: list[str] | None = None, + segment_id: str | None = None, + receipt_id: str | None = None, +) -> dict[str, Any]: + private_operation = operation.startswith("private_") + law = runtime.private if private_operation else runtime.official + if law is None: + scope = "user-private" if private_operation else "official" + raise FileNotFoundError(f"DeepLaw has no active {scope} release") + if not private_operation and runtime.guard_official_epoch: + current_release_id = active_official_release_id() + if current_release_id != law.release_id: + raise RuntimeError( + "official library changed or was disabled; restart the DeepLaw MCP process " + "before reading it" + ) + elif private_operation and runtime.guard_private_epoch: + current_release_id = active_private_release_id() + if current_release_id != law.release_id: + raise RuntimeError( + "user-private library changed; restart the DeepLaw MCP process before reading it" + ) + elif private_operation and not law.database.is_file(): + raise RuntimeError( + "user-private snapshot was removed; restart the DeepLaw MCP process before reading it" + ) + normalized_operation = operation.removeprefix("private_") + if normalized_operation == "info": + normalized_operation = "release_info" + return _execute_law_operation( + law, + operation=cast( + Literal["search", "get", "verify", "release_info"], normalized_operation + ), + query=query, + purpose=purpose, + as_of=as_of, + limit=limit, + max_chars=max_chars, + document_types=document_types, + segment_id=segment_id, + receipt_id=receipt_id, + ) + + def handle_support( *, operation: Operation = "search", @@ -163,12 +235,27 @@ def handle_support( segment_id: str | None = None, receipt_id: str | None = None, database: str | Path | None = None, + private_database: str | Path | None = None, ) -> dict[str, Any]: """Execute one read-only DeepLaw operation outside the MCP transport.""" - with DeepLaw(database) as law: + private_operation = operation.startswith("private_") + selected_database = private_database if private_operation else database + if private_operation and selected_database is None: + selected_database = resolve_private_database() + expected_scope: Literal["official", "user_private"] = ( + "user_private" if private_operation else "official" + ) + with DeepLaw(selected_database, expected_scope=expected_scope) as law: + runtime = _RuntimeContext( + official=None if private_operation else law, + private=law if private_operation else None, + guard_official_epoch=False, + guard_private_epoch=False, + lock=RLock(), + ) return _execute_support( - law, + runtime, operation=operation, query=query, purpose=purpose, @@ -184,8 +271,38 @@ def handle_support( def create_mcp_server() -> Server[_RuntimeContext]: @asynccontextmanager async def lifespan(_: Server[_RuntimeContext]) -> AsyncIterator[_RuntimeContext]: - with DeepLaw() as law: - yield _RuntimeContext(law=law, lock=RLock()) + official: DeepLaw | None = None + private: DeepLaw | None = None + official_explicit = os.environ.get("DEEPLAW_DB") is not None + private_explicit = os.environ.get("DEEPLAW_PRIVATE_DB") is not None + try: + try: + official = DeepLaw(expected_scope="official") + except FileNotFoundError: + if os.environ.get("DEEPLAW_DB") is not None: + raise + try: + private_database = resolve_private_database() + private = DeepLaw(private_database, expected_scope="user_private") + except FileNotFoundError: + if private_explicit: + raise + yield _RuntimeContext( + official=official, + private=private, + guard_official_epoch=( + official is not None + and not official_explicit + and active_official_release_id() == official.release_id + ), + guard_private_epoch=private is not None and not private_explicit, + lock=RLock(), + ) + finally: + if private is not None: + private.close() + if official is not None: + official.close() server: Server[_RuntimeContext] = Server( "DeepLaw", @@ -206,7 +323,7 @@ async def call_tool(name: str, arguments: dict[str, Any]) -> dict[str, Any]: runtime = server.request_context.lifespan_context with runtime.lock: return _execute_support( - runtime.law, + runtime, operation=cast(Operation, arguments.get("operation", "search")), query=str(arguments.get("query", "")), purpose=cast(Purpose, arguments.get("purpose", "auto")), @@ -223,7 +340,7 @@ async def call_tool(name: str, arguments: dict[str, Any]) -> dict[str, Any]: def run_mcp(*, transport: str = "stdio") -> None: if transport != "stdio": - raise ValueError("DeepLaw 0.2 supports only the local stdio MCP transport") + raise ValueError("DeepLaw supports only the local stdio MCP transport") async def serve() -> None: server = create_mcp_server() diff --git a/src/deeplaw/official.py b/src/deeplaw/official.py new file mode 100644 index 0000000..891027e --- /dev/null +++ b/src/deeplaw/official.py @@ -0,0 +1,892 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import tempfile +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from . import __version__ +from .admin_lock import administration_locked +from .ingest import build_release +from .models import BuildReport +from .store import activate_release, default_home, verify_release_artifact +from .util import canonical_date, sha256_bytes, sha256_file + +OFFICIAL_CATALOG_SCHEMA = "deeplaw.official-catalog/v1" +OFFICIAL_STATE_SCHEMA = "deeplaw.official-state/v1" +DEFAULT_CATALOG_URL = ( + "https://raw.githubusercontent.com/Eysn0130/DeepLaw/main/" + "catalogs/deeplaw-official-cn.json" +) +_MAX_CATALOG_BYTES = 64 * 1024 * 1024 +_MAX_SOURCE_BYTES = 512 * 1024 * 1024 +_MAX_DOWNLOAD_ENVELOPE_BYTES = 64 * 1024 +_RELEASE_ID = re.compile(r"^lawrel_[0-9a-f]{32}$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_CATALOG_ID = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +_CATALOG_REQUIRED_FIELDS = { + "schemaVersion", + "catalogId", + "sequence", + "version", + "publishedOn", + "package", + "documents", +} +_CATALOG_OPTIONAL_FIELDS = {"reviewOverlay", "buildPolicy"} +_RETRYABLE_HTTP_STATUS = {429, 500, 502, 503, 504} +_STATE_FIELDS = { + "schema_version", + "enabled", + "active_release_id", + "installed_release_ids", + "catalog", +} +_STATE_CATALOG_FIELDS = { + "catalog_id", + "sequence", + "version", + "published_on", + "sha256", + "source", + "synced_at", +} + + +def _now() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def official_home(home: str | Path | None = None) -> Path: + root = Path(home).expanduser() if home is not None else default_home() + root = root.absolute() + if root.is_symlink(): + raise RuntimeError(f"DeepLaw home must not be a symbolic link: {root}") + return root / "official" + + +def bundled_catalog_path() -> Path: + packaged = Path(__file__).resolve().parent / "catalogs" / "deeplaw-official-cn.json" + if packaged.is_file(): + return packaged + repository = Path(__file__).resolve().parents[2] / "catalogs" / "deeplaw-official-cn.json" + if repository.is_file(): + return repository + raise RuntimeError("bundled DeepLaw official catalog is missing") + + +def _bundled_governance_path(resource: str) -> Path | None: + packaged = Path(__file__).resolve().parent / "governance" / resource + if packaged.is_file(): + return packaged + repository = Path(__file__).resolve().parents[2] / "governance" / resource + if repository.is_file(): + return repository + return None + + +def _secure_directory(path: Path, *, mode: int = 0o700) -> Path: + if path.is_symlink(): + raise RuntimeError(f"official library directory must not be a symbolic link: {path}") + path.mkdir(parents=True, exist_ok=True, mode=mode) + if not path.is_dir(): + raise RuntimeError(f"official library path is not a directory: {path}") + return path + + +def _state_path(home: str | Path | None = None) -> Path: + return official_home(home) / "state.json" + + +def _empty_state() -> dict[str, Any]: + return { + "schema_version": OFFICIAL_STATE_SCHEMA, + "enabled": False, + "active_release_id": None, + "installed_release_ids": [], + "catalog": None, + } + + +def _validate_state(value: Any) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _STATE_FIELDS: + raise RuntimeError("official state does not match its closed contract") + if value.get("schema_version") != OFFICIAL_STATE_SCHEMA: + raise RuntimeError("unsupported official state schema") + if not isinstance(value.get("enabled"), bool): + raise RuntimeError("official enabled state is invalid") + active = value.get("active_release_id") + if active is not None and (not isinstance(active, str) or not _RELEASE_ID.fullmatch(active)): + raise RuntimeError("official active release ID is invalid") + installed = value.get("installed_release_ids") + if ( + not isinstance(installed, list) + or len(installed) > 10_000 + or len(set(installed)) != len(installed) + or any(not isinstance(item, str) or not _RELEASE_ID.fullmatch(item) for item in installed) + ): + raise RuntimeError("official installed release IDs are invalid") + if active is not None and active not in installed: + raise RuntimeError("official active release is not registered as installed") + catalog = value.get("catalog") + if catalog is not None: + if not isinstance(catalog, dict) or set(catalog) != _STATE_CATALOG_FIELDS: + raise RuntimeError("official catalog state does not match its closed contract") + if not _CATALOG_ID.fullmatch(str(catalog.get("catalog_id", ""))): + raise RuntimeError("official catalog identity is invalid") + sequence = catalog.get("sequence") + if isinstance(sequence, bool) or not isinstance(sequence, int) or sequence < 1: + raise RuntimeError("official catalog sequence is invalid") + if not isinstance(catalog.get("version"), str) or not catalog["version"]: + raise RuntimeError("official catalog version is invalid") + canonical_date(str(catalog.get("published_on", "")), field="published_on") + if not _SHA256.fullmatch(str(catalog.get("sha256", ""))): + raise RuntimeError("official catalog SHA-256 is invalid") + for field_name in ("source", "synced_at"): + if not isinstance(catalog.get(field_name), str) or not catalog[field_name]: + raise RuntimeError(f"official catalog {field_name} is invalid") + if value["enabled"] and active is None: + raise RuntimeError("enabled official state requires an active release") + return value + + +def _load_state(home: str | Path | None = None) -> dict[str, Any]: + root = official_home(home) + if root.is_symlink(): + raise RuntimeError(f"official library directory must not be a symbolic link: {root}") + path = root / "state.json" + if path.is_symlink(): + raise RuntimeError(f"official state must not be a symbolic link: {path}") + if not path.exists(): + return _empty_state() + if not path.is_file() or path.stat().st_size > 1024 * 1024: + raise RuntimeError("official state is missing, unsafe, or too large") + try: + value = json.loads(path.read_bytes()) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise RuntimeError("official state cannot be read") from error + return _validate_state(value) + + +def _write_state(state: dict[str, Any], *, home: str | Path | None = None) -> None: + _validate_state(state) + root = _secure_directory(official_home(home)) + path = root / "state.json" + if path.is_symlink(): + raise RuntimeError(f"official state must not be a symbolic link: {path}") + temporary = root / ".state.json.tmp" + if temporary.is_symlink(): + raise RuntimeError(f"official state temporary must not be a symbolic link: {temporary}") + temporary.unlink(missing_ok=True) + payload = json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + temporary.unlink(missing_ok=True) + raise + os.chmod(temporary, 0o600) + os.replace(temporary, path) + + +def _download_bytes(url: str, *, maximum: int, timeout: float = 60.0) -> bytes: + parsed = urlparse(url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + raise ValueError("official network sources must use credential-free HTTPS URLs") + request = Request( + url, + headers={ + "Accept-Encoding": "identity", + "User-Agent": f"DeepLaw/{__version__} official-catalog-client", + }, + ) + with _urlopen_with_retry(request, timeout=timeout) as response: + final = urlparse(response.geturl()) + if ( + final.scheme != "https" + or not final.hostname + or final.username is not None + or final.password is not None + ): + raise RuntimeError("official download redirected to an unsafe URL") + chunks: list[bytes] = [] + size = 0 + while True: + chunk = response.read(min(1024 * 1024, maximum + 1 - size)) + if not chunk: + break + size += len(chunk) + if size > maximum: + raise RuntimeError(f"official download exceeds {maximum} bytes") + chunks.append(chunk) + return b"".join(chunks) + + +def _urlopen_with_retry(request: Request, *, timeout: float) -> Any: + attempts = 5 + for attempt in range(attempts): + try: + return urlopen(request, timeout=timeout) + except HTTPError as error: + if error.code not in _RETRYABLE_HTTP_STATUS or attempt == attempts - 1: + raise + error.close() + except (TimeoutError, URLError): + if attempt == attempts - 1: + raise + time.sleep(0.5 * (2**attempt)) + raise RuntimeError("official download retry loop ended unexpectedly") + + +def _read_catalog(source: str | Path | None) -> tuple[dict[str, Any], bytes, str]: + if source is None: + path = bundled_catalog_path() + payload = path.read_bytes() + label = "bundled" + else: + raw_source = str(source) + if raw_source.startswith("https://"): + payload = _download_bytes(raw_source, maximum=_MAX_CATALOG_BYTES) + label = raw_source + else: + path = Path(source).expanduser() + if path.is_symlink(): + raise ValueError("official catalog must not be a symbolic link") + path = path.resolve(strict=True) + if not path.is_file(): + raise ValueError("official catalog must be a regular file") + payload = path.read_bytes() + label = "local" + if len(payload) > _MAX_CATALOG_BYTES: + raise ValueError("official catalog exceeds the 64 MiB limit") + try: + catalog = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("official catalog is not valid UTF-8 JSON") from error + return _validate_catalog(catalog), payload, label + + +def _validate_catalog(value: Any) -> dict[str, Any]: + if ( + not isinstance(value, dict) + or not set(value) >= _CATALOG_REQUIRED_FIELDS + or set(value) - _CATALOG_REQUIRED_FIELDS - _CATALOG_OPTIONAL_FIELDS + ): + raise ValueError("official catalog does not match its closed contract") + if value.get("schemaVersion") != OFFICIAL_CATALOG_SCHEMA: + raise ValueError("unsupported official catalog schema") + catalog_id = value.get("catalogId") + if not isinstance(catalog_id, str) or not _CATALOG_ID.fullmatch(catalog_id): + raise ValueError("official catalog ID is invalid") + sequence = value.get("sequence") + if isinstance(sequence, bool) or not isinstance(sequence, int) or sequence < 1: + raise ValueError("official catalog sequence must be a positive integer") + version = value.get("version") + if not isinstance(version, str) or not version or len(version) > 100: + raise ValueError("official catalog version is invalid") + published_on = value.get("publishedOn") + if not isinstance(published_on, str): + raise ValueError("official catalog publishedOn is invalid") + canonical_date(published_on, field="publishedOn") + review_overlay = value.get("reviewOverlay") + if review_overlay is not None: + if not isinstance(review_overlay, dict) or set(review_overlay) != { + "resource", + "url", + "sha256", + }: + raise ValueError("official catalog reviewOverlay is invalid") + resource = review_overlay.get("resource") + if ( + not isinstance(resource, str) + or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}\.json", resource) + ): + raise ValueError("official catalog reviewOverlay resource is invalid") + review_url = urlparse(str(review_overlay.get("url", ""))) + if ( + review_url.scheme != "https" + or not review_url.hostname + or review_url.username is not None + or review_url.password is not None + ): + raise ValueError("official catalog reviewOverlay URL is invalid") + if not _SHA256.fullmatch(str(review_overlay.get("sha256", ""))): + raise ValueError("official catalog reviewOverlay SHA-256 is invalid") + build_policy = value.get("buildPolicy") + if build_policy is not None and ( + not isinstance(build_policy, dict) + or set(build_policy) != {"pdfFallback", "allowNeedsOcr"} + or build_policy.get("pdfFallback") not in {"off", "vision-consensus"} + or not isinstance(build_policy.get("allowNeedsOcr"), bool) + ): + raise ValueError("official catalog buildPolicy is invalid") + package = value.get("package") + documents = value.get("documents") + if not isinstance(package, dict): + raise ValueError("official catalog package must be an object") + if not isinstance(documents, list) or not documents or len(documents) > 10_000: + raise ValueError("official catalog documents are invalid") + if package.get("documentCount") != len(documents): + raise ValueError("official catalog documentCount does not match its documents") + seen_paths: set[str] = set() + seen_hashes: set[str] = set() + for document in documents: + if not isinstance(document, dict): + raise ValueError("official catalog document must be an object") + relative_path = document.get("path") + title = document.get("title") + format_name = document.get("format") + source_url = document.get("officialSource") + byte_size = document.get("byteSize") + source_hash = document.get("sha256") + if ( + not isinstance(relative_path, str) + or not relative_path + or relative_path in seen_paths + or Path(relative_path).is_absolute() + or any(part in {".", ".."} for part in Path(relative_path).parts) + ): + raise ValueError("official catalog contains an invalid or duplicate path") + seen_paths.add(relative_path) + if not isinstance(title, str) or not title or len(title) > 500: + raise ValueError(f"official catalog title is invalid: {relative_path}") + suffix = {"DOCX": ".docx", "PDF": ".pdf"}.get(format_name) + if suffix is None or Path(relative_path).suffix.lower() != suffix: + raise ValueError(f"official catalog format is invalid: {relative_path}") + parsed_source = urlparse(str(source_url)) + if ( + not isinstance(source_url, str) + or parsed_source.scheme != "https" + or not parsed_source.hostname + or parsed_source.username is not None + or parsed_source.password is not None + ): + raise ValueError(f"official catalog source URL is invalid: {relative_path}") + if ( + isinstance(byte_size, bool) + or not isinstance(byte_size, int) + or not 1 <= byte_size <= _MAX_SOURCE_BYTES + ): + raise ValueError(f"official catalog byte size is invalid: {relative_path}") + if ( + not isinstance(source_hash, str) + or not _SHA256.fullmatch(source_hash) + or source_hash in seen_hashes + ): + raise ValueError(f"official catalog SHA-256 is invalid or duplicated: {relative_path}") + seen_hashes.add(source_hash) + return value + + +def _save_catalog( + catalog: dict[str, Any], + payload: bytes, + *, + home: str | Path | None, +) -> Path: + root = _secure_directory(official_home(home) / "catalogs") + digest = sha256_bytes(payload) + path = root / f"{catalog['sequence']:08d}-{digest}.json" + if path.exists(): + if path.is_symlink() or not path.is_file() or sha256_file(path) != digest: + raise RuntimeError("stored official catalog failed integrity validation") + return path + temporary = root / f".{path.name}.tmp" + if temporary.is_symlink(): + raise RuntimeError(f"official catalog temporary must not be a symbolic link: {temporary}") + temporary.unlink(missing_ok=True) + temporary.write_bytes(payload) + os.chmod(temporary, 0o444) + os.replace(temporary, path) + return path + + +def _resolve_review_overlay( + catalog: dict[str, Any], + *, + source_label: str, + home: str | Path | None, +) -> Path | None: + declaration = catalog.get("reviewOverlay") + if declaration is None: + return None + expected_hash = declaration["sha256"] + payload: bytes | None = None + bundled = _bundled_governance_path(declaration["resource"]) + if bundled is not None and sha256_file(bundled) == expected_hash: + payload = bundled.read_bytes() + elif source_label == "bundled": + raise RuntimeError("bundled official review overlay is missing or has the wrong hash") + if payload is None: + payload = _download_bytes(declaration["url"], maximum=_MAX_CATALOG_BYTES) + if sha256_bytes(payload) != expected_hash: + raise RuntimeError("official review overlay SHA-256 changed") + root = _secure_directory(official_home(home) / "reviews") + path = root / f"{expected_hash}.json" + if path.exists(): + if path.is_symlink() or not path.is_file() or sha256_file(path) != expected_hash: + raise RuntimeError("stored official review overlay failed integrity validation") + return path + temporary = root / f".{path.name}.tmp" + if temporary.is_symlink(): + raise RuntimeError( + f"official review overlay temporary must not be a symbolic link: {temporary}" + ) + temporary.unlink(missing_ok=True) + temporary.write_bytes(payload) + os.chmod(temporary, 0o444) + os.replace(temporary, path) + return path + + +def _download_source(document: dict[str, Any], destination: Path) -> None: + source_url = _resolve_source_download_url(document) + parsed = urlparse(source_url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + raise ValueError("official network sources must use credential-free HTTPS URLs") + request = Request( + source_url, + headers={ + "Accept-Encoding": "identity", + "User-Agent": f"DeepLaw/{__version__} official-source-client", + }, + ) + temporary = destination.with_name(f".{destination.name}.tmp") + if temporary.is_symlink(): + raise RuntimeError(f"official source temporary must not be a symbolic link: {temporary}") + temporary.unlink(missing_ok=True) + digest = hashlib.sha256() + size = 0 + try: + with ( + _urlopen_with_retry(request, timeout=60.0) as response, + temporary.open("xb") as stream, + ): + final = urlparse(response.geturl()) + if ( + final.scheme != "https" + or not final.hostname + or final.username is not None + or final.password is not None + ): + raise RuntimeError("official download redirected to an unsafe URL") + while chunk := response.read(1024 * 1024): + size += len(chunk) + if size > document["byteSize"] or size > _MAX_SOURCE_BYTES: + raise RuntimeError(f"official source byte size changed: {document['path']}") + digest.update(chunk) + stream.write(chunk) + stream.flush() + os.fsync(stream.fileno()) + if size != document["byteSize"]: + raise RuntimeError(f"official source byte size changed: {document['path']}") + if digest.hexdigest() != document["sha256"]: + raise RuntimeError(f"official source SHA-256 changed: {document['path']}") + os.chmod(temporary, 0o444) + os.replace(temporary, destination) + except BaseException: + temporary.unlink(missing_ok=True) + raise + + +def _resolve_source_download_url(document: dict[str, Any]) -> str: + source_url = str(document["officialSource"]) + parsed = urlparse(source_url) + if parsed.hostname != "flk.npc.gov.cn" or parsed.path != "/law-search/download/pc": + return source_url + + payload = _download_bytes(source_url, maximum=_MAX_DOWNLOAD_ENVELOPE_BYTES) + try: + envelope = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise RuntimeError("National Laws Database download envelope is invalid") from error + if ( + not isinstance(envelope, dict) + or envelope.get("code") != 200 + or not isinstance(envelope.get("data"), dict) + ): + raise RuntimeError("National Laws Database download envelope was rejected") + resolved_url = envelope["data"].get("url") + if not isinstance(resolved_url, str): + raise RuntimeError("National Laws Database download URL is missing") + resolved = urlparse(resolved_url) + if ( + resolved.scheme != "https" + or resolved.hostname != "flkoss.obs-bj2.cucloud.cn" + or resolved.username is not None + or resolved.password is not None + or Path(resolved.path).suffix.lower() != Path(document["path"]).suffix.lower() + ): + raise RuntimeError("National Laws Database download URL is unsafe") + return resolved_url + + +def _materialize_downloaded_sources( + catalog: dict[str, Any], + *, + home: str | Path | None, +) -> Path: + root = _secure_directory(official_home(home)) + cache = _secure_directory(root / "sources") + workspace = Path(tempfile.mkdtemp(prefix=".official-sources-", dir=root)) + try: + for document in catalog["documents"]: + suffix = Path(document["path"]).suffix.lower() + cached = cache / f"{document['sha256']}{suffix}" + if cached.exists(): + if ( + cached.is_symlink() + or not cached.is_file() + or cached.stat().st_size != document["byteSize"] + or sha256_file(cached) != document["sha256"] + ): + raise RuntimeError( + f"official source cache failed validation: {document['path']}" + ) + else: + try: + _download_source(document, cached) + except Exception as error: + raise RuntimeError( + f"official source download failed: {document['path']}: {error}" + ) from error + destination = workspace / document["path"] + destination.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(cached, destination) + except OSError: + shutil.copyfile(cached, destination) + return workspace + except BaseException: + shutil.rmtree(workspace, ignore_errors=True) + raise + + +def _active_pointer(home: str | Path | None = None) -> Path: + root = Path(home).expanduser().absolute() if home is not None else default_home().absolute() + return root / "ACTIVE" + + +def _snapshot_active(home: str | Path | None = None) -> str | None: + active = _active_pointer(home) + if active.is_symlink(): + raise RuntimeError(f"official ACTIVE pointer must not be a symbolic link: {active}") + if not active.exists(): + return None + if not active.is_file() or active.stat().st_size > 128: + raise RuntimeError("official ACTIVE pointer is unsafe") + release_id = active.read_text(encoding="utf-8").strip() + if not _RELEASE_ID.fullmatch(release_id): + raise RuntimeError("official ACTIVE pointer is invalid") + return release_id + + +def _restore_active( + release_id: str | None, + *, + home: str | Path | None = None, +) -> None: + active = _active_pointer(home) + if release_id is None: + active.unlink(missing_ok=True) + return + base_home = ( + Path(home).expanduser().absolute() if home is not None else default_home().absolute() + ) + activate_release(base_home / "releases", release_id) + + +def _remove_active_if_registered(state: dict[str, Any], *, home: str | Path | None = None) -> None: + active = _active_pointer(home) + if active.is_symlink(): + raise RuntimeError(f"official ACTIVE pointer must not be a symbolic link: {active}") + if not active.exists(): + return + if not active.is_file() or active.stat().st_size > 128: + raise RuntimeError("official ACTIVE pointer is unsafe") + release_id = active.read_text(encoding="utf-8").strip() + if release_id in state["installed_release_ids"]: + active.unlink() + + +def _build_report_summary(report: BuildReport, release_dir: Path) -> dict[str, Any]: + review_required_documents = sum( + bool(document.get("review_required")) for document in report.documents + ) + review_required_pages = sum( + bool(page.get("review_required")) + for document in report.documents + for page in document.get("page_evidence", []) + ) + return { + "schema_version": report.schema_version, + "release_id": report.release_id, + "document_count": report.document_count, + "segment_count": report.segment_count, + "relation_count": report.relation_count, + "source_bytes": report.source_bytes, + "extractors": dict(report.extractors), + "warning_count": len(report.warnings), + "review_required_document_count": review_required_documents, + "review_required_page_count": review_required_pages, + "build_report": str(release_dir / "build-report.json"), + } + + +def _reuse_unchanged_release( + state: dict[str, Any], + *, + update: bool, + home: str | Path | None, +) -> dict[str, Any] | None: + release_id = state["active_release_id"] + if release_id is None: + return None + base_home = ( + Path(home).expanduser().absolute() if home is not None else default_home().absolute() + ) + release_dir = base_home / "releases" / release_id + database = release_dir / "deeplaw.sqlite3" + if not database.exists(): + return None + artifact = verify_release_artifact(database) + catalog = state["catalog"] + if catalog is None or artifact.get("source_manifest_sha256") != catalog["sha256"]: + raise RuntimeError("installed official release does not match its catalog state") + + should_enable = state["enabled"] if update else True + previous_active = _snapshot_active(home) + active_changed = should_enable and previous_active != release_id + state_changed = should_enable != state["enabled"] + if active_changed or state_changed: + try: + if should_enable: + activate_release(base_home / "releases", release_id) + if state_changed: + _write_state({**state, "enabled": should_enable}, home=home) + except BaseException: + _restore_active(previous_active, home=home) + raise + return { + "changed": False, + "enabled": should_enable, + "active_release_id": release_id, + "catalog": catalog, + "report": { + "release_id": release_id, + "document_count": artifact["document_count"], + "segment_count": artifact["segment_count"], + "build_report": str(release_dir / "build-report.json"), + "cached": True, + }, + "restart_required": active_changed or state_changed, + } + + +@administration_locked(".official-admin.lock") +def sync_official( + *, + catalog_source: str | Path | None = None, + source_root: str | Path | None = None, + update: bool = False, + pdf_fallback: str | None = None, + home: str | Path | None = None, +) -> dict[str, Any]: + if update and catalog_source is None: + catalog_source = DEFAULT_CATALOG_URL + catalog, payload, source_label = _read_catalog(catalog_source) + declared_build_policy = catalog.get("buildPolicy", {}) + effective_pdf_fallback = pdf_fallback or declared_build_policy.get("pdfFallback", "off") + effective_allow_needs_ocr = bool(declared_build_policy.get("allowNeedsOcr", False)) + state = _load_state(home) + previous = state["catalog"] + digest = sha256_bytes(payload) + if previous is not None: + if catalog["catalogId"] != previous["catalog_id"]: + raise ValueError("official catalog ID cannot change during an update") + if catalog["sequence"] < previous["sequence"]: + raise ValueError("official catalog rollback is not allowed") + if catalog["sequence"] == previous["sequence"] and digest != previous["sha256"]: + raise ValueError("official catalog sequence was rewritten with different content") + if digest == previous["sha256"]: + reused = _reuse_unchanged_release(state, update=update, home=home) + if reused is not None: + return reused + + catalog_path = _save_catalog(catalog, payload, home=home) + review_overlay_path = _resolve_review_overlay( + catalog, + source_label=source_label, + home=home, + ) + base_home = ( + Path(home).expanduser().absolute() if home is not None else default_home().absolute() + ) + releases = _secure_directory(base_home / "releases") + temporary_sources: Path | None = None + if source_root is None: + temporary_sources = _materialize_downloaded_sources(catalog, home=home) + build_source_root = temporary_sources + else: + declared_root = Path(source_root).expanduser() + if declared_root.is_symlink(): + raise ValueError("official source root must not be a symbolic link") + build_source_root = declared_root.resolve(strict=True) + if not build_source_root.is_dir(): + raise ValueError("official source root must be a directory") + try: + release_dir, report = build_release( + source_root=build_source_root, + manifest_path=catalog_path, + output_root=releases, + pdf_fallback=effective_pdf_fallback, + allow_needs_ocr=effective_allow_needs_ocr, + review_overlay_path=review_overlay_path, + ) + finally: + if temporary_sources is not None: + shutil.rmtree(temporary_sources, ignore_errors=True) + + installed = list(state["installed_release_ids"]) + if release_dir.name not in installed: + installed.append(release_dir.name) + should_enable = not update or state["enabled"] or state["catalog"] is None + next_state = { + "schema_version": OFFICIAL_STATE_SCHEMA, + "enabled": should_enable, + "active_release_id": release_dir.name, + "installed_release_ids": installed, + "catalog": { + "catalog_id": catalog["catalogId"], + "sequence": catalog["sequence"], + "version": catalog["version"], + "published_on": catalog["publishedOn"], + "sha256": digest, + "source": source_label, + "synced_at": _now(), + }, + } + previous_active = _snapshot_active(home) + try: + if should_enable: + activate_release(releases, release_dir.name) + _write_state(next_state, home=home) + except BaseException: + _restore_active(previous_active, home=home) + raise + return { + "changed": previous is None or previous["sha256"] != digest, + "enabled": should_enable, + "active_release_id": release_dir.name, + "catalog": next_state["catalog"], + "report": _build_report_summary(report, release_dir), + "restart_required": True, + } + + +@administration_locked(".official-admin.lock") +def disable_official(*, home: str | Path | None = None) -> dict[str, Any]: + state = _load_state(home) + if not state["installed_release_ids"]: + raise FileNotFoundError("DeepLaw official library is not installed") + previous_active = _snapshot_active(home) + try: + _remove_active_if_registered(state, home=home) + state = {**state, "enabled": False} + _write_state(state, home=home) + except BaseException: + _restore_active(previous_active, home=home) + raise + return {"enabled": False, "restart_required": True} + + +@administration_locked(".official-admin.lock") +def enable_official(*, home: str | Path | None = None) -> dict[str, Any]: + state = _load_state(home) + release_id = state["active_release_id"] + if release_id is None: + raise FileNotFoundError("DeepLaw official library is not installed") + base_home = ( + Path(home).expanduser().absolute() if home is not None else default_home().absolute() + ) + database = base_home / "releases" / release_id / "deeplaw.sqlite3" + verify_release_artifact(database) + previous_active = _snapshot_active(home) + try: + activate_release(base_home / "releases", release_id) + state = {**state, "enabled": True} + _write_state(state, home=home) + except BaseException: + _restore_active(previous_active, home=home) + raise + return {"enabled": True, "active_release_id": release_id, "restart_required": True} + + +@administration_locked(".official-admin.lock") +def uninstall_official(*, home: str | Path | None = None) -> dict[str, Any]: + state = _load_state(home) + _remove_active_if_registered(state, home=home) + base_home = ( + Path(home).expanduser().absolute() if home is not None else default_home().absolute() + ) + releases = base_home / "releases" + removed: list[str] = [] + if releases.exists(): + if releases.is_symlink() or not releases.is_dir(): + raise RuntimeError("DeepLaw releases directory is unsafe") + for release_id in state["installed_release_ids"]: + release_dir = releases / release_id + if release_dir.is_symlink(): + raise RuntimeError(f"official release must not be a symbolic link: {release_dir}") + if release_dir.exists(): + shutil.rmtree(release_dir) + removed.append(release_id) + root = official_home(home) + if root.is_symlink(): + raise RuntimeError(f"official library directory must not be a symbolic link: {root}") + if root.exists(): + shutil.rmtree(root) + return {"installed": False, "removed_release_ids": removed, "restart_required": True} + + +def official_status(*, home: str | Path | None = None) -> dict[str, Any]: + state = _load_state(home) + return { + **state, + "installed": bool(state["installed_release_ids"]), + "update_catalog_url": DEFAULT_CATALOG_URL, + "trust_model": ( + "HTTPS team catalog, monotonic sequence, catalog SHA-256, " + "and per-source byte size/SHA-256" + ), + } + + +def active_official_release_id(*, home: str | Path | None = None) -> str | None: + state = _load_state(home) + release_id = state["active_release_id"] + if not state["enabled"] or release_id is None: + return None + return release_id if _snapshot_active(home) == release_id else None diff --git a/src/deeplaw/private_library.py b/src/deeplaw/private_library.py new file mode 100644 index 0000000..cb4fe5e --- /dev/null +++ b/src/deeplaw/private_library.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import json +import os +import re +import shutil +import stat +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .admin_lock import administration_locked +from .ingest import build_release +from .store import activate_release, default_home, resolve_active_database +from .util import canonical_date, sha256_file, stable_id + +PRIVATE_LIBRARY_SCHEMA = "deeplaw.private-library/v1" +PRIVATE_LIBRARY_ID = "local-user" +_MAX_STATE_BYTES = 8 * 1024 * 1024 +_MAX_SOURCE_BYTES = 512 * 1024 * 1024 +_RELEASE_ID = re.compile(r"^lawrel_[0-9a-f]{32}$") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_DOCUMENT_TYPES = { + "law", + "administrative_regulation", + "judicial_interpretation", + "prosecution_standard", + "departmental_rule", + "normative_document", + "case_reference", +} +_STATE_FIELDS = { + "schema_version", + "library_id", + "revision", + "active_release_id", + "documents", + "pdf_fallback", + "allow_needs_ocr", +} +_DOCUMENT_FIELDS = { + "document_id", + "title", + "format", + "source_sha256", + "byte_size", + "storage_name", + "document_type", + "issuer", + "effective_from", + "effective_to", + "added_at", +} + + +def private_home(home: str | Path | None = None) -> Path: + root = Path(home).expanduser() if home is not None else default_home() + root = root.absolute() + if root.is_symlink(): + raise RuntimeError(f"DeepLaw home must not be a symbolic link: {root}") + return root / "private" + + +def _secure_directory(path: Path) -> Path: + if path.is_symlink(): + raise RuntimeError(f"private library directory must not be a symbolic link: {path}") + path.mkdir(parents=True, exist_ok=True, mode=0o700) + if not path.is_dir(): + raise RuntimeError(f"private library path is not a directory: {path}") + os.chmod(path, 0o700) + return path + + +def _empty_state() -> dict[str, Any]: + return { + "schema_version": PRIVATE_LIBRARY_SCHEMA, + "library_id": PRIVATE_LIBRARY_ID, + "revision": 0, + "active_release_id": None, + "documents": [], + "pdf_fallback": "off", + "allow_needs_ocr": False, + } + + +def _validate_state(value: Any) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != _STATE_FIELDS: + raise RuntimeError("private library state does not match its closed contract") + if value.get("schema_version") != PRIVATE_LIBRARY_SCHEMA: + raise RuntimeError("unsupported private library state schema") + if value.get("library_id") != PRIVATE_LIBRARY_ID: + raise RuntimeError("unsupported private library identity") + revision = value.get("revision") + if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0: + raise RuntimeError("private library revision is invalid") + active_release_id = value.get("active_release_id") + if active_release_id is not None and ( + not isinstance(active_release_id, str) or not _RELEASE_ID.fullmatch(active_release_id) + ): + raise RuntimeError("private library active release ID is invalid") + documents = value.get("documents") + if not isinstance(documents, list) or len(documents) > 10_000: + raise RuntimeError("private library documents are invalid") + seen_ids: set[str] = set() + seen_hashes: set[str] = set() + for document in documents: + if not isinstance(document, dict) or set(document) != _DOCUMENT_FIELDS: + raise RuntimeError("private document state does not match its closed contract") + document_id = document.get("document_id") + source_sha256 = document.get("source_sha256") + if ( + not isinstance(document_id, str) + or not re.fullmatch(r"doc_[0-9a-f]{24}", document_id) + or document_id in seen_ids + or not isinstance(source_sha256, str) + or not _SHA256.fullmatch(source_sha256) + or source_sha256 in seen_hashes + ): + raise RuntimeError("private document identity is invalid or duplicated") + seen_ids.add(document_id) + seen_hashes.add(source_sha256) + title = document.get("title") + format_name = document.get("format") + storage_name = document.get("storage_name") + byte_size = document.get("byte_size") + if not isinstance(title, str) or not title or len(title) > 500: + raise RuntimeError("private document title is invalid") + suffix = {"DOCX": ".docx", "PDF": ".pdf", "TXT": ".txt"}.get(format_name) + if suffix is None or storage_name != f"{source_sha256}{suffix}": + raise RuntimeError("private document storage identity is invalid") + if isinstance(byte_size, bool) or not isinstance(byte_size, int) or byte_size < 1: + raise RuntimeError("private document byte size is invalid") + if document.get("document_type") not in _DOCUMENT_TYPES: + raise RuntimeError("private document type is invalid") + issuer = document.get("issuer") + if not isinstance(issuer, str) or not issuer or len(issuer) > 200: + raise RuntimeError("private document issuer is invalid") + for field_name in ("effective_from", "effective_to"): + date_value = document.get(field_name) + if date_value is not None: + if not isinstance(date_value, str): + raise RuntimeError(f"private document {field_name} is invalid") + canonical_date(date_value, field=field_name) + if ( + document.get("effective_from") + and document.get("effective_to") + and document["effective_to"] <= document["effective_from"] + ): + raise RuntimeError("private document effective interval is invalid") + added_at = document.get("added_at") + if not isinstance(added_at, str) or len(added_at) > 40: + raise RuntimeError("private document added_at is invalid") + if not documents and active_release_id is not None: + raise RuntimeError("empty private library cannot have an active release") + if value.get("pdf_fallback") not in {"off", "vision-consensus"}: + raise RuntimeError("private library PDF fallback is invalid") + if not isinstance(value.get("allow_needs_ocr"), bool): + raise RuntimeError("private library OCR policy is invalid") + return value + + +def _state_path(root: Path) -> Path: + return root / "library.json" + + +def _load_state(root: Path) -> dict[str, Any]: + if root.is_symlink(): + raise RuntimeError(f"private library directory must not be a symbolic link: {root}") + path = _state_path(root) + if path.is_symlink(): + raise RuntimeError(f"private library state must not be a symbolic link: {path}") + if not path.exists(): + return _empty_state() + if not path.is_file() or path.stat().st_size > _MAX_STATE_BYTES: + raise RuntimeError("private library state is missing, unsafe, or too large") + try: + value = json.loads(path.read_bytes()) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise RuntimeError("private library state cannot be read") from error + return _validate_state(value) + + +def _write_state(root: Path, state: dict[str, Any]) -> None: + _validate_state(state) + path = _state_path(root) + if path.is_symlink(): + raise RuntimeError(f"private library state must not be a symbolic link: {path}") + payload = json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + temporary = path.with_suffix(".tmp") + if temporary.is_symlink(): + raise RuntimeError(f"private state temporary must not be a symbolic link: {temporary}") + temporary.unlink(missing_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + descriptor = os.open(temporary, flags, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + temporary.unlink(missing_ok=True) + raise + os.chmod(temporary, 0o600) + os.replace(temporary, path) + + +def _materialize_source(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + os.link(source, destination) + except OSError: + shutil.copyfile(source, destination) + + +def _manifest_document(document: dict[str, Any]) -> dict[str, Any]: + value: dict[str, Any] = { + "path": f"documents/{document['storage_name']}", + "title": document["title"], + "format": document["format"], + "officialSource": f"private://source/{document['source_sha256']}", + "byteSize": document["byte_size"], + "sha256": document["source_sha256"], + "documentType": document["document_type"], + "issuer": document["issuer"], + "authorityRank": 0, + "status": "unknown", + "note": ( + "用户私有法律资料；未经 DeepLaw 官方团队审核，" + "不得标记或引用为 DeepLaw 官方法源。" + ), + } + if document["effective_from"] is not None: + value["effectiveDate"] = document["effective_from"] + if document["effective_to"] is not None: + value["effectiveTo"] = document["effective_to"] + return value + + +def _remove_active(root: Path) -> None: + active = root / "ACTIVE" + if active.is_symlink(): + raise RuntimeError(f"private ACTIVE pointer must not be a symbolic link: {active}") + active.unlink(missing_ok=True) + + +def _read_active(root: Path) -> str | None: + active = root / "ACTIVE" + if active.is_symlink(): + raise RuntimeError(f"private ACTIVE pointer must not be a symbolic link: {active}") + if not active.exists(): + return None + if not active.is_file() or active.stat().st_size > 128: + raise RuntimeError("private ACTIVE pointer is unsafe") + release_id = active.read_text(encoding="utf-8").strip() + if not _RELEASE_ID.fullmatch(release_id): + raise RuntimeError("private ACTIVE pointer is invalid") + return release_id + + +def _restore_active(root: Path, release_id: str | None) -> None: + if release_id is None: + _remove_active(root) + return + activate_release(root / "releases", release_id) + os.chmod(root / "ACTIVE", 0o600) + + +def _cleanup_releases(root: Path, *, keep: str | None) -> None: + releases = root / "releases" + if not releases.exists(): + return + if releases.is_symlink() or not releases.is_dir(): + raise RuntimeError("private releases directory is unsafe") + for child in releases.iterdir(): + if child.name == keep: + continue + if child.is_symlink(): + raise RuntimeError(f"private release must not be a symbolic link: {child}") + if child.is_dir(): + shutil.rmtree(child) + + +def _cleanup_sources(root: Path, documents: list[dict[str, Any]]) -> None: + sources = root / "sources" + if not sources.exists(): + return + if sources.is_symlink() or not sources.is_dir(): + raise RuntimeError("private sources directory is unsafe") + retained = {document["storage_name"] for document in documents} + for child in sources.iterdir(): + if child.name not in retained: + if child.is_symlink() or not child.is_file(): + raise RuntimeError(f"private source cache contains an unsafe entry: {child}") + child.unlink() + + +def _publish_snapshot( + root: Path, + state: dict[str, Any], + documents: list[dict[str, Any]], + *, + pdf_fallback: str, + allow_needs_ocr: bool, +) -> dict[str, Any]: + previous_active = _read_active(root) + effective_pdf_fallback = ( + "vision-consensus" + if "vision-consensus" in {state["pdf_fallback"], pdf_fallback} + else "off" + ) + effective_allow_needs_ocr = state["allow_needs_ocr"] or allow_needs_ocr + next_state = { + **state, + "revision": state["revision"] + 1, + "documents": documents, + "pdf_fallback": effective_pdf_fallback, + "allow_needs_ocr": effective_allow_needs_ocr, + } + if not documents: + next_state["active_release_id"] = None + try: + _remove_active(root) + _write_state(root, next_state) + except BaseException: + _restore_active(root, previous_active) + raise + _cleanup_releases(root, keep=None) + _cleanup_sources(root, documents) + return next_state + + workspace = Path(tempfile.mkdtemp(prefix=".private-build-", dir=root)) + try: + source_root = workspace / "source" + for document in documents: + source = root / "sources" / document["storage_name"] + if source.is_symlink() or not source.is_file(): + raise RuntimeError( + f"private source is missing or unsafe: {document['document_id']}" + ) + if source.stat().st_size != document["byte_size"] or sha256_file(source) != document[ + "source_sha256" + ]: + raise RuntimeError( + f"private source failed integrity check: {document['document_id']}" + ) + _materialize_source(source, source_root / "documents" / document["storage_name"]) + manifest = { + "package": { + "name": "DeepLaw user private legal library", + "documentCount": len(documents), + }, + "documents": [_manifest_document(document) for document in documents], + } + manifest_path = workspace / "manifest.json" + manifest_path.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + release_dir, _ = build_release( + source_root=source_root, + manifest_path=manifest_path, + output_root=root / "releases", + pdf_fallback=effective_pdf_fallback, + allow_needs_ocr=effective_allow_needs_ocr, + source_scope="user_private", + library_id=PRIVATE_LIBRARY_ID, + artifact_mode=0o400, + ) + try: + activate_release(root / "releases", release_dir.name) + os.chmod(root / "ACTIVE", 0o600) + next_state["active_release_id"] = release_dir.name + _write_state(root, next_state) + except BaseException: + _restore_active(root, previous_active) + raise + _cleanup_releases(root, keep=release_dir.name) + _cleanup_sources(root, documents) + return next_state + finally: + shutil.rmtree(workspace, ignore_errors=True) + + +@administration_locked(".private-admin.lock") +def add_private_document( + source: str | Path, + *, + title: str | None = None, + document_type: str = "normative_document", + issuer: str = "用户提供（未经 DeepLaw 官方审核）", + effective_from: str | None = None, + effective_to: str | None = None, + confirm_no_case_data: bool = False, + pdf_fallback: str = "off", + allow_needs_ocr: bool = False, + home: str | Path | None = None, +) -> dict[str, Any]: + if not confirm_no_case_data: + raise ValueError( + "private add requires explicit confirmation that the file is a legal reference, " + "not Analytix case material" + ) + declared_source = Path(source).expanduser() + if declared_source.is_symlink(): + raise ValueError("private source must not be a symbolic link") + resolved_source = declared_source.resolve(strict=True) + if not stat.S_ISREG(resolved_source.stat().st_mode): + raise ValueError("private source must be a regular file") + if resolved_source.stat().st_size > _MAX_SOURCE_BYTES: + raise ValueError("private source exceeds the 512 MiB limit") + suffix = resolved_source.suffix.lower() + format_name = {".docx": "DOCX", ".pdf": "PDF", ".txt": "TXT"}.get(suffix) + if format_name is None: + if suffix == ".doc": + raise ValueError("legacy DOC is unsupported; convert it to DOCX, PDF, or UTF-8 TXT") + raise ValueError("private source format must be DOCX, PDF, or UTF-8 TXT") + normalized_title = (title or resolved_source.stem).strip() + if not normalized_title or len(normalized_title) > 500: + raise ValueError("private document title must contain 1 to 500 characters") + if document_type not in _DOCUMENT_TYPES: + raise ValueError(f"unsupported private document type: {document_type}") + normalized_issuer = issuer.strip() + if not normalized_issuer or len(normalized_issuer) > 200: + raise ValueError("private document issuer must contain 1 to 200 characters") + for field_name, value in (("effective_from", effective_from), ("effective_to", effective_to)): + if value is not None: + canonical_date(value, field=field_name) + if effective_from and effective_to and effective_to <= effective_from: + raise ValueError("effective_to must be after effective_from") + + root = _secure_directory(private_home(home)) + sources = _secure_directory(root / "sources") + _secure_directory(root / "releases") + state = _load_state(root) + source_sha256 = sha256_file(resolved_source) + document_id = stable_id("doc", source_sha256, normalized_title) + if any( + item["document_id"] == document_id or item["source_sha256"] == source_sha256 + for item in state["documents"] + ): + raise ValueError("private library already contains this source") + storage_name = f"{source_sha256}{suffix}" + stored_source = sources / storage_name + created_source = False + if stored_source.exists(): + if stored_source.is_symlink() or not stored_source.is_file(): + raise RuntimeError("private source cache entry is unsafe") + if ( + stored_source.stat().st_size != resolved_source.stat().st_size + or sha256_file(stored_source) != source_sha256 + ): + raise RuntimeError("private source cache entry failed integrity validation") + else: + temporary = sources / f".{storage_name}.tmp" + temporary.unlink(missing_ok=True) + shutil.copyfile(resolved_source, temporary) + if sha256_file(temporary) != source_sha256: + temporary.unlink(missing_ok=True) + raise RuntimeError("private source changed while it was copied") + os.chmod(temporary, 0o600) + os.replace(temporary, stored_source) + created_source = True + + document = { + "document_id": document_id, + "title": normalized_title, + "format": format_name, + "source_sha256": source_sha256, + "byte_size": resolved_source.stat().st_size, + "storage_name": storage_name, + "document_type": document_type, + "issuer": normalized_issuer, + "effective_from": effective_from, + "effective_to": effective_to, + "added_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + } + try: + next_state = _publish_snapshot( + root, + state, + [*state["documents"], document], + pdf_fallback=pdf_fallback, + allow_needs_ocr=allow_needs_ocr, + ) + except BaseException: + if created_source: + stored_source.unlink(missing_ok=True) + raise + return { + "document": document, + "active_release_id": next_state["active_release_id"], + "document_count": len(next_state["documents"]), + } + + +@administration_locked(".private-admin.lock") +def delete_private_document( + document_id: str, + *, + pdf_fallback: str = "off", + allow_needs_ocr: bool = False, + home: str | Path | None = None, +) -> dict[str, Any]: + root = _secure_directory(private_home(home)) + state = _load_state(root) + documents = [item for item in state["documents"] if item["document_id"] != document_id] + if len(documents) == len(state["documents"]): + raise KeyError(f"unknown private document: {document_id}") + next_state = _publish_snapshot( + root, + state, + documents, + pdf_fallback=pdf_fallback, + allow_needs_ocr=allow_needs_ocr, + ) + return { + "deleted_document_id": document_id, + "active_release_id": next_state["active_release_id"], + "document_count": len(documents), + "restart_required": True, + } + + +def list_private_documents(*, home: str | Path | None = None) -> dict[str, Any]: + root = private_home(home) + state = _load_state(root) if root.exists() else _empty_state() + documents = [ + {key: value for key, value in document.items() if key != "storage_name"} + for document in state["documents"] + ] + return { + "schema_version": PRIVATE_LIBRARY_SCHEMA, + "library_id": PRIVATE_LIBRARY_ID, + "revision": state["revision"], + "active_release_id": state["active_release_id"], + "document_count": len(documents), + "documents": documents, + "access_model": "local_os_user", + "pdf_fallback": state["pdf_fallback"], + "allow_needs_ocr": state["allow_needs_ocr"], + } + + +def resolve_private_database( + *, + explicit_db: str | Path | None = None, + home: str | Path | None = None, +) -> Path: + if explicit_db is None: + explicit_db = os.environ.get("DEEPLAW_PRIVATE_DB") + return resolve_active_database( + explicit_db=explicit_db, + home=private_home(home), + use_env_db=False, + ) + + +def active_private_release_id(*, home: str | Path | None = None) -> str | None: + return _read_active(private_home(home)) diff --git a/src/deeplaw/search.py b/src/deeplaw/search.py index c01b220..41fdf97 100644 --- a/src/deeplaw/search.py +++ b/src/deeplaw/search.py @@ -188,7 +188,13 @@ def _target_query_forms(query: str) -> tuple[str, ...]: class DeepLaw: - def __init__(self, database: str | Path | None = None, *, home: str | Path | None = None): + def __init__( + self, + database: str | Path | None = None, + *, + home: str | Path | None = None, + expected_scope: Literal["official", "user_private"] | None = None, + ): self.database = resolve_active_database(explicit_db=database, home=home) self.artifact = verify_release_artifact(self.database) self.connection = connect_readonly(self.database) @@ -201,6 +207,12 @@ def __init__(self, database: str | Path | None = None, *, home: str | Path | Non if self.artifact.get("release_id") != self.release_id: self.connection.close() raise RuntimeError("release database metadata does not match release.json") + self.collection_scope = str(self.artifact.get("collection_scope", "official")) + if expected_scope is not None and self.collection_scope != expected_scope: + self.connection.close() + raise RuntimeError( + f"expected {expected_scope} release, got {self.collection_scope}" + ) release = self.info.get("release", {}) artifact_release = { key: value for key, value in self.artifact.items() if key != "database_sha256" @@ -256,7 +268,7 @@ def search(self, request: SearchRequest) -> SearchResponse: self._target_document_ids(request.query) ) candidates = self._candidate_rows(request, route) - primary_rows: list[tuple[sqlite3.Row, str]] = [] + ranked_rows: list[tuple[sqlite3.Row, str, bool]] = [] uncertain_rows: list[tuple[sqlite3.Row, str]] = [] temporal_outside_count = 0 seen: set[tuple[str, str | None]] = set() @@ -276,47 +288,41 @@ def search(self, request: SearchRequest) -> SearchResponse: ) if temporal_classification == "outside_effective_interval": temporal_outside_count += 1 - elif temporal_classification == "unverified_metadata": + elif ( + temporal_classification == "unverified_metadata" + or bool(row["extraction_review_required"]) + ): uncertain_rows.append((row, temporal_classification)) + ranked_rows.append((row, temporal_classification, True)) else: - primary_rows.append((row, temporal_classification)) + ranked_rows.append((row, temporal_classification, False)) evidence: list[EvidenceCard] = [] uncertain_evidence: list[EvidenceCard] = [] used_characters = 0 result_limit = min(request.limit, 3) if route in {"navigation", "exact"} else request.limit - def append_cards( - rows: list[tuple[sqlite3.Row, str]], - target: list[EvidenceCard], - ) -> None: - nonlocal used_characters - for row, temporal_classification in rows: - if len(evidence) + len(uncertain_evidence) >= result_limit: - break - budget = min( - 800 if route != "navigation" else 320, - request.max_chars - used_characters, - ) - if budget < 100: - break - card = self._card_from_row( - row, - request, - route=route, - max_excerpt_chars=budget, - temporal_classification=cast( - Literal[ - "not_evaluated", "verified_in_scope", "unverified_metadata" - ], - temporal_classification, - ), - ) - used_characters += len(card.excerpt) - target.append(card) - - append_cards(primary_rows, evidence) - append_cards(uncertain_rows, uncertain_evidence) + for row, temporal_classification, is_uncertain in ranked_rows: + if len(evidence) + len(uncertain_evidence) >= result_limit: + break + budget = min( + 800 if route != "navigation" else 320, + request.max_chars - used_characters, + ) + if budget < 100: + break + card = self._card_from_row( + row, + request, + route=route, + max_excerpt_chars=budget, + temporal_classification=cast( + Literal["not_evaluated", "verified_in_scope", "unverified_metadata"], + temporal_classification, + ), + ) + used_characters += len(card.excerpt) + (uncertain_evidence if is_uncertain else evidence).append(card) graph_paths = self._graph_paths( tuple(evidence), @@ -330,11 +336,15 @@ def append_cards( graph_paths=graph_paths, as_of=request.as_of, ) + temporal_uncertain_count = sum( + temporal_classification == "unverified_metadata" + for _, temporal_classification in uncertain_rows + ) gaps = self._search_gaps( route=route, exact_target_resolved=exact_target_resolved, temporal_intent=temporal_intent, - temporal_uncertain_count=len(uncertain_rows), + temporal_uncertain_count=temporal_uncertain_count, temporal_outside_count=temporal_outside_count, evidence_count=len(evidence), obligation_coverage=obligation_coverage, @@ -344,12 +354,26 @@ def append_cards( "检索结果是研究证据候选，不等同于本案法律适用结论。", "DeepLaw 未使用模型记忆、自动 Web 回退或向量 top-k 注入。", ] + if self.collection_scope == "user_private": + notices.insert( + 0, + "当前结果来自用户私有资料库，未经 DeepLaw 官方团队审核，" + "不得冒充官方法源。", + ) all_returned_evidence = (*evidence, *uncertain_evidence) - if uncertain_evidence: + if any( + card.temporal_classification == "unverified_metadata" + for card in uncertain_evidence + ): notices.append( "时效检索中，效力起点缺失或状态未验证的候选已从主证据分离；" "正式引用前必须复核。" ) + if any(card.extraction_review_required for card in uncertain_evidence): + notices.append( + "存在未完成人工对照的抽取风险，相关候选已从主证据分离；" + "引用前必须按页对照原件。" + ) if temporal_outside_count: notices.append( f"另有 {temporal_outside_count} 项候选按已知状态或效力区间不属于目标时点，" @@ -367,7 +391,7 @@ def append_cards( if not evidence: if uncertain_evidence: notices.append( - "当前 release 未形成已验证的主证据；不确定候选不得替代正式时点核验。" + "当前 release 未形成已验证的主证据；不确定候选必须先解决时效或抽取风险。" ) else: notices.append("当前 release 未找到足够证据；这不表示相关法律不存在。") diff --git a/src/deeplaw/store.py b/src/deeplaw/store.py index 1240dad..a7227b0 100644 --- a/src/deeplaw/store.py +++ b/src/deeplaw/store.py @@ -43,6 +43,8 @@ "reviewer_kind", "review_scope", "review_covered_documents", + "collection_scope", + "library_id", } @@ -161,6 +163,25 @@ def _validate_release_manifest(manifest: Any, *, directory_name: str) -> dict[st manifest.get("derived_wiki"), bool ): raise RuntimeError("release derived-index flags are invalid") + collection_scope = manifest.get("collection_scope", "official") + if collection_scope not in {"official", "user_private"}: + raise RuntimeError("release collection_scope is invalid") + library_id = manifest.get("library_id") + if collection_scope == "official" and library_id is not None: + raise RuntimeError("official release must not declare a private library_id") + if collection_scope == "user_private": + if ( + not isinstance(library_id, str) + or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", library_id) + ): + raise RuntimeError("user-private release library_id is invalid") + if ( + manifest["temporal_status"] != "requires_human_review" + or manifest["redistribution_status"] != "not_assessed" + or manifest["vector_index"] + or manifest["derived_wiki"] + ): + raise RuntimeError("user-private release cannot claim official review authority") return manifest @@ -469,13 +490,14 @@ def resolve_active_database( *, explicit_db: str | Path | None = None, home: str | Path | None = None, + use_env_db: bool = True, ) -> Path: if explicit_db: database = Path(explicit_db).expanduser().absolute() if not database.exists(): raise FileNotFoundError(database) return database - env_db = os.environ.get("DEEPLAW_DB") + env_db = os.environ.get("DEEPLAW_DB") if use_env_db else None if env_db: database = Path(env_db).expanduser().absolute() if not database.exists(): @@ -515,9 +537,26 @@ def activate_release(output_root: Path, release_id: str) -> Path: if not _RELEASE_ID.fullmatch(release_id): raise ValueError(f"invalid DeepLaw release ID: {release_id}") var_root = output_root.parent + if var_root.is_symlink() or output_root.is_symlink(): + raise RuntimeError("DeepLaw home and releases directory must not be symbolic links") + database = output_root / release_id / "deeplaw.sqlite3" + verify_release_artifact(database) active = var_root / "ACTIVE" + if active.is_symlink(): + raise RuntimeError(f"DeepLaw ACTIVE pointer must not be a symbolic link: {active}") temporary = active.with_suffix(".tmp") - temporary.write_text(f"{release_id}\n", encoding="utf-8") + if temporary.is_symlink(): + raise RuntimeError(f"DeepLaw ACTIVE temporary must not be a symbolic link: {temporary}") + temporary.unlink(missing_ok=True) + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(f"{release_id}\n") + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + temporary.unlink(missing_ok=True) + raise os.replace(temporary, active) return active diff --git a/tests/test_library_scopes.py b/tests/test_library_scopes.py new file mode 100644 index 0000000..1218451 --- /dev/null +++ b/tests/test_library_scopes.py @@ -0,0 +1,603 @@ +from __future__ import annotations + +import asyncio +import json +import os +import stat +import sys +from pathlib import Path +from urllib.error import HTTPError + +import pytest +from jsonschema import Draft202012Validator +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + +import deeplaw.official as official_module +from deeplaw.ingest import build_release +from deeplaw.mcp_server import handle_support, tool_definition +from deeplaw.models import SearchRequest +from deeplaw.official import ( + bundled_catalog_path, + disable_official, + enable_official, + official_status, + sync_official, + uninstall_official, +) +from deeplaw.private_library import ( + add_private_document, + delete_private_document, + list_private_documents, + private_home, + resolve_private_database, +) +from deeplaw.search import DeepLaw +from deeplaw.store import resolve_active_database + +from .helpers import manifest_document, write_docx, write_manifest + + +def _write_catalog( + path: Path, + documents: list[dict[str, object]], + *, + sequence: int, +) -> Path: + value = { + "schemaVersion": "deeplaw.official-catalog/v1", + "catalogId": "deeplaw-test-official", + "sequence": sequence, + "version": f"2026.07.15.{sequence}", + "publishedOn": "2026-07-15", + "package": { + "name": "DeepLaw official test catalog", + "retrievedOn": "2026-07-15", + "reviewedOn": "2026-07-15", + "documentCount": len(documents), + }, + "documents": documents, + } + path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + return path + + +def test_bundled_official_catalog_matches_its_public_contract() -> None: + repository = Path(__file__).resolve().parents[1] + schema = json.loads( + (repository / "contracts/official-catalog.v1.schema.json").read_text(encoding="utf-8") + ) + catalog = json.loads(bundled_catalog_path().read_text(encoding="utf-8")) + + Draft202012Validator(schema).validate(catalog) + assert catalog["catalogId"] == "deeplaw-cn-official" + assert catalog["package"]["documentCount"] == 28 + assert len(catalog["documents"]) == 28 + assert catalog["buildPolicy"] == { + "pdfFallback": "vision-consensus", + "allowNeedsOcr": True, + } + assert catalog["reviewOverlay"]["resource"] == "core-2026-07-14.ai-review.json" + + +def test_official_install_update_disable_enable_and_uninstall(tmp_path: Path) -> None: + source = tmp_path / "source" + first = source / "中华人民共和国测试法.docx" + write_docx(first, ["中华人民共和国测试法", "第一条 官方目录第一版。"]) + documents = [manifest_document(source, first.name, title="中华人民共和国测试法")] + catalog_v1 = _write_catalog(tmp_path / "catalog-v1.json", documents, sequence=1) + home = tmp_path / "home" + + installed = sync_official( + catalog_source=catalog_v1, + source_root=source, + home=home, + ) + release_v1 = installed["active_release_id"] + database_v1 = resolve_active_database(home=home) + with DeepLaw(database_v1, expected_scope="official") as law: + result = law.search(SearchRequest(query="中华人民共和国测试法 第一条")) + assert result.evidence[0].title == "中华人民共和国测试法" + assert official_status(home=home)["catalog"]["sequence"] == 1 + + with pytest.MonkeyPatch.context() as patch: + patch.setattr( + official_module, + "build_release", + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("unchanged catalog must reuse its immutable release") + ), + ) + unchanged = sync_official( + catalog_source=catalog_v1, + source_root=source, + update=True, + home=home, + ) + assert unchanged["changed"] is False + assert unchanged["restart_required"] is False + assert unchanged["report"]["cached"] is True + + second = source / "测试实施办法.docx" + write_docx(second, ["测试实施办法", "第一条 官方目录新增法律资料用于版本更新验证。"]) + documents_v2 = [ + *documents, + manifest_document(source, second.name, title="测试实施办法"), + ] + catalog_v2 = _write_catalog(tmp_path / "catalog-v2.json", documents_v2, sequence=2) + updated = sync_official( + catalog_source=catalog_v2, + source_root=source, + update=True, + home=home, + ) + release_v2 = updated["active_release_id"] + + assert release_v2 != release_v1 + assert (home / "releases" / release_v1 / "deeplaw.sqlite3").is_file() + assert resolve_active_database(home=home).parent.name == release_v2 + with DeepLaw(resolve_active_database(home=home), expected_scope="official") as law: + result = law.search(SearchRequest(query="测试实施办法 第一条")) + assert result.evidence[0].title == "测试实施办法" + + disable_official(home=home) + with pytest.raises(FileNotFoundError): + resolve_active_database(home=home) + assert (home / "releases" / release_v2 / "deeplaw.sqlite3").is_file() + + enable_official(home=home) + assert resolve_active_database(home=home).parent.name == release_v2 + + rewritten = json.loads(catalog_v2.read_text(encoding="utf-8")) + rewritten["version"] = "rewritten" + rewritten_path = tmp_path / "rewritten.json" + rewritten_path.write_text(json.dumps(rewritten), encoding="utf-8") + with pytest.raises(ValueError, match="rewritten"): + sync_official( + catalog_source=rewritten_path, + source_root=source, + update=True, + home=home, + ) + + uninstalled = uninstall_official(home=home) + assert set(uninstalled["removed_release_ids"]) == {release_v1, release_v2} + assert not (home / "official").exists() + with pytest.raises(FileNotFoundError): + resolve_active_database(home=home) + + +def test_official_install_downloads_catalog_sources_and_verifies_hashes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "downloaded.docx" + write_docx(source, ["官方下载测试法", "第一条 下载内容必须通过大小和哈希校验。"]) + document = manifest_document(tmp_path, source.name, title="官方下载测试法") + document["officialSource"] = "https://example.gov.cn/downloaded.docx" + catalog_path = _write_catalog(tmp_path / "catalog.json", [document], sequence=1) + catalog_payload = catalog_path.read_bytes() + source_payload = source.read_bytes() + calls: list[str] = [] + + def fake_download(url: str, *, maximum: int, timeout: float = 60.0) -> bytes: + calls.append(url) + assert maximum > 0 + assert timeout > 0 + if url.endswith("catalog.json"): + return catalog_payload + raise AssertionError(f"unexpected download URL: {url}") + + def fake_source_download(item: dict[str, object], destination: Path) -> None: + calls.append(str(item["officialSource"])) + assert item["byteSize"] == len(source_payload) + destination.write_bytes(source_payload) + + monkeypatch.setattr(official_module, "_download_bytes", fake_download) + monkeypatch.setattr(official_module, "_download_source", fake_source_download) + home = tmp_path / "home" + result = sync_official( + catalog_source="https://catalog.example/catalog.json", + home=home, + ) + + assert result["report"]["document_count"] == 1 + assert result["report"]["warning_count"] == 0 + assert "documents" not in result["report"] + assert calls == [ + "https://catalog.example/catalog.json", + "https://example.gov.cn/downloaded.docx", + ] + cached_sources = list((home / "official" / "sources").iterdir()) + assert len(cached_sources) == 1 + assert cached_sources[0].read_bytes() == source_payload + + +def test_official_source_download_streams_and_fails_closed_on_hash_change( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "source.docx" + write_docx(source, ["流式下载测试法", "第一条 下载器不得把大型法源一次性装入内存。"]) + payload = source.read_bytes() + document = manifest_document(tmp_path, source.name, title="流式下载测试法") + document["officialSource"] = "https://example.gov.cn/source.docx" + + class FakeResponse: + def __init__(self, value: bytes): + self.value = value + self.offset = 0 + + def __enter__(self) -> FakeResponse: + return self + + def __exit__(self, *_: object) -> None: + return None + + def geturl(self) -> str: + return str(document["officialSource"]) + + def read(self, size: int) -> bytes: + chunk = self.value[self.offset : self.offset + min(size, 17)] + self.offset += len(chunk) + return chunk + + def fake_urlopen(*_: object, **__: object) -> FakeResponse: + return FakeResponse(payload) + + monkeypatch.setattr(official_module, "urlopen", fake_urlopen) + destination = tmp_path / "cache" / "source.docx" + destination.parent.mkdir() + official_module._download_source(document, destination) + assert destination.read_bytes() == payload + + destination.unlink() + document["sha256"] = "0" * 64 + with pytest.raises(RuntimeError, match="SHA-256"): + official_module._download_source(document, destination) + assert not destination.exists() + assert not list(destination.parent.glob(".*.tmp")) + + +def test_official_source_materialization_cleans_failed_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + document = { + "path": "failed.docx", + "officialSource": "https://example.gov.cn/failed.docx", + "byteSize": 123, + "sha256": "1" * 64, + } + catalog = {"documents": [document]} + home = tmp_path / "home" + monkeypatch.setattr( + official_module, + "_download_source", + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("network failed")), + ) + + with pytest.raises(RuntimeError, match="source download failed"): + official_module._materialize_downloaded_sources(catalog, home=home) + + official_root = home / "official" + assert not list(official_root.glob(".official-sources-*")) + + +def test_national_laws_database_download_envelope_is_resolved_and_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + document = { + "path": "中华人民共和国测试法.docx", + "officialSource": ( + "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=test" + ), + } + signed_url = ( + "https://flkoss.obs-bj2.cucloud.cn/prod/test.docx?" + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=test" + ) + observed: dict[str, object] = {} + + def fake_download(url: str, *, maximum: int, timeout: float = 60.0) -> bytes: + observed.update(url=url, maximum=maximum, timeout=timeout) + return json.dumps({"code": 200, "data": {"url": signed_url}}).encode() + + monkeypatch.setattr(official_module, "_download_bytes", fake_download) + + assert official_module._resolve_source_download_url(document) == signed_url + assert observed == { + "url": document["officialSource"], + "maximum": 64 * 1024, + "timeout": 60.0, + } + + monkeypatch.setattr( + official_module, + "_download_bytes", + lambda *_args, **_kwargs: json.dumps( + { + "code": 200, + "data": {"url": "https://attacker.example/test.docx"}, + } + ).encode(), + ) + with pytest.raises(RuntimeError, match="unsafe"): + official_module._resolve_source_download_url(document) + + +def test_official_download_retries_only_transient_transport_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + attempts = 0 + delays: list[float] = [] + + class Response: + def close(self) -> None: + return None + + def fake_urlopen(*_args: object, **_kwargs: object) -> Response: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise HTTPError("https://example.gov.cn/source", 502, "bad gateway", {}, None) + return Response() + + monkeypatch.setattr(official_module, "urlopen", fake_urlopen) + monkeypatch.setattr(official_module.time, "sleep", delays.append) + response = official_module._urlopen_with_retry( + official_module.Request("https://example.gov.cn/source"), + timeout=1.0, + ) + + assert isinstance(response, Response) + assert attempts == 3 + assert delays == [0.5, 1.0] + + +def test_private_library_is_physical_separate_explicit_and_deletable(tmp_path: Path) -> None: + home = tmp_path / "home" + official_source = tmp_path / "official-source" + official_document = official_source / "official.docx" + write_docx(official_document, ["中华人民共和国官方测试法", "第一条 官方内容。"]) + official_manifest = write_manifest( + official_source / "manifest.json", + [ + manifest_document( + official_source, + official_document.name, + title="中华人民共和国官方测试法", + ) + ], + ) + official_release, _ = build_release( + source_root=official_source, + manifest_path=official_manifest, + output_root=home / "releases", + activate=True, + ) + official_hash = (official_release / "release.json").read_bytes() + + private_source = tmp_path / "user-upload.docx" + write_docx( + private_source, + ["用户研究资料", "第一条 用户私有法律资料内容用于隔离检索验证。"], + ) + with pytest.raises(ValueError, match="confirmation"): + add_private_document(private_source, home=home) + + added = add_private_document( + private_source, + title="用户研究资料", + confirm_no_case_data=True, + home=home, + ) + document_id = added["document"]["document_id"] + private_database = resolve_private_database(home=home) + root = private_home(home) + + assert private_database.parent.parent == root / "releases" + assert stat.S_IMODE(root.stat().st_mode) == 0o700 + assert stat.S_IMODE((root / "library.json").stat().st_mode) == 0o600 + assert stat.S_IMODE(private_database.stat().st_mode) == 0o400 + private_sources = list((root / "sources").iterdir()) + assert len(private_sources) == 1 + assert stat.S_IMODE(private_sources[0].stat().st_mode) == 0o600 + assert str(private_source) not in (root / "library.json").read_text(encoding="utf-8") + assert (official_release / "release.json").read_bytes() == official_hash + + with DeepLaw(private_database, expected_scope="user_private") as law: + response = law.search(SearchRequest(query="用户研究资料 第一条")) + card = response.evidence[0] + assert response.notices[0].startswith("当前结果来自用户私有资料库") + assert card.authority_rank == 0 + assert card.official_source.startswith("private://source/") + with pytest.raises(RuntimeError, match="expected official"): + DeepLaw(private_database, expected_scope="official") + + mcp_result = handle_support( + operation="private_search", + query="用户研究资料 第一条", + private_database=private_database, + ) + assert mcp_result["release_id"] == private_database.parent.name + assert mcp_result["evidence"][0]["title"] == "用户研究资料" + assert list_private_documents(home=home)["document_count"] == 1 + + deleted = delete_private_document(document_id, home=home) + assert deleted["document_count"] == 0 + assert deleted["restart_required"] is True + assert not (root / "ACTIVE").exists() + assert not list((root / "sources").iterdir()) + assert not list((root / "releases").iterdir()) + with pytest.raises(FileNotFoundError): + resolve_private_database(home=home) + assert resolve_active_database(home=home) == official_release / "deeplaw.sqlite3" + assert (official_release / "release.json").read_bytes() == official_hash + + +def test_official_update_and_uninstall_do_not_mutate_private_library(tmp_path: Path) -> None: + home = tmp_path / "home" + official_source = tmp_path / "official-source" + first = official_source / "official-v1.docx" + write_docx(first, ["官方第一版测试法", "第一条 官方第一版内容。"]) + first_documents = [manifest_document(official_source, first.name, title="官方第一版测试法")] + catalog_v1 = _write_catalog(tmp_path / "catalog-v1.json", first_documents, sequence=1) + sync_official(catalog_source=catalog_v1, source_root=official_source, home=home) + + private_source = tmp_path / "private-reference.txt" + private_source.write_text( + "用户私有参考规范\n第一条 官方目录更新和卸载不得改动本资料。\n", + encoding="utf-8", + ) + add_private_document( + private_source, + title="用户私有参考规范", + confirm_no_case_data=True, + home=home, + ) + private_root = private_home(home) + private_database = resolve_private_database(home=home) + state_before = (private_root / "library.json").read_bytes() + database_before = private_database.read_bytes() + + second = official_source / "official-v2.docx" + write_docx(second, ["官方第二版测试法", "第一条 官方目录新增内容。"]) + catalog_v2 = _write_catalog( + tmp_path / "catalog-v2.json", + [ + *first_documents, + manifest_document(official_source, second.name, title="官方第二版测试法"), + ], + sequence=2, + ) + sync_official( + catalog_source=catalog_v2, + source_root=official_source, + update=True, + home=home, + ) + uninstall_official(home=home) + + assert (private_root / "library.json").read_bytes() == state_before + assert private_database.read_bytes() == database_before + assert resolve_private_database(home=home) == private_database + with DeepLaw(private_database, expected_scope="user_private") as law: + response = law.search(SearchRequest(query="用户私有参考规范")) + assert response.evidence[0].title == "用户私有参考规范" + + +def test_private_txt_is_supported_but_legacy_doc_is_rejected(tmp_path: Path) -> None: + home = tmp_path / "home" + text_source = tmp_path / "reference.txt" + text_source.write_text("用户法律参考资料\n第一条 UTF-8 文本资料可以检索。\n", encoding="utf-8") + + result = add_private_document( + text_source, + confirm_no_case_data=True, + home=home, + ) + with DeepLaw(resolve_private_database(home=home), expected_scope="user_private") as law: + response = law.search(SearchRequest(query="UTF-8 文本资料")) + assert result["document"]["format"] == "TXT" + assert response.evidence + + legacy = tmp_path / "legacy.doc" + legacy.write_bytes(b"not a supported legacy Word file") + with pytest.raises(ValueError, match="legacy DOC"): + add_private_document(legacy, confirm_no_case_data=True, home=home) + + +def test_mcp_private_surface_stays_read_only_and_explicit() -> None: + tool = tool_definition() + operations = { + branch["properties"]["operation"].get("const", "search") + for branch in tool.inputSchema["oneOf"] + } + + assert {"private_search", "private_get", "private_verify", "private_info"} <= operations + assert not {"add", "upload", "delete", "update", "remember"} & operations + assert tool.annotations.readOnlyHint is True + assert tool.annotations.destructiveHint is False + + +def test_mcp_rejects_private_reads_after_the_snapshot_is_deleted(tmp_path: Path) -> None: + home = tmp_path / "home" + source = tmp_path / "private.docx" + write_docx(source, ["私有删除测试资料", "第一条 删除后旧 MCP 进程必须拒绝继续读取。"]) + added = add_private_document(source, confirm_no_case_data=True, home=home) + + async def exercise() -> None: + environment = {**os.environ, "DEEPLAW_HOME": str(home)} + environment.pop("DEEPLAW_DB", None) + environment.pop("DEEPLAW_PRIVATE_DB", None) + parameters = StdioServerParameters( + command=sys.executable, + args=["-m", "deeplaw", "mcp", "--stdio"], + cwd=Path(__file__).resolve().parents[1], + env=environment, + ) + async with ( + stdio_client(parameters) as (read_stream, write_stream), + ClientSession(read_stream, write_stream) as session, + ): + await session.initialize() + before = await session.call_tool( + "law_support", + {"operation": "private_search", "query": "私有删除测试资料 第一条"}, + ) + assert before.isError is False + + delete_private_document(added["document"]["document_id"], home=home) + after = await session.call_tool( + "law_support", + {"operation": "private_info"}, + ) + assert after.isError is True + assert "restart" in str(after.content).lower() + + asyncio.run(exercise()) + + +def test_mcp_rejects_official_reads_after_the_catalog_is_disabled(tmp_path: Path) -> None: + home = tmp_path / "home" + source_root = tmp_path / "source" + source = source_root / "official.docx" + write_docx(source, ["官方停用测试法", "第一条 停用后旧 MCP 进程必须拒绝继续读取。"]) + catalog = _write_catalog( + tmp_path / "catalog.json", + [manifest_document(source_root, source.name, title="官方停用测试法")], + sequence=1, + ) + sync_official(catalog_source=catalog, source_root=source_root, home=home) + + async def exercise() -> None: + environment = {**os.environ, "DEEPLAW_HOME": str(home)} + environment.pop("DEEPLAW_DB", None) + environment.pop("DEEPLAW_PRIVATE_DB", None) + parameters = StdioServerParameters( + command=sys.executable, + args=["-m", "deeplaw", "mcp", "--stdio"], + cwd=Path(__file__).resolve().parents[1], + env=environment, + ) + async with ( + stdio_client(parameters) as (read_stream, write_stream), + ClientSession(read_stream, write_stream) as session, + ): + await session.initialize() + before = await session.call_tool( + "law_support", + {"operation": "release_info"}, + ) + assert before.isError is False + + disable_official(home=home) + after = await session.call_tool( + "law_support", + {"operation": "release_info"}, + ) + assert after.isError is True + assert "restart" in str(after.content).lower() + + asyncio.run(exercise()) diff --git a/tests/test_search_execution.py b/tests/test_search_execution.py index 6d1a2b3..1fb35b1 100644 --- a/tests/test_search_execution.py +++ b/tests/test_search_execution.py @@ -7,8 +7,10 @@ from jsonschema import Draft202012Validator from referencing import Registry, Resource +import deeplaw.ingest as ingest_module +from deeplaw.evaluate import evaluate_file from deeplaw.ingest import build_release -from deeplaw.models import SearchRequest +from deeplaw.models import ExtractionQuality, ExtractionResult, SearchRequest, TextBlock from deeplaw.search import DeepLaw from .helpers import manifest_document, write_docx, write_manifest @@ -185,6 +187,99 @@ def test_as_of_separates_missing_or_unverified_temporal_metadata( _validate_search_response(response.to_dict()) +def test_extraction_review_required_is_separated_from_primary_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "source" + document = source / "review-required.docx" + safe_document = source / "safe-distractor.docx" + write_docx(document, ["抽取复核测试法", "第一条 未完成人工对照的文本不得进入主证据。"]) + write_docx( + safe_document, + ["普通测试办法", "第一条 人工对照的文本属于普通测试内容。"], + ) + manifest = write_manifest( + source / "manifest.json", + [ + manifest_document(source, document.name, title="抽取复核测试法"), + manifest_document(source, safe_document.name, title="普通测试办法"), + ], + ) + + original_extract_document = ingest_module.extract_document + + def extract_with_risk(path: Path, *args: object, **kwargs: object) -> ExtractionResult: + if path.name != document.name: + return original_extract_document(path, *args, **kwargs) + return ExtractionResult( + blocks=( + TextBlock(text="抽取复核测试法", paragraph=1), + TextBlock(text="第一条 未完成人工对照的文本不得进入主证据。", paragraph=2), + ), + quality=ExtractionQuality( + extractor="test-ocr", + extractor_version="test-ocr/v1", + block_count=2, + page_count=1, + character_count=30, + needs_ocr=True, + review_required=True, + warnings=("page 1: test review required",), + ), + ) + + monkeypatch.setattr( + ingest_module, + "extract_document", + extract_with_risk, + ) + release, _ = build_release( + source_root=source, + manifest_path=manifest, + output_root=tmp_path / "releases", + allow_needs_ocr=True, + ) + + with DeepLaw(release / "deeplaw.sqlite3") as law: + response = law.search( + SearchRequest(query="未完成人工对照的文本", limit=1) + ) + + assert not response.evidence + assert response.uncertain_evidence + assert response.uncertain_evidence[0].title == "抽取复核测试法" + assert all(card.extraction_review_required for card in response.uncertain_evidence) + assert any("抽取风险" in notice for notice in response.notices) + assert any( + item.status == "uncertain" for item in response.obligation_coverage if item.required + ) + _validate_search_response(response.to_dict()) + + cases = tmp_path / "review-required-cases.jsonl" + cases.write_text( + json.dumps( + { + "id": "review-required", + "query": "未完成人工对照的文本", + "expected_titles": ["抽取复核测试法"], + "expected_bucket": "uncertain_evidence", + "expected_extraction_review_required": True, + "max_evidence": 1, + }, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + report = evaluate_file(release / "deeplaw.sqlite3", cases, limit=1) + assert report["schema_version"] == "deeplaw.eval-report/v2" + assert report["overall_pass_rate"] == 1.0 + assert report["results"][0]["expected_bucket"] == "uncertain_evidence" + assert report["results"][0]["evidence_count"] == 0 + assert report["results"][0]["uncertain_evidence_count"] == 1 + + def test_as_of_excludes_known_future_interval_and_reports_gap(tmp_path: Path) -> None: database = _build( tmp_path, diff --git a/uv.lock b/uv.lock index 18548be..1d9991b 100644 --- a/uv.lock +++ b/uv.lock @@ -388,7 +388,7 @@ wheels = [ [[package]] name = "deeplaw" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "mcp" },