diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..b3dcfebb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Fixtures are byte-exact inputs: line endings, BOMs and legacy encodings are +# exactly what the parsers are being tested on, so git must not normalise them on +# checkout. Without this a CRLF fixture arrives as LF and its byte ranges drift. +tests/fixtures/** -text -diff +# The bundled derivation fixtures resolve anchors to raw byte ranges as well. +src/derive/fixtures/** -text -diff diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7d5565f..6fdae092 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,54 +1,96 @@ name: CI on: + # `ds/**` is not decoration: the 19 per-feature PRs are a **stack** — each targets + # the branch before it (`ds/02` → `ds/01`, … , `ds/21` → `ds/20`), and `ds/01` + # targets `pre-v2-baseline`. With this list limited to the three integration + # branches, not one of them could ever run CI: every PR sat at "no checks + # reported" while its body claimed tests. A base branch that no trigger matches is + # a gate that does not exist. push: - branches: [dot-skill, main] + branches: [dot-skill-test, dot-skill, main, 'ds/**'] pull_request: - branches: [dot-skill, main] + branches: [dot-skill-test, dot-skill, main, 'ds/**'] + # Manual re-runs without pushing anything. + workflow_dispatch: jobs: test: - name: Python ${{ matrix.python-version }} + name: Node ${{ matrix.node-version }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.11"] + node-version: ["20", "22"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: actions/setup-node@v4 with: - python-version: ${{ matrix.python-version }} - cache: pip + node-version: ${{ matrix.node-version }} - - name: Install dependencies + - name: Syntax-check every module run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + find bin src scripts tests -name '*.mjs' -print0 | xargs -0 -n1 node --check - - name: Compile all Python sources - run: python -m compileall -q tools + - name: Unit tests + # `npm test`, not a bare `node --test`: the bare form also collects + # `scripts/blind-test.mjs` (it matches `**/*-test.mjs`) and records its + # usage error as a failing test, so CI went red while every local command + # looked green. + run: npm test - - name: Run unit tests - run: | - if [ -d tests ]; then - python -m unittest discover -s tests -p 'test_*.py' -v - else - echo "No tests/ directory yet — skipping unittest discover." - fi - - lint: - name: Ruff + - name: Prompt contract lint + run: node scripts/prompt-lint.mjs + + - name: Skill template freshness + run: node scripts/generate-template.mjs --check + + # Every demand of the v2 objective, mapped to the artefact that proves it. + # Acceptance itself runs in the next job, so this is scope-only. + - name: Objective audit + run: node scripts/audit-objective.mjs --skip-acceptance + + # Release hygiene: versions, schema marker, carried directories, gates. + - name: Release check + run: node scripts/check_release.mjs + + acceptance: + name: Acceptance (public corpus) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + # The visual-check phase drives a real browser; playwright is a dev + # dependency and never ships with the package. + - name: Install Playwright + run: | + npm install --no-save playwright@1.62.1 + npx playwright install --with-deps chromium + + - name: End-to-end acceptance on the bundled corpus + run: node scripts/acceptance.mjs --evidence "$RUNNER_TEMP/evidence" + + # The second public corpus is multi-source (two chat exports + a document) + # and dated, so the same phases run on a corpus the derivation was designed for. + - name: End-to-end acceptance on the multi-source corpus + run: node scripts/acceptance.mjs --corpus tests/fixtures/public-corpus/synthetic-multisource --evidence "$RUNNER_TEMP/evidence-multisource" + + # Scope audit *with* the acceptance run it normally performs. + - name: Objective audit (includes acceptance) + run: DISTILLY_PLAYWRIGHT_ROOT="$PWD" node scripts/audit-objective.mjs + + - uses: actions/upload-artifact@v4 + if: always() with: - python-version: "3.11" - - name: Install ruff - run: pip install ruff - - name: Run ruff (non-blocking for now) - run: ruff check tools/ || true + name: acceptance-evidence + path: | + ${{ runner.temp }}/evidence + ${{ runner.temp }}/evidence-multisource + if-no-files-found: ignore + diff --git a/.gitignore b/.gitignore index 2415c029..71290c97 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,27 @@ playwright-data/ /tmp/feishu_*.txt /tmp/email_*.txt /tmp/dingtalk_*.txt + +# Every `knowledge/` is user data, wherever the person directory lives. This is an +# unanchored directory pattern, so it also matches source and fixture directories of +# the same name; the re-includes below must come *after* it, otherwise the later +# `knowledge/` rule excludes them again. Tracked files are unaffected either way, +# so only a newly added module would be lost — and it would be lost silently. knowledge/ +# ...except the synthetic ledger fixtures, which are test data and must be tracked +!src/derive/fixtures/**/knowledge/ +!src/derive/fixtures/**/knowledge/** +# ...and except the source modules: `src/knowledge/**` is code, not an export +!src/knowledge/ +!src/knowledge/** # OS .DS_Store Thumbs.db + +# 本地证据（截图/回执/diff），不入库 +dst-evidence/ +# 根级渲染产物：契约 §2 把页面放在 evidence/renders/。 +# 前导斜杠是必须的 —— 裸写 evidence/ 会匹配 docs/evidence/，那里的文字证据是要入库的。 +/evidence/ +*.evidence.local diff --git a/INSTALL.md b/INSTALL.md index f051a40c..09e3b034 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -25,8 +25,44 @@ --- + + +## v2：命令入口与宿主适配 + +v2 只有一个命令入口：**`bin/distilly.mjs`**（Node ≥ 20，零依赖，见 `docs/v2/CONTRACT.md`）。 + +```bash +node bin/distilly.mjs install # 装到该宿主的全局 Skill 目录 +node bin/distilly.mjs install --force # 已有安装先备份成 *.backup-<时间戳> 再替换 +node bin/distilly.mjs install --path

# 装到自定义路径（末段目录必须叫 distilly） +node bin/distilly.mjs --help +``` + +- **宿主 id、全局/项目级目录、确切安装命令、双语注意事项、装完怎么验证**：见 + **[docs/v2/HOSTS.md](docs/v2/HOSTS.md)**。该表由 `src/hosts/agents.mjs` 生成，`tests/agents.test.mjs` + 强制它与 `bin/distilly.mjs` 的落盘目录一致。 +- 当前支持 8 个宿主：`claude-code` · `codex` · `opencode` · `openclaw` · `hermes` · `deepseek-harness` · + `grok-build` · `pi`。别名：`claude`、`deepseek`、`grok`。 +- 两条路线等价：`npx -y skills add titanwings/distilly --skill distilly …`（AgentSkills CLI）或直接 + `git clone https://github.com/titanwings/distilly <目标目录>`；逐字命令同样在 `docs/v2/HOSTS.md`。 + + + +### ⚠️ Deprecated：`python3 tools/*.py` 安装器 + +下面「选择你的平台」各节里的 `python3 tools/install_*_skill.py` 与手工 `git clone` 是**迁移期兼容路径，已废弃**： +v2 不再要求用户手动跑 Python。契约（`docs/v2/CONTRACT.md` §1）约定旧的 `python3 tools/xxx.py` 调用由 +`bin/distilly.mjs` 转发并打印 deprecation 警告，转发层在 PR③ 删除；在当前集成分支上这些命令仍然等价于 +直接执行对应的 Python 脚本。旧内容只为排查老安装而保留，**新安装请走 `bin/distilly.mjs` 或 +`docs/v2/HOSTS.md` 里的一行命令**。 + +--- + ## 选择你的平台 +> ⚠️ **Deprecated（旧安装路径）**：本节保留旧版按平台展开的说明。宿主目录与确切命令的最新版本在 +> **[docs/v2/HOSTS.md](docs/v2/HOSTS.md)**；下面的 `python3 tools/*.py` 调用见上一节的废弃说明。 + ### A. Claude Code（推荐） 本项目遵循官方 [AgentSkills](https://agentskills.io) 标准，整个 repo 就是 skill 目录。克隆到 Claude skills 目录即可： @@ -533,3 +569,4 @@ distilly/ ← clone 到宿主的 skills/distilly/（例如 .claude ├── versions/ # 历史版本 └── knowledge/ # 原始材料归档 ``` + diff --git a/INSTALL_EN.md b/INSTALL_EN.md index b916f542..fb91f756 100644 --- a/INSTALL_EN.md +++ b/INSTALL_EN.md @@ -3,8 +3,53 @@ > Distilly was formerly known as **Colleague Skill / colleague-skill**. The creator > name and canonical install directory are now `distilly`. + + +## v2: command entrypoint and host adaptation + +v2 has exactly one command entrypoint: **`bin/distilly.mjs`** (Node >= 20, zero +dependencies, see `docs/v2/CONTRACT.md`). + +```bash +node bin/distilly.mjs install # install into that host's global Skill directory +node bin/distilly.mjs install --force # back up an existing install as *.backup-, then replace +node bin/distilly.mjs install --path

# install into a custom path (final directory must be `distilly`) +node bin/distilly.mjs --help +``` + +- **Host ids, global/project directories, the exact install commands, bilingual + notes and how to verify an install** live in + **[docs/v2/HOSTS.md](docs/v2/HOSTS.md)**. That table is generated from + `src/hosts/agents.mjs`, and `tests/agents.test.mjs` forces it to agree with the + destinations in `bin/distilly.mjs`. +- Eight hosts are supported today: `claude-code`, `codex`, `opencode`, + `openclaw`, `hermes`, `deepseek-harness`, `grok-build`, `pi`. Aliases: + `claude`, `deepseek`, `grok`. +- The two routes are equivalent — `npx -y skills add titanwings/distilly + --skill distilly …` (AgentSkills CLI) or a plain + `git clone https://github.com/titanwings/distilly `; both are quoted + verbatim in `docs/v2/HOSTS.md`. + + + +### ⚠️ Deprecated: the `python3 tools/*.py` installers + +The `python3 tools/install_*_skill.py` calls and manual clones below are +**migration-era compatibility paths and are deprecated**: v2 no longer asks users +to run Python by hand. The contract (`docs/v2/CONTRACT.md` §1) says the entrypoint +forwards the old `python3 tools/xxx.py` calls with a deprecation warning and that +the forwarding layer is removed in PR③; on the current integration branch those +commands are still equivalent to running the Python script directly. The old +sections are kept for troubleshooting legacy installs only — **use +`bin/distilly.mjs` or the one-liners in `docs/v2/HOSTS.md` for new installs**. + ## Install Distilly +> ⚠️ **Deprecated (legacy install path)**: this section keeps the old per-host +> walkthrough. The current host directories and exact commands are in +> **[docs/v2/HOSTS.md](docs/v2/HOSTS.md)**; the `python3 tools/*.py` calls are +> explained in the deprecation note above. + Clone the repository into a Skills directory discovered by your host, keeping the destination directory name `distilly`: @@ -157,3 +202,4 @@ keep only copyright-safe paraphrases with source URLs in research notes, and delete the temporary file after review. Xquik is independent of X Corp. “Twitter” and “X” are trademarks of X Corp. + diff --git a/README.md b/README.md index b529f9c3..64333e04 100644 --- a/README.md +++ b/README.md @@ -4,336 +4,188 @@
-# 🧬 Distilly +# Distilly -**Formerly: Colleague Skill / colleague-skill.** +### Distill how they think into Person Profiles for Agents. -### Distill a person's experience, judgment, voice, and ways of working into a reusable Person Profile for AI agents and compatible bots. - -**Messages · documents · interviews · public sources → Distilly → Person Profile → Agent / Bot** +**Colleague Skill / colleague-skill (original name)** [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) -[![Python 3.9+](https://img.shields.io/badge/Python-3.9%2B-blue.svg)](https://python.org) -[![AgentSkills](https://img.shields.io/badge/AgentSkills-Standard-green)](https://agentskills.io) -[![Stars](https://img.shields.io/github/stars/titanwings/colleague-skill?style=social)](https://github.com/titanwings/distilly/stargazers) - -[![Discord](https://img.shields.io/badge/Discord-Join%20Community-5865F2?logo=discord&logoColor=white)](https://discord.gg/NVX66RxWZv) - -
- - - -
- -🧑‍💼  Your colleague quit, your mentor graduated, your teammate transferred — taking their whole playbook and context with them?
-💞  Your family, old friends, partner drifting apart — and you want to hold on to the way it felt to be with them?
-🌟  Your favorite author, idol, thinker you'll never meet — but you want to know what they'd say about your question? - -
- -### ✨ One project, many kinds of people. - -
- -Distilly is the person-modeling layer for agents. It turns the materials you provide into a portable, source-grounded Person Profile built from observable experience, decision patterns, expression, and ways of working; it does not claim to clone the person behind them. - -Colleagues · partners · family · old friends · idols · public figures · fictional characters — even yourself - -**Source material + your description → a source-grounded Person Profile → your Agent or compatible Bot** - -> A Person Profile is the reusable output. The current release packages each profile as an Agent Skill so supported hosts can install and invoke it. The canonical creator Skill is named `distilly`; install it in a `distilly` directory. The former name above remains for search continuity and project history. - -
- -[🆕 What Distilly does](#-what-distilly-does-today) · [📦 Data Sources](#-supported-data-sources) · [⚡ Install](#-install) · [🚀 Usage](#-usage) · [✨ Demo](#-demo) · [📝 Citation](#-citation) · [💬 Discord](https://discord.gg/NVX66RxWZv) - -[**Chinese**](docs/lang/README_ZH.md) · [**Spanish**](docs/lang/README_ES.md) · [**German**](docs/lang/README_DE.md) · [**Japanese**](docs/lang/README_JA.md) · [**Russian**](docs/lang/README_RU.md) · [**Portuguese**](docs/lang/README_PT.md) · [**Korean**](docs/lang/README_KO.md) - - - ---- - -

- -### 🎉 2026.08.13 Milestone — **the project has passed 20K ⭐!** - -Massive thanks to everyone who starred — we'll keep shipping, keep distilling. - -
- -> 🧬 **2026.08.24 Update** — The creator is now named **Distilly** end to end and documents native local Skill discovery for Claude Code, Hermes, OpenClaw, Codex, DeepSeek Harness, Pi, Grok Build, and OpenCode. Grok Bot is listed separately as a saved-Skill workflow preview. - -> 📝 **2026.06.01 Update** — **[The COLLEAGUE.SKILL technical report](https://arxiv.org/pdf/2605.31264) is now available**. The most rewarding part was not simply publishing a paper, but seeing the community grow the gallery to 215 skills contributed by 165 people, with more than 100,000 stars across the skill cards. The paper's Acknowledgements explicitly recognize every community contributor. - -> 🗺️ **2026.04.13** — **The Distilly Roadmap is live!** What began as Colleague Skill is growing beyond colleagues: distill people into Skills that Agents can reuse. 👉 **[Full Roadmap](ROADMAP.md)** · **[💬 Discord](https://discord.gg/NVX66RxWZv)** - -> 🌐 **2026.04.07** — Community gallery is live! Any skill / meta-skill can drive traffic directly to your own GitHub repo. No middleman. 👉 **[titanwings.github.io/colleague-skill-site](https://titanwings.github.io/colleague-skill-site/)** - -
- -Created by [@titanwings](https://github.com/titanwings) +[![Node.js](https://img.shields.io/badge/Node.js-22.19%2B-339933.svg)](https://nodejs.org/) +[![Codex Preview](https://img.shields.io/badge/Codex-Developer%20Preview-black)](https://github.com/titanwings/distilly/tree/distilly-plugin)
---- - -## 🆕 What Distilly does today - -### 1️⃣ From Colleague Skill to Distilly - -The project is no longer limited to the colleague scenario. Its `distilly` creator builds source-grounded Person Profiles for three person families with one workflow, then packages each profile as an Agent Skill. - -### 2️⃣ Three character families - - - - - - - - - - - - - - - - - - - - - -
🧑‍💼 colleague💞 relationship🌟 celebrity
Coworkers · mentors · teammates · up/downstream partnersExes · partners · parents · friends · close familyPublic figures · creators · public voices · fictional characters
Builds a Work Skill + Persona from material-derived technical standards, workflows, expression, and workplace behavior. Supports Lark / DingTalk / Slack collection.Organizes material-derived expression patterns, emotional triggers, conflict patterns, and repair patterns into a reusable Persona Skill.Ships with a six-dimension research toolchain (subtitles → transcript cleanup → research merge → quality check) for organizing observable decisions, expression, and mental models.
- -Each family has its own source-collection strategy, analysis dimensions, and Person Profile structure. - -### 3️⃣ More Agent hosts - -The old version only ran in Claude Code. Distilly now supports native local Skill discovery across eight agent hosts. - - - - - - - - - - - - - - -
Claude CodeHermes AgentOpenClawCodex
DeepSeek HarnessPi coding agentGrok BuildOpenCode
- -**Grok Bot preview:** Grok Bot supports saved/private Skills, but its official docs do not describe direct local `SKILL.md` imports. Distilly's workflow can be migrated manually into a saved Skill; direct repo installation is not yet verified. - -Each generated Person Profile is packaged as an Agent Skill and can be installed into any supported host. +Distilly is a local-first product for turning a person's source material, working habits, judgment, and voice into a versioned **Person Profile for Agents**. The profile can be recalled temporarily during a run or explicitly installed as a long-lived host Skill. The storage authority stays local; no additional model API key is required. ---- +## 这一支（`dot-skill-test`）：人物 Skill + 证据脊柱 -## 📦 Supported Data Sources +**把一个人的原材料蒸馏成一个可调用的人物 Skill，外加一份每条结论都能回指到原文的画像页。** +零运行时依赖，只要 Node ≥ 20。 -| Logo | Source | Messages | Docs / Wiki | Notes | -|:----:|--------|:--------:|:-----------:|-------| -| Lark | Lark (auto) | ✅ API | ✅ | Just enter a name, fully automatic | -| DingTalk | DingTalk (auto) | ⚠️ Browser | ✅ | DingTalk API doesn't support message history | -| Slack | Slack (auto) | ✅ API | — | Requires admin to install Bot; free plan limited to 90 days | -| X | Public X posts | ✅ API | — | Optional, bounded celebrity research candidates through metered third-party service Xquik | -| WeChat | WeChat chat history | ✅ SQLite | — | Export first with WeChatMsg or PyWxDump | -| 📄 | PDF / Images / Screenshots | — | ✅ | Manual upload | -| Lark | Lark JSON export | ✅ | ✅ | Manual upload | -| ✉️ | Email `.eml` / `.mbox` | ✅ | — | Manual upload | -| 📝 | Markdown / direct paste | ✅ | ✅ | Manual input | - ---- +| 交付物 | 路径 | +| --- | --- | +| 人物 Skill（可直接装进宿主运行） | `skills///SKILL.md` + `work.md` `persona.md` `work_skill.md` `persona_skill.md` `manifest.json` `meta.json` | +| 画像页（单文件、离线、双主题） | `views/.html` + `evidence/renders/receipt.json` | -## ⚡ Install +三个 family：`colleague` / `relationship` / `celebrity`。 +八个宿主：Claude Code · Codex · opencode · OpenClaw · Hermes · **DeepSeek Harness** · Grok Build · Pi。 -### 🤖 For Agents +**这份实现与那条 Plugin 路线的区别，一句话**：它不追求"像不像"，它保证"凭什么这么说"—— +每条结论都能回指到原文的字节区间（`[k00NN]` 锚点），派生可复跑（同一输入两次字节相同）， +交付物由机械门禁压住（验收 17 项，含"产物齐 / Layer 0–5 齐 / 悬空锚点=0"）。 -Open any supported local Agent host and send: +### 装到一个宿主（以 DeepSeek Harness 为例） -> Install Distilly from `https://github.com/titanwings/distilly`, then verify that this host can discover it. - -The Agent installs Distilly as a Skill named `distilly` in the correct host directory. +```bash +git clone https://github.com/titanwings/distilly.git && cd distilly +git checkout dot-skill-test +node bin/distilly.mjs install deepseek-harness # → $DSH_HOME/skills/distilly +``` -### 👤 For Humans +装完即可被 DSH 发现（技能目录被 watch，无需重启），输入 `/distilly` 或直接让 Agent 开始蒸馏。 +其它宿主把 `deepseek-harness` 换成 `claude-code` / `codex` / `opencode` / `openclaw` / `hermes` / +`grok-build` / `pi`；每条宿主的确切路径与命令见 [`docs/v2/HOSTS.md`](docs/v2/HOSTS.md)。 -Clone Distilly into the Skills directory used by your host: +### 怎么验 ```bash -git clone https://github.com/titanwings/distilly +npm test # 391 项，Node 20 与 22 各一遍 +node scripts/acceptance.mjs --corpus tests/fixtures/public-corpus/synthetic-interview --person lin-gong # 17/17 +DISTILLY_PLAYWRIGHT_ROOT=<含 node_modules 的目录> node scripts/audit-objective.mjs # 15/15 ``` -Host paths, migration, Windows, generated-profile installation, and credential setup are in the **[Install Guide](INSTALL_EN.md)**. +**当前状态、已知缺口、分支与 PR 清单**：[`docs/v2/STATUS.md`](docs/v2/STATUS.md)。 +契约 [`docs/v2/CONTRACT.md`](docs/v2/CONTRACT.md) · 验收 [`docs/v2/ACCEPTANCE.md`](docs/v2/ACCEPTANCE.md)。 --- -## 🚀 Usage - -In your Agent, say: - -> Use Distilly to create a Person Profile for ``. +## 另一条产品线：`distilly-plugin`（Plugin Developer Preview） -Then: +> **下面这一节以及其后的「Install the Developer Preview / Host compatibility / The first usable +> flow / Host status / Local material formats」各节，描述的不是本分支的代码**，而是 +> `distilly-plugin` 分支上的 Plugin（MCP / Panel / SQLite）路线。本分支不构建、也不运行它们； +> 保留在这里只是不让那条线的信息丢失。 -1. Choose `colleague`, `relationship`, or `celebrity`. -2. Add a description or source material. Every field is optional. -3. Review the result and let Distilly generate the profile. +This `distilly-plugin` branch carries the unreleased `0.1.0-preview.1` Developer Preview; the repository's default branch is `dot-skill`, the separate legacy implementation, so a bare clone lands on that line instead of this one. Codex, OpenClaw `2026.3.24`, and Hermes `v0.9.0` each have an immutable real-host transport-capacity fixture. The OpenClaw and Hermes measurements use a deterministic synthetic fixture server through the real host executable, model, and MCP transport; they do not by themselves certify packaged restart or the full product lifecycle. Setup remains fail-closed for any unrecorded host version or changed release tuple. This branch is not a tagged release or an npm package yet. -The result is packaged as an Agent Skill named `{character}-{slug}`. For the optional celebrity research pipeline and advanced tools, see the **[Install Guide](INSTALL_EN.md)**. +[Chinese](docs/lang/README_ZH.md) · [Español](docs/lang/README_ES.md) · [Deutsch](docs/lang/README_DE.md) · [日本語](docs/lang/README_JA.md) · [한국어](docs/lang/README_KO.md) · [Português](docs/lang/README_PT.md) · [Русский](docs/lang/README_RU.md) ---- - -## ✨ Demo - -One from each family. - -
+## Install the Developer Preview -### 🌟 celebrity — distilling Andrej Karpathy +### For an agent -> Six-dimension research: works · interviews · decisions · expression DNA · external evaluations · timeline
-> Case adapted from community work [alchaincyf/karpathy-skill](https://github.com/alchaincyf/karpathy-skill) +Give your coding agent the following task and let it run the commands in a fresh checkout: -``` -User ❯ Everyone's talking about Agents now — what do you think the 2026 AI Agent looks like? - -celebrity.skill ❯ Everyone's doing demos, nobody's doing eval. - The Agent bottleneck was never model intelligence — it's scaffolding: - a strict evaluation harness, fine-grained error replay, - controllable tool-calling budgets, clean failure-mode analysis. - Most products are still stuck at "wrap an API + a few tools + a system prompt." - That's a demo, not an agent. - In 2026 I'm not betting on the strongest model — I'm betting on the best - scaffolding and the strictest eval. -``` +> Install the Distilly Developer Preview from the `distilly-plugin` branch, build it with Node 22.19+ (or Node 24), run `distilly setup --host codex`, run `distilly doctor --host codex`, and report the result. Do not modify another branch. -
+The exact checkout and setup commands are shown below so the agent can verify every step. -### 🧑‍💼 colleague — distilling a ByteDance L2-1 backend +### For a human -> Input: `ByteDance L2-1 backend engineer, INTJ, blame-shifter, ByteDance-style` +Requirements: Node.js `22.19+` or `24`, pnpm `10.32+`, and a locally installed Codex CLI. From a terminal: +```bash +git clone --branch distilly-plugin https://github.com/titanwings/distilly.git +cd distilly +corepack enable +pnpm install --frozen-lockfile +pnpm run build +node packages/cli/lib/bin.js setup --host codex +node packages/cli/lib/bin.js doctor --host codex ``` -User ❯ Can you review this API design? - -colleague.skill ❯ Hold on — what's the impact? You haven't explained the context. - (after reading) N+1 query, fix it. Use the standard - {code, message, data} response format. That's the spec, - don't ask why. -User ❯ This bug was introduced by you, right? +Restart Codex after setup. The launcher registers the self-contained Plugin and its five MCP tools. To remove the host integration while keeping all local people, profiles, and source data: -colleague.skill ❯ Does the timeline match? That feature touched multiple places, - there were other changes too. +```bash +node packages/cli/lib/bin.js uninstall --host codex ``` -
- -### 💞 relationship — distilling someone you have a crush on +To install one approved profile as a persistent Skill after a profile has been created, use its exact subject id: -> Upload half a year of chat logs + "sensitive, quiet but stubborn, will actually reply seriously when it matters" - -``` -User ❯ Did you think about me today? - -relationship.skill ❯ ...I did, a little bit. Why are you asking? +```bash +node packages/cli/lib/bin.js install subject_<32 lowercase hex characters> --host codex ``` -
- -📚 More real-world cases in the **[community gallery](https://titanwings.github.io/colleague-skill-site/)** — 100+ skills and counting - -
- ---- +## Host compatibility and explicit Legacy fallback -## 🔧 Features +Codex uses the native Plugin preview above. The Preview also includes compatibility bindings for OpenClaw and Hermes: -### 🧱 Generated Skill Structure +- **OpenClaw** loads the Claude-compatible bundle from `~/.openclaw/extensions/distilly` and its real `.mcp.json`. Check discovery with `openclaw plugins inspect distilly --json`. +- **Hermes** installs the canonical Skill at `~/.hermes/skills/distilly`, a managed wrapper at `~/.distilly/bin/distilly-hermes`, and an MCP entry in `~/.hermes/config.yaml`. `resources` and `prompts` are disabled so the exposed surface remains five tools; check it with `hermes mcp test distilly`. -Distilly's current creator uses **Persona** as the universal base, with family-specific modules layered on top: +The CLI recognizes both hosts and enables setup when the installed version matches the recorded real-host transport fixture. The current net budgets, measured in isolated clean sessions with `openai-codex/gpt-5.4`, are 65,536 serialized bytes for OpenClaw and 49,752 for Hermes (the same conservative byte/token accounting used by the Codex fixture). These are transport/value lower bounds for the recorded probe, not a guarantee of remaining context in every model or user session. Any unrecorded version, release digest, tool descriptor, or serializer tuple returns `host_unsupported` before writing an unverified integration. There is no automatic switch to the legacy implementation. -| Family | Persona Content | Additional Modules | -|--------|-----------------|-------------------| -| 🧑‍💼 **colleague** | 6-layer personality: hard rules → identity → expression → decisions → interpersonal → Correction | ➕ **Work Skill**: scope, workflow, output preferences, experience knowledge base | -| 💞 **relationship** | Expression DNA · emotional triggers · conflict pattern · repair pattern | — | -| 🌟 **celebrity** | Mental models · decision heuristics · expression DNA · external-evaluation contrast | ➕ Six-dimension research dossier (works / interviews / decisions / timeline...) | +Until a host has a verified Plugin binding, you can explicitly choose the maintained `dot-skill` branch as a **Legacy Skill compatibility mode**: -> **Execution**: Receive task → Persona selects material-derived preferences and tone → Additional modules fill in execution detail → Produce a source-grounded response +> Install Distilly in Legacy Skill compatibility mode from the `dot-skill` branch into this host's normal Skills directory, using a clean checkout whose final directory is named `distilly`. Verify discovery and report the installed Git commit. Do not run Plugin setup or claim SQLite, five-tool MCP, Panel, or Plugin lifecycle support. -### 🧬 Evolution +For a manual install, replace `` with the complete final path in the [detailed install guide](INSTALL.md), including the last `distilly` component, and create its parent first: -- 📥 **Append files** → auto-analyze delta → merge into relevant sections, never overwrite existing conclusions -- 💬 **Conversation correction** → say "they wouldn't do that, they'd be xxx" → writes to the Correction layer, takes effect immediately -- 🕰️ **Version control** → auto-archive on every update, rollback to any previous version -- 🔬 **Celebrity research pipeline** → subtitles → transcript cleanup → six-dimension research → quality check - ---- +```bash +git clone --single-branch --branch dot-skill --depth 1 \ + https://github.com/titanwings/distilly.git \ + +git -C rev-parse HEAD +``` -## ⚠️ Notes +This is an explicit, separate file-based implementation—not an automatic runtime fallback. It does not share a supported data model with the Plugin, and a failed Plugin preflight never switches modes. The compatibility promise currently covers local files and pasted text only. Do not enable legacy collectors while the Plugin uses the same home directory: current legacy collectors can write credential configuration into the same `~/.distilly/` namespace and remain outside the Preview's reviewed security boundary. Keep exactly one `distilly` active in any host discovery scope and verify which copy the host loaded. -**Source material quality = Person Profile quality** — and quality sources differ across families: +## The first usable flow -| Family | Source priority (high → low) | -|--------|------------------------------| -| 🧑‍💼 **colleague** | Their **own long-form writing** (design docs / review comments) **›** **decision-making replies** **›** casual group chat | -| 💞 **relationship** | Complete chat history **›** letters / social posts / diaries **›** third-party descriptions | -| 🌟 **celebrity** | First-person books / blogs / long interviews **›** decision records (launches, commits, Q&A) **›** verified first-person short-form posts **›** third-party commentary | +On Codex, the complete flow below is verified. OpenClaw `2026.3.24` and Hermes `v0.9.0` have the same briefing transport path verified against their recorded capacity fixture; their packaged restart, long-lived Skill, and uninstall lifecycle checks remain separate. Restart the selected host and ask it to research and distill a person. Supply only the files, text, or public URLs you want included. Distilly then: -- **colleague** Lark-compatible auto-collection: requires adding the App bot to relevant group chats -- **relationship**: longer time spans are better; material covering both conflict and repair is ideal -- **celebrity**: avoid feeding only second-hand interpretations -- This is still a demo version — please file issues if you find bugs! +1. resolves or creates the person; +2. imports the selected material with deterministic local parsers; +3. creates a pending research job and a complete evidence-bound briefing; +4. commits a versioned Person Profile; +5. returns the profile or a complete temporary prompt for the current run; +6. accepts an explicit correction and sends a candidate to review; +7. lets you promote, reject, or roll back the candidate in the local Panel; and +8. installs the approved profile as a self-contained host Skill when you ask it to. ---- +The model-facing surface remains exactly five MCP tools: -## 📄 Technical Report +`distilly_get` · `distilly_ingest` · `distilly_pending` · `distilly_commit` · `distilly_correct` -> **[COLLEAGUE.SKILL: Automated AI Skill Generation via Expert Knowledge Distillation](https://arxiv.org/pdf/2605.31264)** ([arXiv](https://arxiv.org/abs/2605.31264) · [arXiv PDF](https://arxiv.org/pdf/2605.31264)) -> -> This is the paper for **COLLEAGUE.SKILL / colleague-skill**, Distilly's predecessor. It covers the Work Skill + Persona two-layer architecture, multi-source data collection, and Skill generation mechanics — the theoretical foundation for today's `colleague` family. Separate papers on the relationship / celebrity family extensions are planned. +Distilly never silently truncates a complete briefing or profile prompt. If a verified host budget cannot carry the complete value, it reports a bounded capacity error with measurements and keeps the stored data unchanged. ---- +## Host status -## 📝 Citation +| Host | Native Plugin | Current compatibility route | +| --- | --- | --- | +| Codex | Fully verified in this release branch | Native Plugin | +| Claude Code | Binding included; exact host fixture still needed | Explicit `dot-skill` Legacy Skill | +| OpenClaw | Transport-capacity fixture recorded for `2026.3.24` (65,536-byte net budget); lifecycle pending | Claude-compatible bundle + discovery smoke | +| Hermes | Transport-capacity fixture recorded for `v0.9.0` (49,752-byte net budget); lifecycle pending | Managed Skill + MCP configuration | +| DeepSeek Harness (DSH) | Community binding planned | Explicit `dot-skill` Legacy Skill | +| Pi agent | Community binding planned | Explicit `dot-skill` Legacy Skill | +| Grok Build | Community binding planned | Explicit `dot-skill` Legacy Skill | +| OpenCode | Community binding planned | Explicit `dot-skill` Legacy Skill | +| Grok Bot | Community binding planned | Manual saved/private Skill only; local repository import is not claimed | -If you use **Distilly** or **COLLEAGUE.SKILL** in your research or applications, please cite the technical report: +Host compatibility is a binding concern. Legacy Skill discovery is useful continuity, but it does not make a host a verified Plugin target. -```bibtex -@misc{zhou2026colleagueskill, - title = {COLLEAGUE.SKILL: Automated AI Skill Generation via Expert Knowledge Distillation}, - author = {Tianyi Zhou and Dongrui Liu and Leitao Yuan and Jing Shao and Xia Hu}, - year = {2026}, - eprint = {2605.31264}, - archivePrefix = {arXiv}, - primaryClass = {cs.AI}, - url = {https://arxiv.org/abs/2605.31264} -} -``` +## Local material formats -You can also use the machine-readable citation metadata in [CITATION.cff](CITATION.cff). +The first Preview accepts explicit local `TXT`, `Markdown`, `JSON`, and `SRT/VTT` files. It also accepts pasted text and public URLs through the host's visible research flow. Files are read only from the paths or sources the user supplies; symlinked selected files and duplicate file names are rejected. PDF, email, provider exports, and hosted connectors are follow-up work. ---- +## 📣 2026-09 update: help expand coding-agent Plugins -## ⭐ Star History +Codex, OpenClaw, and Hermes now have real host/version capacity fixtures. We need community support to provide the same evidence for **Claude Code, DeepSeek Harness (DSH), Pi agent, Grok Build, OpenCode, and Grok Bot**, then to build and validate their coding-agent Plugin packages. I will actively review those contributions and keep the public contracts, release digests, and host behavior aligned. - - - - - Star History Chart - - +See the full call for contributors in [UPDATES.md](UPDATES.md) and the current priorities in [ROADMAP.md](ROADMAP.md). ---- +## Project documents -
+- [Detailed Preview installation](INSTALL.md) +- [Changelog](CHANGELOG.md) +- [Architecture and shipped-state map](docs/architecture.md) +- [Testing contract](docs/testing.md) +- [Development workflow](docs/development.md) +- [Design corpus](docs/design/README.md) +- [Release manifest](plugins/release-manifest.json) +- [Contributing](CONTRIBUTING.md) -**MIT License** © [titanwings](https://github.com/titanwings) +Distilly is released under the [MIT License](LICENSE). Created by [@titanwings](https://github.com/titanwings). -
diff --git a/SKILL.md b/SKILL.md index 7f9fbcc9..462b1d70 100644 --- a/SKILL.md +++ b/SKILL.md @@ -11,11 +11,21 @@ allowed-tools: Read, Write, Edit, Bash > > 本 Skill 支持中英文。根据用户第一条消息的语言，全程使用同一语言回复。下方提供了两种语言的指令，按用户语言选择对应版本执行。 -> **Skill Root / Skill 根目录**: Before reading a bundled prompt or running a bundled script, resolve the absolute directory of the `SKILL.md` that the host actually loaded. In the instructions below, `{distilly_skill_root}` means that exact directory. Claude Code exposes it as `${CLAUDE_SKILL_DIR}`; on every other host, use the loaded-skill path supplied by that host's discovery context. Do not assume the shell's current working directory is the Skill root, and do not guess or hard-code an install path. If the host does not expose the loaded path or more than one Distilly installation is ambiguous, ask the user to identify the active installation before running code. +> **Skill Root / Skill 根目录**: Before reading a bundled prompt or running a bundled command, resolve the absolute directory of the `SKILL.md` that the host actually loaded. In the instructions below, `{distilly_skill_root}` means that exact directory. Claude Code exposes it as `${CLAUDE_SKILL_DIR}`; on every other host, use the loaded-skill path supplied by that host's discovery context. Do not assume the shell's current working directory is the Skill root, and do not guess or hard-code an install path. If the host does not expose the loaded path or more than one Distilly installation is ambiguous, ask the user to identify the active installation before running code. > -> Keep the shell in the user's current workspace so relative output paths such as `./skills/...` remain project-local. Resolve every `tools/...` and `prompts/...` resource against `{distilly_skill_root}`. For example, execute the bundled `tools/example.py` as `python3 "{distilly_skill_root}/tools/example.py"`; replace the placeholder with the resolved absolute path in the actual tool call. +> Keep the shell in the user's current workspace so relative output paths such as `./skills/...` remain project-local. Resolve every `prompts/...` resource against `{distilly_skill_root}`. The only supported entrypoint is the `distilly` CLI; do not call the bundled Python tools directly (they are deprecated, see the migration table below). > -> 在读取内置 prompt 或运行脚本前，先取得宿主实际加载的这份 `SKILL.md` 所在绝对目录；下文以 `{distilly_skill_root}` 表示。Claude Code 可用 `${CLAUDE_SKILL_DIR}`，其他宿主使用其 Skill discovery 上下文提供的实际路径。不要假定 shell 当前目录就是 Skill 目录，也不要猜测或硬编码安装路径。shell 应继续停留在用户工作区，使 `./skills/...` 等输出仍写入当前项目；所有 `tools/...`、`prompts/...` 都必须从 `{distilly_skill_root}` 解析。 +> **How to run that CLI**: there is no global `distilly` on `PATH` — the CLI *is* the file inside this Skill. Invoke it as +> `node "{distilly_skill_root}/bin/distilly.mjs" […]` (it is also executable, so `"{distilly_skill_root}/bin/distilly.mjs" ` works once the file mode survives the copy). Every command in the tables below is written `distilly ` as shorthand for that. If `node` is missing, say so and stop rather than reimplementing a step by hand. +> +> **Where things are written**: `--base-dir ` means the workspace root (the directory holding `skills/`) in *every* command — `harvest`, `retrospect`, `view`, `doctor`, `skill`. When a command needs the level that directly contains `/` instead, that is `--skills-dir ` (`skill …`) or `--dir ` (`retrospect`). Never pass both `--base-dir` and `--skills-dir`; the CLI rejects it rather than guessing. +> +> 在读取内置 prompt 或执行内置命令前，先取得宿主实际加载的这份 `SKILL.md` 所在绝对目录；下文以 `{distilly_skill_root}` 表示。Claude Code 可用 `${CLAUDE_SKILL_DIR}`，其他宿主使用其 Skill discovery 上下文提供的实际路径。不要假定 shell 当前目录就是 Skill 目录，也不要猜测或硬编码安装路径。shell 应继续停留在用户工作区，使 `./skills/...` 等输出仍写入当前项目；所有 `prompts/...` 都必须从 `{distilly_skill_root}` 解析。唯一受支持的入口是 `distilly` CLI，不要直接调用仓库里的 Python 工具（它们已废弃，见下方迁移对照表）。 +> +> **这个 CLI 怎么调**：`PATH` 上没有全局 `distilly`——CLI 就是本 Skill 目录里的那个文件。写成 +> `node "{distilly_skill_root}/bin/distilly.mjs" <子命令> […]`（该文件也是可执行的，拷贝时若保留了执行位，`"{distilly_skill_root}/bin/distilly.mjs" <子命令>` 也可以）。下文所有表格里的 `distilly <子命令>` 都是它的简写。没有 `node` 就如实说明并停下，不要手工重做某一步。 +> +> **东西写到哪**：`--base-dir <工作区>` 在**每一条**命令里都表示工作区根（下面有 `skills/`）——`harvest`、`retrospect`、`view`、`doctor`、`skill` 一致。需要"直接存放 `/` 的那一层"时用 `--skills-dir `（`skill …`）或 `--dir <人物目录>`（`retrospect`）。两个同时给会被拒绝，而不是猜一个。 # Distilly 创建器 @@ -52,613 +62,233 @@ Grok Bot 可以把流程保存为 private Skill，但目前没有官方的本地 兼容更新别名： - `/update-colleague {slug}` -当用户要求查看已生成的 Skill 时，执行下方“管理操作”里的列出命令。 +当用户要求查看已生成的 Skill 时，执行下方"管理操作"里的列出命令。 --- -## 工具使用规则 +## 命令契约（唯一入口） -本 Skill 运行在任意兼容宿主中，只要求宿主能够读取本地文件并执行 Bash / Python 命令。使用以下工具约定： +所有采集、派生、渲染都走 `distilly`。命令名与 `docs/v2/CONTRACT.md` §1 的命令表逐字一致；不要发明子命令或字段。 -| 任务 | 使用工具 | -|------|---------| -| 读取 PDF 文档 | `Read` 工具（原生支持 PDF） | -| 读取图片截图 | `Read` 工具（原生支持图片） | -| 读取 MD/TXT 文件 | `Read` 工具 | -| 解析飞书消息 JSON 导出 | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_parser.py"` | -| 飞书全自动采集（推荐） | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_auto_collector.py"` | -| 飞书文档（浏览器登录态） | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_browser.py"` | -| 飞书文档（MCP App Token） | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_mcp_client.py"` | -| 钉钉全自动采集 | `Bash` → `python3 "{distilly_skill_root}/tools/dingtalk_auto_collector.py"` | -| 采集公开 X 帖子候选证据 | `Bash` → `python3 "{distilly_skill_root}/tools/research/xquik_public_posts.py"` | -| 解析邮件 .eml/.mbox | `Bash` → `python3 "{distilly_skill_root}/tools/email_parser.py"` | -| 写入/更新 Skill 文件 | `Write` / `Edit` 工具 | -| 版本管理 | `Bash` → `python3 "{distilly_skill_root}/tools/version_manager.py"` | -| 列出已有 Skill | `Bash` → `python3 "{distilly_skill_root}/tools/skill_writer.py" --action list` | - -**基础目录**： -- `colleague` → `./skills/colleague/{slug}/` -- `relationship` → `./skills/relationship/{slug}/` -- `celebrity` → `./skills/celebrity/{slug}/` - -如需改为全局路径，用 `--base-dir` 指向对应 character family 的根目录。 +| 任务 | 命令 | +|------|------| +| 零凭据：目录/文件 → `knowledge/` | `distilly harvest ` | +| 解析 ChatGPT / Claude / Slack / Telegram / Discord 导出 | `distilly parse-chat ` | +| 解析邮件 | `distilly parse-email ` | +| 解析字幕 | `distilly parse-subtitle ` | +| 解析文档 | `distilly parse-doc ` | +| 解析归档（X 官方归档 / Takeout / 社交平台导出） | `distilly parse-archive ` | +| 纯派生 → `evidence/derived/*.json` | `distilly retrospect` | +| 需要 key / OAuth 的渠道采集 | `distilly collect ` | +| 浏览器 computer use（必须带同意 token） | `distilly collect x --mode browser --consent ` | +| 音视频转写（可选后端） | `distilly transcribe ` | +| 把 LLM 自己读到的内容登记进账本 | `distilly note --from ` | +| 同意授权管理 | `distilly consent ` | +| 视图检查与渲染 | `distilly view check`；`distilly view render [--shareable]` | +| 证据体检 | `distilly doctor` | +| 生成 Skill 的创建/更新/列出/版本 | `distilly skill ` | +| 宿主安装 | `distilly install `；`distilly uninstall` | + +- 所有子命令支持 `--json` 回执；`--help` 有中文/英文两段。 +- 需要 key / OAuth 的渠道：先向用户说明将读取哪个渠道、可拿到什么，拿到同意后才运行 `distilly collect`。 +- computer-use 类命令必须带 `--consent `；没有 token 时命令以 `exit 2` 结束并在回执里写"等待用户同意"。 +- 密钥只从 `~/.distilly/*_config.json` 或环境变量读取；回执、日志、对话里只出现配置文件名，永不出现值。 + +### 迁移对照表（旧写法一律 deprecated） + +| 旧写法（deprecated） | 新写法 | +|----------------------|--------| +| `tools/feishu_parser.py`（deprecated） | `distilly parse-chat` | +| `tools/feishu_auto_collector.py`（deprecated） | `distilly collect feishu` | +| `tools/feishu_browser.py`（deprecated） | `distilly collect feishu --mode browser --consent ` | +| `tools/feishu_mcp_client.py`（deprecated） | `distilly collect feishu` | +| `tools/dingtalk_auto_collector.py`（deprecated） | `distilly collect dingtalk` | +| `tools/email_parser.py`（deprecated） | `distilly parse-email` | +| `tools/research/xquik_public_posts.py`（deprecated） | `distilly collect x` | +| `tools/research/transcribe_audio.py`（deprecated） | `distilly transcribe` | +| `tools/research/srt_to_transcript.py`（deprecated） | `distilly parse-subtitle` | +| `tools/skill_writer.py`（deprecated） | `distilly skill create` / `distilly skill update` / `distilly skill list` | +| `tools/version_manager.py`（deprecated） | `distilly skill version` | +| `tools/install_generated_skill.py`（deprecated） | `distilly install ` | +| `tools/research/quality_check.py`（deprecated） | `distilly doctor` | +| `tools/research/merge_research.py`（deprecated） | 暂无契约替代：只做派生，走 `distilly retrospect`，研究笔记合并细节见已知缺口 | +| `tools/research/download_subtitles.sh`（deprecated） | 暂无契约替代：先让用户提供本地字幕文件，再走 `distilly parse-subtitle` | + +迁移期允许两者并存，但新写法优先；只要 Python 工具还在被引用，就必须保留 `deprecated` 标注。 --- -## 主流程：创建新 Skill - -### Step 0：确认 character family - -如果用户使用的是 `/distilly`，先确认本次要蒸馏的是哪一类： - -1. `colleague` -2. `relationship` -3. `celebrity` - -如果上层宿主已经显式把 family 传进来，则直接固定对应的 character family。 - -如果当前 family 是 `celebrity`，还必须确认 research profile： - -1. `budget-friendly` -2. `budget-unfriendly` - -默认使用 `budget-friendly`。只有当用户明确要求更深研究、更高置信度、或者愿意接受更慢更贵的蒸馏流程时，才切到 `budget-unfriendly`。 - -### Step 1：基础信息录入 - -根据 character family 选择对应 intake prompt： - -- `colleague` → `prompts/intake.md` -- `relationship` → `prompts/relationship/intake.md` -- `celebrity` → `prompts/celebrity/intake.md` - -`colleague` 和 `relationship` 只问 3 个问题。 -`celebrity` 按 `prompts/celebrity/intake.md` 问 4 个问题，其中第 4 个问题必须确认 `research_profile`。 - -默认的 3 个基础问题： - -1. **花名/代号**（必填） -2. **基本信息**（一句话：公司、职级、职位、性别，想到什么写什么） - - 示例：`字节 2-1 后端工程师 男` -3. **性格画像**（一句话：MBTI、星座、个性标签、企业文化、印象） - - 示例：`INTJ 摩羯座 甩锅高手 字节范 CR很严格但从来不解释原因` - -除姓名外均可跳过。收集完后汇总确认，再进入下一步。 - -### Step 2：原材料导入 - -询问用户提供原材料，展示四种方式供选择： +## 磁盘契约（LLM 只能写这些） ``` -原材料怎么提供？ +skills/// + SKILL.md work.md persona.md work_skill.md persona_skill.md manifest.json meta.json + knowledge/{docs,messages,emails}/ + knowledge/raw//... # 原样字节，只增不改 + knowledge/text/.md # 归一化正文，段落锚点 [k0012] / [k0012:t3] + knowledge/index.json # 账本 {id,kind,origin,fetched_at,bytes,sha256,credentialed,method,warnings[]} + evidence/derived/*.json # retrospect 派生，每条结论带 evidence 锚点 + views/.view.json # LLM 只写章节/顺序/强调（不含事实） + views/.html # render 产物：单文件、离线、双主题 + evidence/renders/receipt.json # render 回执（sha256 + 字节数 + 内联来源） +``` - [A] 飞书自动采集（推荐） - 输入姓名，自动拉取消息记录 + 文档 + 多维表格 +- LLM 可以写：`views/.view.json`（只写章节、顺序、强调）、临时工作文件、以及通过 `distilly note --from ` 登记的"model-read"来源。 +- LLM 不可以写：`knowledge/raw/**`（原样字节，只增不改）、`knowledge/index.json`、`evidence/derived/*.json`（由 `distilly retrospect` 生成）、`evidence/renders/receipt.json`。 +- 截图、回执图、diff 图不入库（`.gitignore` 已含 `dst-evidence/`）；本地产物放 `/tmp/dst-evidence//`。 +- 锚点格式统一为 `[k00NN]`（4 位补零）或 `[k00NN:tM]`（带轮次）。任何结论必须带 `文件 + 锚点`，没有证据就写 `unknown`。 - [B] 钉钉自动采集 - 输入姓名，自动拉取文档 + 多维表格 - 消息记录通过浏览器采集（钉钉 API 不支持历史消息） +--- - [C] 飞书链接 - 直接给文档/Wiki 链接（浏览器登录态 或 MCP） +## 五步主线 - [D] 上传文件 - PDF / 图片 / 导出 JSON / 邮件 .eml +创建、追加、纠正都走同一条主线：**Collect → Derive → Read → Distill → Render**。 - [E] 直接粘贴内容 - 把文字复制进来 +| 步骤 | 必须存在的产物 | 计数判据 | sha256 从哪来 | 失败怎么办 | +|------|----------------|----------|---------------|------------| +| 1 Collect | `knowledge/raw//**`、`knowledge/text/.md`、`knowledge/index.json` | 每个落地来源 1 条账本条目；每个 text 文件 ≥1 个锚点 | `distilly --json` 回执的 `outputs[].sha256`，与 `knowledge/index.json` 的 `sha256` 逐字节一致 | 非零退出：记录命令、stderr、补救步骤；0 条落地来源时停下，不得进入 Derive | +| 2 Derive | `evidence/derived/*.json` | 每条派生结论带 evidence 锚点；连跑两次字节相同 | 回执 `outputs[].sha256`；两次运行 sha256 相同 | 非零退出：先修 `knowledge/index.json` 完整性；不得手写派生 JSON | +| 3 Read | 无新文件，产出"读了什么"的复述 | 按文件列出：文件 → 条数 → 锚点数 | 引用 `knowledge/index.json` 的 `sha256`，不自算 | 文件缺失或锚点为 0：回到 Step 1 补齐，不得凭记忆写结论 | +| 4 Distill | `work.md`、`persona.md`，celebrity 另有 research/audit/synthesis/validation | 每个维度有锚点或 `unknown`；celebrity 有明确 `PASS/FAIL` | 引用被引用的来源 sha256（来自账本） | 证据不足：标 `（原材料不足）` / candidate，并说明需要补什么材料 | +| 5 Render | `views/.view.json`、`views/.html`、`evidence/renders/receipt.json` | 回执 sha256 与 html 实际 sha256 一致；`distilly doctor` 锚点回指率可查 | `evidence/renders/receipt.json` 的 sha256 | 渲染失败：保留 view.json，不发布，报告错误 | -可以混用，也可以跳过（仅凭手动信息生成）。 -``` +任何一步的失败都不允许"静默降级"：要么修好，要么把失败写进对用户的汇报和回执的 `warnings[]` / `unavailable[]`。 ---- +### Step 1：Collect（采集） -#### 方式 A：飞书自动采集（推荐） +1. 先读 `prompts/collectors.md`，按"什么时候用哪条命令"选路。 +2. 零凭据来源（本地文件、导出包、字幕、文档、归档）直接走 `distilly harvest`、`distilly parse-chat`、`distilly parse-email`、`distilly parse-subtitle`、`distilly parse-doc`、`distilly parse-archive`。 +3. 需要 key / OAuth 的渠道（飞书、Slack、钉钉、X、Discord、Reddit、Notion、Gmail）先征求用户同意，再走 `distilly collect `；同意范围用 `distilly consent ` 管理。 +4. 浏览器 computer use 必须按 `prompts/computer-use.md` 执行：先问再动、只读白名单、默认 ≤20 屏 / ≤10 分钟 / 每分钟 ≤6 次滚动、每屏落盘原文 + URL + 时间 + 截图（截图只放本地）、可中断；`distilly collect x --mode browser --consent ` 没有 token 就直接退出，不要绕过。 +5. 用户只能"贴文字/截图"时，用 `distilly note --from ` 登记来源（`method:"model-read"`），不要假装它是采集来的。 +6. 音视频先 `distilly transcribe`，再解析字幕；不要把整段 transcript 抄进仓库。 -首次使用需配置： -```bash -python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" --setup -``` +**完成判据**：`knowledge/index.json` 里每个落地来源一条账本条目（含 `id`、`kind`、`origin`、`fetched_at`、`bytes`、`sha256`、`credentialed`、`method`、`warnings[]`）；每个 `knowledge/text/.md` 至少 1 个锚点；回执 `inputs[]`/`outputs[]` 的 sha256 与账本一致；不可用渠道进 `unavailable[]`。 +**失败怎么办**：命令非零退出时，把命令原文、stderr、补救步骤（例如缺凭据要配置哪个 `~/.distilly/*_config.json`）告诉用户，然后停下等指示；如果 0 条来源落地，不要进入 Step 2。 -**群聊采集**（使用 tenant_access_token，需 bot 在群内）： -```bash -python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" \ - --name "{name}" \ - --output-dir ./knowledge/{slug} \ - --msg-limit 1000 \ - --doc-limit 20 -``` +### 第 1.5 步：语料体检（进入 Derive 之前） -**私聊采集**（需要 user_access_token + 私聊 chat_id）： - -私聊消息只能通过用户身份（user_access_token）获取，应用身份无权访问私聊。 - -**前置条件**： - -用户需要提供以下信息： -1. **飞书应用凭证**：`app_id` 和 `app_secret`（在飞书开放平台创建自建应用获取） -2. **用户权限**：应用需开通以下用户权限（scope）： - - `im:message` — 以用户身份读取/发送消息 - - `im:chat` — 以用户身份读取会话列表 -3. **OAuth 授权码（code）**：用户在浏览器中完成 OAuth 授权后，从回调 URL 中获取 - -如果用户缺少以上任何信息，引导他们完成配置。不要假设用户已经配好了。 - -**获取 user_access_token 的完整流程**： - -当用户提供了 app_id、app_secret，并确认已开通用户权限后： - -1. 帮用户生成 OAuth 授权链接： - ``` - https://open.feishu.cn/open-apis/authen/v1/authorize?app_id={APP_ID}&redirect_uri=http://www.example.com&scope=im:message%20im:chat - ``` - > ⚠️ 注意：`redirect_uri` 需要在飞书应用的「安全设置 → 重定向 URL」中添加 `http://www.example.com` - -2. 用户在浏览器打开链接，登录并授权 -3. 页面会跳转到 `http://www.example.com?code=xxx`，用户复制 code 给你 -4. 用 code 换取 token： - ```bash - python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" --exchange-code {CODE} - ``` - 或者你自己写 Python 脚本调飞书 API 换取： - ```python - # 1. 获取 app_access_token - POST https://open.feishu.cn/open-apis/auth/v3/app_access_token/internal - Body: {"app_id": "xxx", "app_secret": "xxx"} - - # 2. 用 code 换 user_access_token - POST https://open.feishu.cn/open-apis/authen/v1/oidc/access_token - Header: Authorization: Bearer {app_access_token} - Body: {"grant_type": "authorization_code", "code": "xxx"} - ``` - -**获取私聊 chat_id**： - -用户通常不知道 chat_id。当用户有了 user_access_token 但没有 chat_id 时，你应该**自己写 Python 脚本**来获取： - -- **方法**：用 user_access_token 向对方的 open_id 发一条消息，返回值中会包含 chat_id - ```python - POST https://open.feishu.cn/open-apis/im/v1/messages?receive_id_type=open_id - Header: Authorization: Bearer {user_access_token} - Body: {"receive_id": "{对方open_id}", "msg_type": "text", "content": "{\"text\":\"你好\"}"} - # 返回值中的 chat_id 就是私聊会话 ID - ``` -- **注意**：`GET /im/v1/chats` 不会返回私聊会话，这是飞书 API 的限制，不是权限问题，不要尝试用这个接口找私聊 -- 如果用户不知道对方的 open_id，可以用 tenant_access_token 调通讯录 API 搜索： - ```python - GET https://open.feishu.cn/open-apis/contact/v3/scopes - # 返回应用可见范围内所有用户的 open_id - ``` - -**执行采集**： - -拿到 user_access_token 和 chat_id 后： -```bash -python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" \ - --open-id {对方open_id} \ - --p2p-chat-id {chat_id} \ - --user-token {user_access_token} \ - --name "{name}" \ - --output-dir ./knowledge/{slug} \ - --msg-limit 1000 -``` +1. 采集完成后先跑 `distilly doctor --require-shape`，读回执里的 `shape[]`。 +2. `shape[].verdict === "FAIL"` 时**停下**，把 `shape[].reasons[]` 原文告诉用户，并说明要补什么 + （典型：这是多人材料但只有 3% 的单元能归到某个说话人——需要这个人自己的产出： + 本人访谈/演讲字幕、本人文章、本人邮件，而不是会议流水）。 +3. 常见阈值（写在 `shape[].reasons[]` 里，不用背）：可引用单元 < 20 → FAIL；多人材料但可归属 + 单元 < 40%，或最活跃的人 < 20% → FAIL。没有说话人标注**不**判失败（本人文章、单人录音是正常的）。 +4. `verdict` 是 `PASS` 才继续 Step 2。 -**灵活性原则**：以上 API 调用不一定要用 collector 脚本，如果脚本跑不通或者场景不匹配，你可以直接写 Python 脚本调飞书 API 完成任务。核心 API 参考： -- 获取 token：`POST /auth/v3/app_access_token/internal`、`POST /authen/v1/oidc/access_token` -- 发消息（获取 chat_id）：`POST /im/v1/messages?receive_id_type=open_id` -- 拉消息：`GET /im/v1/messages?container_id_type=chat&container_id={chat_id}` -- 查通讯录：`GET /contact/v3/scopes`、`GET /contact/v3/users/{user_id}` - -自动采集内容： -- 群聊：所有与他共同群聊中他发出的消息（过滤系统消息、表情包） -- 私聊：与他的私聊完整对话（含双方消息，用于理解对话语境） -- 他创建/编辑的飞书文档和 Wiki -- 相关多维表格（如有权限） - -采集完成后用 `Read` 读取输出目录下的文件： -- `knowledge/{slug}/messages.txt` → 消息记录（群聊 + 私聊） -- `knowledge/{slug}/docs.txt` → 文档内容 -- `knowledge/{slug}/collection_summary.json` → 采集摘要 - -如果采集失败，根据报错自行判断原因并尝试修复，常见问题： -- 群聊采集：bot 未添加到群聊 -- 私聊采集：user_access_token 过期（有效期 2 小时，可用 refresh_token 刷新） -- 权限不足：引导用户在飞书开放平台开通对应权限并重新授权 -- 或改用方式 B/C +**为什么有这一步**：不先问"这份材料撑不撑得起一个人"，就会一路跑到 Step 4，产出的是 +"会议室的画像"而不是"这个人的画像"。这件事真实发生过一次。 ---- +### Step 2：Derive（派生） -#### 方式 B：钉钉自动采集 +1. 派生之前不要读 `evidence/derived/*`——先跑 `distilly retrospect`。 +2. `distilly retrospect` 只做纯派生：输入是 `knowledge/**`，输出是 `evidence/derived/*.json`，每条结论带 evidence 锚点。 +3. 为验证确定性，连跑两次；同一输入两次的 sha256 必须相同。 -首次使用需配置： -```bash -python3 "{distilly_skill_root}/tools/dingtalk_auto_collector.py" --setup -``` +**完成判据**：`evidence/derived/*.json` 存在；回执给出 `anchors.total` / `anchors.cited`；两次运行 `outputs[].sha256` 相同。 +**失败怎么办**：非零退出说明输入侧有问题——回到 Step 1 检查账本与 text 锚点；绝不手写、手改派生 JSON 来"跑通"。 -然后输入姓名，一键采集： -```bash -python3 "{distilly_skill_root}/tools/dingtalk_auto_collector.py" \ - --name "{name}" \ - --output-dir ./knowledge/{slug} \ - --msg-limit 500 \ - --doc-limit 20 \ - --show-browser # 首次使用加此参数，完成钉钉登录 -``` +### Step 3：Read（阅读） -采集内容： -- 他创建/编辑的钉钉文档和知识库 -- 多维表格 -- 消息记录（⚠️ 钉钉 API 不支持历史消息拉取，自动切换浏览器采集） +1. 读的顺序：`knowledge/index.json` → `knowledge/text/*.md` → `evidence/derived/*.json`。 +2. 先向用户复述"读了哪些文件、各多少条、多少锚点"，再写结论。 +3. 每条结论后面跟 `文件 + 锚点`（例如 `knowledge/text/feishu.md [k0042]`）。 +4. 找不到证据的结论写 `unknown`，并说明缺什么材料可以补上。 +5. 事实与候选分开：有具体锚点支撑的才算事实；派生文件里的模式、倾向、推断一律按候选处理，候选不能升级为结论。 +6. 全文细节规范见 `prompts/retrospection.md`。 -采集完成后 `Read` 读取： -- `knowledge/{slug}/docs.txt` -- `knowledge/{slug}/bitables.txt` -- `knowledge/{slug}/messages.txt` +**完成判据**：复述清单里的每个文件都能在账本里回指；被引用的锚点都真实存在于 `knowledge/text/**`；没有无锚点的结论。 +**失败怎么办**：文件缺失或锚点为 0 时回到 Step 1 补齐；不要凭记忆或常识补写内容。 -如消息采集失败，提示用户截图聊天记录后上传。 +### Step 4：Distill（蒸馏） ---- +先用第 0 步确认的 family 解析执行矩阵： -#### 方式 D：上传文件 - -- **PDF / 图片**：`Read` 工具直接读取 -- **飞书消息 JSON 导出**： - ```bash - python3 "{distilly_skill_root}/tools/feishu_parser.py" --file {path} --target "{name}" --output /tmp/feishu_out.txt - ``` - 然后 `Read /tmp/feishu_out.txt` -- **邮件文件 .eml / .mbox**： - ```bash - python3 "{distilly_skill_root}/tools/email_parser.py" --file {path} --target "{name}" --output /tmp/email_out.txt - ``` - 然后 `Read /tmp/email_out.txt` -- **Markdown / TXT**：`Read` 工具直接读取 +| character | intake | persona analyzer | persona builder | merger | storage root | +|-----------|--------|------------------|-----------------|--------|--------------| +| `colleague` | `prompts/intake.md` | `prompts/persona_analyzer.md` | `prompts/persona_builder.md` | `prompts/merger.md` | `./skills/colleague/{slug}` | +| `relationship` | `prompts/relationship/intake.md` | `prompts/relationship/persona_analyzer.md` | `prompts/relationship/persona_builder.md` | `prompts/relationship/merger.md` | `./skills/relationship/{slug}` | +| `celebrity` | `prompts/celebrity/intake.md` | `prompts/celebrity/persona_analyzer.md` | `prompts/celebrity/persona_builder.md` | `prompts/celebrity/merger.md` | `./skills/celebrity/{slug}` | ---- +所有 family 共用：Work analyzer `prompts/work_analyzer.md`、Work builder `prompts/work_builder.md`、Correction handler `prompts/correction_handler.md`。 -#### 方式 C：飞书链接 +两条线： -用户提供飞书文档/Wiki 链接时，询问读取方式： +- **线路 A（Work Skill）**：参考 `prompts/work_analyzer.md`，提取负责系统、技术规范、工作流程、输出偏好、经验知识；celebrity 场景下 `work` 更偏方法论、判断框架、决策习惯。 +- **线路 B（Persona）**：用当前 family 的 persona analyzer；`celebrity` + `research_profile=budget-unfriendly` 时改用 `prompts/celebrity/budget_unfriendly/persona_analyzer.md`。把用户填的标签翻译为具体行为规则，并从材料里提取表达风格、决策模式、人际行为。 -``` -检测到飞书链接，选择读取方式： - - [1] 浏览器方案（推荐） - 复用你本机 Chrome 的登录状态 - ✅ 内部文档、需要权限的文档都能读 - ✅ 无需配置 token - ⚠️ 需要本机安装 Chrome + playwright - - [2] MCP 方案 - 通过飞书 App Token 调用官方 API - ✅ 稳定，不依赖浏览器 - ✅ 可以读消息记录（需要群聊 ID） - ⚠️ 需要先配置 App ID / App Secret - ⚠️ 内部文档需要管理员给应用授权 - -选择 [1/2]： -``` +写文件时不要手工拼 `skills/{family}/{slug}` 文件树，统一走 writer：把 `meta.json` / `work.md` / `persona.md` 写到临时文件，再调 `distilly skill create`（或 `distilly skill update`）。人物 Skill 的安装走 `distilly install `。 -**选 1（浏览器方案）**： -```bash -python3 "{distilly_skill_root}/tools/feishu_browser.py" \ - --url "{feishu_url}" \ - --target "{name}" \ - --output /tmp/feishu_doc_out.txt -``` -首次使用若未登录，会弹出浏览器窗口要求登录（一次性）。 +**完成判据**：每个维度都有锚点或明确的 `（原材料不足）`；每条行为规则具体可执行；celebrity 的 audit / validation 给出明确 `PASS` 或 `FAIL`；`distilly doctor` 能报告证据覆盖率、不可用渠道、锚点回指率。celebrity 场景下的 research 门槛见下方子流程。 +**失败怎么办**：证据不足的维度标 `（原材料不足，建议追加相关文档）` 并降级为 candidate；`source_grounding` 不达标时保留 `FAIL` 并说明还缺什么，绝不用泛化链接刷过检查。 -**选 2（MCP 方案）**： +### Step 5：Render（渲染） -首次使用需初始化配置： -```bash -python3 "{distilly_skill_root}/tools/feishu_mcp_client.py" --setup -``` +1. 先 `distilly view check`，确认锚点都能回指到 `knowledge/index.json`。 +2. 写 `views/.view.json`：只写章节、顺序、强调，不写事实。 +3. `distilly view render` 生成单文件、离线、双主题的 `views/.html`，并写 `evidence/renders/receipt.json`（sha256 + 字节数 + 内联来源）。 +4. 对外分享时才用 `distilly view render --shareable`，并先让用户确认。 +5. 用 `distilly doctor` 复核证据覆盖率、不可用渠道、锚点回指率、computer-use 占比。 -之后直接读取： -```bash -python3 "{distilly_skill_root}/tools/feishu_mcp_client.py" \ - --url "{feishu_url}" \ - --output /tmp/feishu_doc_out.txt -``` +**完成判据**：`views/.html` 与 `evidence/renders/receipt.json` 同时存在；回执 sha256 与 html 实际 sha256 一致；内部链接 0 坏链。 +**失败怎么办**：渲染失败时保留 `views/.view.json`，不发布 HTML，把错误与缺失来源报告给用户。 -读取消息记录（需要群聊 ID，格式 `oc_xxx`）： -```bash -python3 "{distilly_skill_root}/tools/feishu_mcp_client.py" \ - --chat-id "oc_xxx" \ - --target "{name}" \ - --limit 500 \ - --output /tmp/feishu_msg_out.txt -``` +### 第 0 步（前置）：确认 family 与 intake -两种方式输出后均用 `Read` 读取结果文件，进入分析流程。 +如果用户使用的是 `/distilly`，先确认本次要蒸馏的是哪一类： ---- +1. `colleague` +2. `relationship` +3. `celebrity` -#### 方式 E：直接粘贴 +如果上层宿主已经显式把 family 传进来，则直接固定对应的 character family。 -用户粘贴的内容直接作为文本原材料，无需调用任何工具。 +如果当前 family 是 `celebrity`，还必须确认 research profile： ---- +1. `budget-friendly` +2. `budget-unfriendly` -如果用户说"没有文件"或"跳过"，仅凭 Step 1 的手动信息生成 Skill。 +默认使用 `budget-friendly`。只有当用户明确要求更深研究、更高置信度、或者愿意接受更慢更贵的蒸馏流程时，才切到 `budget-unfriendly`。 -### Step 3：分析原材料 +根据 family 选择 intake prompt：`colleague` → `prompts/intake.md`；`relationship` → `prompts/relationship/intake.md`；`celebrity` → `prompts/celebrity/intake.md`。`colleague` 和 `relationship` 只问 3 个问题；`celebrity` 问 4 个问题，其中第 4 个必须确认 `research_profile`。 -先根据 character family 解析本次的执行矩阵： +默认的 3 个基础问题： -| character | intake | persona analyzer | persona builder | merger | storage root | -|-----------|--------|------------------|-----------------|--------|--------------| -| `colleague` | `prompts/intake.md` | `prompts/persona_analyzer.md` | `prompts/persona_builder.md` | `prompts/merger.md` | `./skills/colleague/{slug}` | -| `relationship` | `prompts/relationship/intake.md` | `prompts/relationship/persona_analyzer.md` | `prompts/relationship/persona_builder.md` | `prompts/relationship/merger.md` | `./skills/relationship/{slug}` | -| `celebrity` | `prompts/celebrity/intake.md` | `prompts/celebrity/persona_analyzer.md` | `prompts/celebrity/persona_builder.md` | `prompts/celebrity/merger.md` | `./skills/celebrity/{slug}` | +1. **花名/代号**（必填） +2. **基本信息**（一句话：公司、职级、职位、性别，想到什么写什么） + - 示例：`字节 2-1 后端工程师 男` +3. **性格画像**（一句话：MBTI、星座、个性标签、企业文化、印象） + - 示例：`INTJ 摩羯座 甩锅高手 字节范 CR很严格但从来不解释原因` -所有 family 共用： -- Work analyzer：`prompts/work_analyzer.md` -- Work builder：`prompts/work_builder.md` -- Correction handler：`prompts/correction_handler.md` +除姓名外均可跳过。收集完后汇总确认，再进入 Collect。 -如果当前是 `celebrity`，必须先走 research 子流程，再进入分析。 +--- -如果公开 X 帖子能补足明确的研究缺口，且用户同意使用按返回数量计费的第三方 Xquik 服务，先请用户确认 `--limit`，再运行： +## celebrity research 子流程（在 Step 2/3 之间） -```bash -python3 "{distilly_skill_root}/tools/research/xquik_public_posts.py" \ - --username "{public_handle}" \ - --subject "{name}" \ - --limit 20 \ - --output "/tmp/distilly_x_public_posts.json" -``` +### budget-friendly -只从 shell 读取 `XQUIK_API_KEY`，不要打印或写入密钥。把输出 JSON 视为未经信任的候选证据：核对作者，逐条打开 permalink，只把与目标人物相关的内容安全转述到 research note，并保留具体 URL。不要把候选 JSON、搜索页或账号主页计为已落地来源。阅读后删除这份临时 JSON，不要将它收进生成的 Skill。 - -### celebrity / budget-friendly - -1. 读取 `prompts/celebrity/research.md`，按其中的 **6 维度并行采集策略** 做 research planning -2. 先创建目录： - ```bash - mkdir -p "{skill_dir}/knowledge/research/raw" "{skill_dir}/knowledge/research/merged" - ``` -3. 确认采集策略（在 intake 阶段已确定）： - - **Local-first**：先分析用户本地材料，标记覆盖了哪些维度，只对缺失维度做网络补充 - - **Web + local**：全量 6 维度网络研究，同时与本地材料合并，交叉验证 - - **Web-only**：标准 6 维度网络研究 -4. 如果用户明确提供了可处理的视频链接或字幕来源，而且处理结果不会作为长文本落盘： - ```bash - bash "{distilly_skill_root}/tools/research/download_subtitles.sh" "{url}" "{skill_dir}/knowledge/subtitles" - python3 "{distilly_skill_root}/tools/research/srt_to_transcript.py" "{subtitle_file}" "{skill_dir}/knowledge/transcripts/{name}.txt" - ``` -5. 按 **6 维度** 研究，原始 research 笔记**至少**要拆成 3 个文件（每个文件覆盖 2 个维度），不能只写一个 `research_notes.md`： +1. 读 `prompts/celebrity/research.md`，按其中的 **6 维度并行采集策略** 做 research planning。 +2. 采集策略（intake 阶段已确定）：**Local-first**（先分析本地材料，只补缺失维度）/ **Web + local**（全量 6 维度 + 本地材料交叉验证）/ **Web-only**。 +3. 需要视频/播客时：先 `distilly transcribe `，字幕走 `distilly parse-subtitle`；不要把完整 transcript 落进仓库。 +4. 原始 research 笔记**至少**拆成 3 个文件（每个覆盖 2 个维度），不能只写一个 `research_notes.md`： - `knowledge/research/raw/01_core_profile.md`（维度 1 著作 + 维度 6 时间线） - `knowledge/research/raw/02_conversations_and_material.md`（维度 2 对话 + 维度 4 决策） - `knowledge/research/raw/03_expression_and_reception.md`（维度 3 表达 DNA + 维度 5 他者视角） -6. 研究过程中必须遵守 **品味原则**（详见 research prompt）： - - 长文 > 金句，争议 > 共识，变化 > 固定，一手 > 二手 - - 遵守 **信源黑名单**：永不引用知乎、微信公众号、百度百科、内容农场 - - 遵守 **信源优先级**：用户本地材料 > 一手著作 > 长访谈 > 决策记录 > 社交媒体 > 外部分析 > 二手转述 -7. 合并 research： - ```bash - python3 "{distilly_skill_root}/tools/research/merge_research.py" "{skill_dir}" - ``` - 输出：`knowledge/research/merged/summary.md` -8. 读取 `knowledge/research/merged/summary.md`，确认： - - `Files scanned >= 3` - - `Unique URLs >= 2` - - `Potential long quote lines = 0` - - research notes 里的 URL 必须是**实际打开过的具体页面**，不是平台首页、搜索页、话题页或占位路径 - 如果不满足，继续补 research notes，直到满足或明确记录搜集受限原因。 -9. **质量关卡（Phase 1.5）**：在进入分析之前，必须向用户展示结构化采集摘要： - ``` - ┌──────────────────────────────┬──────────┬─────────────────────────────┐ - │ 维度 │ 来源数 │ 关键发现 │ - ├──────────────────────────────┼──────────┼─────────────────────────────┤ - │ 1 著作 │ N │ [核心论点 / 缺失] │ - │ 2 对话 │ N │ [关键模式 / 缺失] │ - │ 3 表达 DNA │ N │ [风格标记 / 缺失] │ - │ 4 决策 │ N │ [决策模式 / 缺失] │ - │ 5 他者视角 │ N │ [外部观点 / 缺失] │ - │ 6 时间线 │ N │ [认知轨迹 / 缺失] │ - ├──────────────────────────────┼──────────┼─────────────────────────────┤ - │ 矛盾点 │ N │ [摘要] │ - │ 薄弱维度 │ [列表] │ 补充方案：[计划] │ - │ 冷门人物？ │ 是/否 │ │ - └──────────────────────────────┴──────────┴─────────────────────────────┘ - ``` - 等待用户确认后再继续。如果用户指出问题或需要某个维度更深入，先补充研究。 -10. **冷门人物检测**：如果总来源 < 10 条，按冷门人物协议处理： - - 心智模型限制为 2–3 个 - - 薄弱模型标注"基于有限信息" - - 扩大诚实边界章节 - - 告知用户提供什么补充材料可以改善质量 -11. celebrity 的后续分析输入必须优先使用： - - 一手材料（信源权重 1-3） - - merged research summary - - 用户提供的补充描述 - -### celebrity / budget-unfriendly - -1. 先读取： - - `prompts/celebrity/budget_unfriendly/research.md` - - `references/celebrity_budget_unfriendly_framework.md` -2. 先创建目录： - ```bash - mkdir -p "{skill_dir}/knowledge/research/raw" "{skill_dir}/knowledge/research/merged" "{skill_dir}/knowledge/research/reviews" - ``` -3. 确认采集策略（在 intake 阶段已确定）：local-first / web+local / web-only -4. 按 **6-track 独立文件结构** 写 research notes（不可合并，不可克隆观察）： - - `knowledge/research/raw/01_writings.md`（维度 1：著作与系统思考） - - `knowledge/research/raw/02_conversations.md`（维度 2：即兴对话与压力应对） - - `knowledge/research/raw/03_expression_dna.md`（维度 3：语言指纹） - - `knowledge/research/raw/04_decisions.md`（维度 4：行为与选择） - - `knowledge/research/raw/05_external_views.md`（维度 5：他者视角与批评） - - `knowledge/research/raw/06_timeline.md`（维度 6：认知轨迹） -5. 研究过程必须遵守 **品味原则 + 信源黑名单 + 信源优先级**（见 research prompt），每条 evidence 必须标注 source weight (1-7)。 -6. 合并 research： - ```bash - python3 "{distilly_skill_root}/tools/research/merge_research.py" "{skill_dir}" - ``` -7. 读取 `knowledge/research/merged/summary.md`，确认最低门槛： - - `Files scanned >= 6` - - `Unique URLs >= 8` - - `Primary-source markers >= 3` - - `Source metadata blocks >= 6` - - `Contradiction bullets >= 6` - - `Inference bullets >= 6` - - `Potential long quote lines = 0` - - `Track coverage count = 6` - - research notes 里的 URL 必须是**实际打开过的具体页面**，不是平台首页、搜索页、话题页或占位路径 - 如果不满足，继续补对应 track，而不是直接进入后续 review。 -8. **质量关卡（Phase 1.5）**：在进入 audit 之前，向用户展示结构化采集摘要（含 primary 比例、矛盾数、候选 mental models、known-answer 候选、薄弱维度、冷门人物判定）。等待用户确认后再继续。 -9. 再读取： - - `prompts/celebrity/budget_unfriendly/audit.md` - - `prompts/celebrity/budget_unfriendly/synthesis.md` - - `references/celebrity_budget_unfriendly_template.md` -10. 先生成 `knowledge/research/reviews/research_audit.md` - - 审计必须明确给出 `PASS / FAIL` - - audit 必须检查：信源层级合规（无黑名单）、primary 比例 > 50%、品味原则遵守、冷门人物评估 - - 如果 audit 是 `FAIL`，按 audit 给出的 Backfill Tasks 补齐，不要跳到 synthesis -11. **提炼关卡（Phase 2.5）**：audit 通过后，向用户展示候选 mental models 摘要（含三重门判定、evidence anchors、failure modes）。确认合理性后再进入 synthesis。 -12. 再生成 `knowledge/research/reviews/synthesis.md` - - 必须对候选 mental models 做 triple-gate 判断： - - cross-context recurrence - - generative power - - exclusivity - - 同时提取智识谱系种子（influenced by / diverged from）和 Agentic Protocol 种子（该人物会如何分析新问题的维度列表） -13. 再按 `prompts/celebrity/budget_unfriendly/validation.md` 生成： - - `knowledge/research/reviews/validation.md` - - validation 必须明确给出 `PASS / FAIL` - - 必须做 known-answer check（至少 2 题）+ edge-case check（1 题）+ voice check（100 字盲测）+ copyright check + Agentic Protocol check - - 如果 validation 是 `FAIL`，必须先修 draft 再继续 -14. budget-unfriendly 的后续分析输入必须优先使用： - - 6-track raw notes - - merged research summary - - research audit - - synthesis review（含智识谱系种子、Agentic Protocol 种子） - - validation review - - 用户补充材料 - -两种 celebrity profile 的共同约束： - -- 如果外部搜集失败或被平台验证拦截： - - 明确告诉用户搜集受限的原因 - - 保留已有 research 原始材料和 merged summary - - 继续生成，但把 `source_grounding` 视为未完成 - - **不要**为了通过质量检查而编造 URL、引用、书名、视频标题，或塞入泛化主页链接 -- **不要**把完整 transcript、完整字幕、长段原文抄进仓库 -- 只允许保留结构化摘要、来源元信息和极短引用，避免版权风险 - -完成 family 解析后，再按两条线分析： - -**线路 A（Work Skill）**： -- 参考 `prompts/work_analyzer.md` -- 提取：负责系统、技术规范、工作流程、输出偏好、经验知识 -- celebrity 场景下，`work` 更偏方法论、判断框架、决策习惯，不要机械套成“工作职责” - -**线路 B（Persona）**： -- 使用当前 family 对应的 persona analyzer -- 如果 `celebrity` 且 `research_profile=budget-unfriendly`，改用： - - `prompts/celebrity/budget_unfriendly/persona_analyzer.md` -- 将用户填写的标签翻译为具体行为规则 -- 从原材料中提取：表达风格、决策模式、人际行为 -- celebrity 场景下，必须保留： - - mental models - - decision heuristics - - expression DNA - - contradictions - - honest boundaries - -### Step 4：生成并预览 - -使用 `prompts/work_builder.md` 生成 Work 内容。 -使用当前 family 对应的 persona builder 生成 Persona 内容。 - -具体映射： -- `colleague` → `prompts/persona_builder.md` -- `relationship` → `prompts/relationship/persona_builder.md` -- `celebrity` → `prompts/celebrity/persona_builder.md` -- `celebrity` + `budget-unfriendly` → `prompts/celebrity/budget_unfriendly/persona_builder.md` - -向用户展示摘要（各 5-8 行），询问： -``` -Work Skill 摘要： - - 负责：{xxx} - - 技术栈：{xxx} - - CR 重点：{xxx} - ... - -Persona 摘要： - - 核心性格：{xxx} - - 表达风格：{xxx} - - 决策模式：{xxx} - ... - -确认生成？还是需要调整？ -``` +5. 品味原则：长文 > 金句，争议 > 共识，变化 > 固定，一手 > 二手。信源黑名单：永不引用知乎、微信公众号、百度百科、内容农场。信源优先级：用户本地材料 > 一手著作 > 长访谈 > 决策记录 > 社交媒体 > 外部分析 > 二手转述。 +6. 合并研究笔记后确认 `Files scanned >= 3`、`Unique URLs >= 2`、`Potential long quote lines = 0`；notes 里的 URL 必须是实际打开过的具体页面，不是平台首页、搜索页、话题页或占位路径。 +7. **质量关卡（Phase 1.5）**：进入分析前向用户展示结构化采集摘要（6 维度来源数 + 关键发现 + 矛盾点 + 薄弱维度 + 冷门人物判定），等用户确认再继续。 +8. **冷门人物检测**：总来源 < 10 条时，心智模型限制为 2–3 个，薄弱模型标"基于有限信息"，扩大诚实边界章节，并告诉用户补什么材料能改善质量。 +9. 分析输入优先使用：一手材料（信源权重 1-3）> 合并后的 research summary > 用户补充描述。 -### Step 5：写入文件 - -用户确认后，不要手工拼接 `skills/colleague/{slug}` 这类文件树。统一走 writer： - -1. 先解析当前 storage root： - - `colleague` → `./skills/colleague` - - `relationship` → `./skills/relationship` - - `celebrity` → `./skills/celebrity` -2. 用 `Write` 工具写三个临时文件： - - `/tmp/distilly_{slug}_meta.json` - - `/tmp/distilly_{slug}_work.md` - - `/tmp/distilly_{slug}_persona.md` -3. `meta.json` 至少包含： - - `name` - - `display_name` - - `character` - - `research_profile`（当 character=`celebrity` 时必填） - - `classification.language`（必须设置为用户当前语言，例如 `zh-CN` 或 `en`） - - `profile` - - `tags` - - `knowledge_sources` -4. 然后调用： - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action create \ - --character {character} \ - --research-profile {research_profile} \ - --slug {slug} \ - --name "{name}" \ - --meta /tmp/distilly_{slug}_meta.json \ - --work /tmp/distilly_{slug}_work.md \ - --persona /tmp/distilly_{slug}_persona.md \ - --base-dir {resolved_base_dir} - ``` -5. 该命令会统一生成： - - `SKILL.md` - - `work.md` - - `persona.md` - - `work_skill.md` - - `persona_skill.md` - - `manifest.json` - - `meta.json` - - 如需把生成后的角色 Skill 安装到宿主： - - Claude Code：追加 `--install-claude-skill` - - OpenClaw：追加 `--install-openclaw-skill` - - Codex：追加 `--install-codex-skill` - - Hermes：运行 `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host hermes --force`；可信项目可追加 `--skills-dir .hermes/skills`，先运行 `hermes skills trust`，然后新建会话或运行 `/reload-skills`。只有已在 Hermes 的 `skills.external_dirs` 中显式配置时，才使用 `~/.agents/skills` - - DeepSeek Harness：运行 `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host deepseek-harness --force`；项目级安装追加 `--skills-dir .dsh/skills` - - Pi：运行 `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host pi --force`；项目级安装追加 `--skills-dir .pi/skills`，调用命令为 `/skill:{character}-{slug}` - - Grok Build：运行 `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host grok-build --force`；项目级安装追加 `--skills-dir .grok/skills` - - OpenCode：运行 `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host opencode --force`；项目级安装追加 `--skills-dir .opencode/skills` - - 统一安装器只写入自包含的 `SKILL.md` 和安装元数据，会在安装副本中规范旧版 frontmatter；不要手动复制整个生成目录，其中可能包含私有原始材料 - - Claude Code on Windows：可再追加 `--install-claude-command-shim` -6. 如果当前是 `celebrity`，创建完成后必须再跑一次质量检查： - ```bash - python3 "{distilly_skill_root}/tools/research/quality_check.py" "{resolved_base_dir}/{slug}/SKILL.md" --profile {research_profile} - ``` -7. 如果 `celebrity` 的质量检查仍然提示 `source_grounding` 失败： - - 可以补写诚实的来源说明和局限说明 - - 但只有在拿到真实、具体、可追溯的外部来源时，才能补充 URL - - **不要**用站点首页、topic 页、搜索页、个人空间首页等泛化链接来“刷过”检查 - - 如果没有真实来源，就保留 FAIL，并向用户说明后续需要补哪些材料 - -告知用户时，文件位置必须按当前 family 返回，不要默认写成 colleague。 +### budget-unfriendly + +1. 先读 `prompts/celebrity/budget_unfriendly/research.md` 和 `references/celebrity_budget_unfriendly_framework.md`。 +2. 按 **6-track 独立文件结构** 写 research notes（不可合并、不可克隆观察）：`01_writings.md` / `02_conversations.md` / `03_expression_dna.md` / `04_decisions.md` / `05_external_views.md` / `06_timeline.md`。 +3. 每条 evidence 必须标注 source weight (1-7)；遵守品味原则 + 信源黑名单 + 信源优先级。 +4. 最低门槛：`Files scanned >= 6`、`Unique URLs >= 8`、`Primary-source markers >= 3`、`Source metadata blocks >= 6`、`Contradiction bullets >= 6`、`Inference bullets >= 6`、`Potential long quote lines = 0`、`Track coverage count = 6`。不满足就补对应 track，不要跳到 review。 +5. 依次生成 `knowledge/research/reviews/research_audit.md`（明确 `PASS/FAIL`，检查信源层级、primary 比例 > 50%、品味原则、冷门人物）→ `synthesis.md`（triple gate：cross-context recurrence / generative power / exclusivity；提取智识谱系种子与 Agentic Protocol 种子）→ 按 `prompts/celebrity/budget_unfriendly/validation.md` 生成 `validation.md`（known-answer ≥2 题 + edge-case 1 题 + voice check 100 字盲测 + copyright check + Agentic Protocol check，明确 `PASS/FAIL`）。 +6. 任何 `FAIL` 都先补材料再继续；不要为了通过检查编造 URL、引用、书名或视频标题。 --- @@ -666,30 +296,13 @@ Persona 摘要： 用户提供新文件或文本时： -1. 按 Step 2 的方式读取新内容 -2. 根据当前 family 解析 base dir -3. 用 `Read` 读取现有 `{resolved_base_dir}/{slug}/work.md` 和 `persona.md` -4. 使用当前 family 对应的 merger prompt 分析增量内容 -5. 存档当前版本（用 Bash）： - ```bash - python3 "{distilly_skill_root}/tools/version_manager.py" \ - --action backup \ - --character {character} \ - --slug {slug} \ - --base-dir {resolved_base_dir} - ``` -6. 把 work/persona 增量分别写到临时 patch 文件 -7. 调用： - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action update \ - --character {character} \ - --slug {slug} \ - --work-patch /tmp/distilly_{slug}_work_patch.md \ - --persona-patch /tmp/distilly_{slug}_persona_patch.md \ - --base-dir {resolved_base_dir} - ``` -8. 如果当前是 `celebrity`，更新后再次执行 quality check +1. 按 Step 1 的 Collect 流程采集新内容（本地文件走 `distilly harvest`，导出走 `distilly parse-chat`，粘贴走 `distilly note --from -`）。 +2. 跑 `distilly retrospect` 刷新派生，再按 Step 3 复述"读了什么、多少条、多少锚点"。 +3. 根据当前 family 解析 base dir，读取现有 `{resolved_base_dir}/{slug}/work.md` 和 `persona.md`。 +4. 使用当前 family 对应的 merger prompt 分析增量内容。 +5. 用 `distilly skill version` 存档当前版本。 +6. 把 work/persona 增量分别写到临时 patch 文件，再走 `distilly skill update`。 +7. 如果当前是 `celebrity`，更新后用 `distilly doctor` 复核证据覆盖率。 --- @@ -697,76 +310,72 @@ Persona 摘要： 用户表达"不对"/"应该是"时： -1. 参考 `prompts/correction_handler.md` 识别纠正内容 -2. 判断属于 Work（技术/流程）还是 Persona（性格/沟通） -3. 如果属于 Work： - - 生成 `/tmp/distilly_{slug}_work_patch.md` - - patch 必须是可替换的 `##` section，不要直接手改最终文件 - - 调用： - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action update \ - --character {character} \ - --slug {slug} \ - --work-patch /tmp/distilly_{slug}_work_patch.md \ - --base-dir {resolved_base_dir} - ``` -4. 如果属于 Persona： - - 将 correction 写入 `/tmp/distilly_{slug}_correction.json` - - 单条纠正可直接写成 `{scene, wrong, correct}` - - 多条 persona 纠正可写成 `{"persona_corrections": [{...}, {...}]}` - - 调用： - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action update \ - --character {character} \ - --slug {slug} \ - --correction-json /tmp/distilly_{slug}_correction.json \ - --base-dir {resolved_base_dir} - ``` -5. 如果当前是 `celebrity`，更新后再次执行 quality check -6. 不要直接手改 `work.md`、`persona.md`、`SKILL.md`、`meta.json`；统一通过 writer 更新 +1. 参考 `prompts/correction_handler.md` 识别纠正内容。 +2. 判断属于 Work（技术/流程）还是 Persona（性格/沟通）。 +3. 如果属于 Work：生成可替换 `##` section 的 patch 临时文件，走 `distilly skill update`，不要直接手改 `work.md`。 +4. 如果属于 Persona：把 correction 写成 `{scene, wrong, correct}`（多条写成 `{"persona_corrections": [...]}`）的临时 JSON，走 `distilly skill update`。 +5. 纠正若与现有结论冲突，先向用户展示冲突再决定；纠正内容本身也要带锚点或标注为"用户口述，无锚点"。 +6. 如果当前是 `celebrity`，更新后用 `distilly doctor` 复核。 --- ## 管理操作 列出三类 Skill： + ```bash -python3 "{distilly_skill_root}/tools/skill_writer.py" --action list --character colleague --base-dir ./skills/colleague -python3 "{distilly_skill_root}/tools/skill_writer.py" --action list --character relationship --base-dir ./skills/relationship -python3 "{distilly_skill_root}/tools/skill_writer.py" --action list --character celebrity --base-dir ./skills/celebrity +distilly skill list ``` 回滚某个 Skill 版本： -```bash -# colleague -python3 "{distilly_skill_root}/tools/version_manager.py" --action rollback --character colleague --slug {slug} --version {version} --base-dir ./skills/colleague -# relationship -python3 "{distilly_skill_root}/tools/version_manager.py" --action rollback --character relationship --slug {slug} --version {version} --base-dir ./skills/relationship - -# celebrity -python3 "{distilly_skill_root}/tools/version_manager.py" --action rollback --character celebrity --slug {slug} --version {version} --base-dir ./skills/celebrity +```bash +distilly skill version ``` -删除某个 Skill： -确认 character 后执行： +删除某个 Skill（确认 character 后）： + ```bash -# colleague rm -rf skills/colleague/{slug} - -# relationship rm -rf skills/relationship/{slug} - -# celebrity rm -rf skills/celebrity/{slug} ``` +宿主安装：`distilly install `；卸载：`distilly uninstall`。 + +列出与撤销已授予的同意： + +```bash +distilly consent list +distilly consent revoke +``` + +--- + +## 必须 + +- 先列"读了哪些文件、各多少条、多少锚点"，再写结论；每条结论带 `文件 + 锚点`。 +- 无证据写 `unknown`；候选不当结论。 +- 每一步都按"五步主线"的完成判据检查产物，再进入下一步。 + +## 禁止 + +- 不改写引文，不伪造 URL、锚点、书名、视频标题，不用平台首页刷来源。 +- 密钥只从 `~/.distilly/*_config.json` 或环境变量读，永不写进对话、文件、回执或日志。 +- 不自己拼平台 API 请求：所有网络采集都通过 `distilly collect` / `distilly harvest` / `distilly parse-chat` / `distilly parse-email` / `distilly parse-subtitle` / `distilly parse-doc` / `distilly parse-archive` / `distilly transcribe`。 +- 不静默降级：失败、不可用渠道、没跑的步骤都要说清楚。 + +## 回执 + +- 读过哪些文件、各多少条、多少锚点。 +- 生成/更新了哪些文件，各自 sha256（来自 `distilly` 的 `--json` 回执、`knowledge/index.json` 或 `evidence/renders/receipt.json`）。 +- 哪些渠道不可用（`unavailable[]`）。 +- 哪些步骤没跑、为什么。 + --- --- -# English Version +## English # Distilly Creator @@ -803,716 +412,313 @@ Enter evolution mode when the user says: Compatibility update alias: - `/update-colleague {slug}` -When the user asks to see generated skills, use the list commands in "Management Operations" below. +When the user asks to see generated skills, use the list command in "Management Operations" below. --- -## Tool Usage Rules +## Command Contract (single entrypoint) -This Skill runs in any compatible host that can read local files and execute Bash / Python commands. Use the following tool conventions: +Every collection, derivation, and render step goes through `distilly`. Command names match the command table in `docs/v2/CONTRACT.md` §1 word for word; do not invent subcommands or fields. -| Task | Tool | -|------|------| -| Read PDF documents | `Read` tool (native PDF support) | -| Read image screenshots | `Read` tool (native image support) | -| Read MD/TXT files | `Read` tool | -| Parse Lark message JSON export | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_parser.py"` | -| Lark auto-collect (recommended) | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_auto_collector.py"` | -| Lark docs (browser session) | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_browser.py"` | -| Lark docs (MCP App Token) | `Bash` → `python3 "{distilly_skill_root}/tools/feishu_mcp_client.py"` | -| DingTalk auto-collect | `Bash` → `python3 "{distilly_skill_root}/tools/dingtalk_auto_collector.py"` | -| Collect public X post candidates | `Bash` → `python3 "{distilly_skill_root}/tools/research/xquik_public_posts.py"` | -| Parse email .eml/.mbox | `Bash` → `python3 "{distilly_skill_root}/tools/email_parser.py"` | -| Write/update Skill files | `Write` / `Edit` tool | -| Version management | `Bash` → `python3 "{distilly_skill_root}/tools/version_manager.py"` | -| List existing Skills | `Bash` → `python3 "{distilly_skill_root}/tools/skill_writer.py" --action list` | - -**Base directories**: -- `colleague` → `./skills/colleague/{slug}/` -- `relationship` → `./skills/relationship/{slug}/` -- `celebrity` → `./skills/celebrity/{slug}/` - -For a global path, use `--base-dir` with the storage root for that character family. - -The Lark-labelled compatibility collectors currently connect to the China-region `open.feishu.cn` / `feishu.cn` endpoints. International `larksuite.com` tenant routing is not implemented yet. +| Task | Command | +|------|---------| +| Zero-credential: directory/file → `knowledge/` | `distilly harvest ` | +| Parse ChatGPT / Claude / Slack / Telegram / Discord exports | `distilly parse-chat ` | +| Parse email | `distilly parse-email ` | +| Parse subtitles | `distilly parse-subtitle ` | +| Parse documents | `distilly parse-doc ` | +| Parse archives (X archive / Takeout / social exports) | `distilly parse-archive ` | +| Pure derivation → `evidence/derived/*.json` | `distilly retrospect` | +| Collection on channels needing key / OAuth | `distilly collect ` | +| Browser computer use (consent token required) | `distilly collect x --mode browser --consent ` | +| Audio/video transcription (optional backend) | `distilly transcribe ` | +| Register what the model itself read | `distilly note --from ` | +| Consent management | `distilly consent ` | +| View check and render | `distilly view check`; `distilly view render [--shareable]` | +| Evidence health check | `distilly doctor` | +| Create/update/list/version a generated Skill | `distilly skill ` | +| Host install | `distilly install `; `distilly uninstall` | + +- Every subcommand supports a `--json` receipt; `--help` has a Chinese and an English section. +- For channels needing a key or OAuth: first tell the user which channel will be read and what it yields, and only run `distilly collect` after they agree. +- Computer-use commands require `--consent `; without a token the command ends with `exit 2` and its receipt says it is waiting for user consent. +- Credentials are read only from `~/.distilly/*_config.json` or environment variables; receipts, logs, and chat only ever contain the config file name, never a value. + +### Migration table (all legacy forms are deprecated) + +| Legacy form (deprecated) | New form | +|--------------------------|----------| +| `tools/feishu_parser.py` (deprecated) | `distilly parse-chat` | +| `tools/feishu_auto_collector.py` (deprecated) | `distilly collect feishu` | +| `tools/feishu_browser.py` (deprecated) | `distilly collect feishu --mode browser --consent ` | +| `tools/feishu_mcp_client.py` (deprecated) | `distilly collect feishu` | +| `tools/dingtalk_auto_collector.py` (deprecated) | `distilly collect dingtalk` | +| `tools/email_parser.py` (deprecated) | `distilly parse-email` | +| `tools/research/xquik_public_posts.py` (deprecated) | `distilly collect x` | +| `tools/research/transcribe_audio.py` (deprecated) | `distilly transcribe` | +| `tools/research/srt_to_transcript.py` (deprecated) | `distilly parse-subtitle` | +| `tools/skill_writer.py` (deprecated) | `distilly skill create` / `distilly skill update` / `distilly skill list` | +| `tools/version_manager.py` (deprecated) | `distilly skill version` | +| `tools/install_generated_skill.py` (deprecated) | `distilly install ` | +| `tools/research/quality_check.py` (deprecated) | `distilly doctor` | +| `tools/research/merge_research.py` (deprecated) | No contract replacement yet: remains derivation only, use `distilly retrospect`; merging research notes is a known gap | +| `tools/research/download_subtitles.sh` (deprecated) | No contract replacement yet: ask the user for a local subtitle file, then use `distilly parse-subtitle` | + +Both forms may coexist during migration, but the new form wins; whenever a Python tool is still referenced, keep the `deprecated` marker. --- -## Main Flow: Create a New Skill +## Disk Contract (what the model may write) -### Step 0: Confirm the character family +``` +skills/// + SKILL.md work.md persona.md work_skill.md persona_skill.md manifest.json meta.json + knowledge/{docs,messages,emails}/ + knowledge/raw//... # raw bytes, append-only + knowledge/text/.md # normalized text, paragraph anchors [k0012] / [k0012:t3] + knowledge/index.json # ledger {id,kind,origin,fetched_at,bytes,sha256,credentialed,method,warnings[]} + evidence/derived/*.json # retrospect output, every conclusion carries evidence anchors + views/.view.json # the model writes only sections/order/emphasis (no facts) + views/.html # render output: single file, offline, dual theme + evidence/renders/receipt.json # render receipt (sha256 + bytes + inlined sources) +``` -If the user entered `/distilly`, first confirm which family should be distilled: +- The model may write: `views/.view.json` (sections, order, emphasis only), temporary working files, and sources registered through `distilly note --from ` with `method:"model-read"`. +- The model must not write: `knowledge/raw/**` (raw bytes, append-only), `knowledge/index.json`, `evidence/derived/*.json` (produced by `distilly retrospect`), `evidence/renders/receipt.json`. +- Screenshots, receipts, and diff images are never committed (`.gitignore` already covers `dst-evidence/`); local artifacts live in `/tmp/dst-evidence//`. +- Anchor format is always `[k00NN]` (4-digit zero-padded) or `[k00NN:tM]` (with turn index). Every conclusion carries `file + anchor`; with no evidence, write `unknown`. -1. `colleague` -2. `relationship` -3. `celebrity` +--- -If the host already passed an explicit family, lock the character family immediately. +## Five-Step Mainline -If the current family is `celebrity`, also confirm the research profile: +Creation, append, and correction all follow one mainline: **Collect → Derive → Read → Distill → Render**. -1. `budget-friendly` -2. `budget-unfriendly` - -Default to `budget-friendly`. Only switch to `budget-unfriendly` when the user explicitly wants deeper research, higher confidence, or accepts a slower and more expensive distillation pass. +| Step | Required artifacts | Count criteria | Where sha256 comes from | What to do on failure | +|------|--------------------|----------------|-------------------------|-----------------------| +| 1 Collect | `knowledge/raw//**`, `knowledge/text/.md`, `knowledge/index.json` | 1 ledger entry per grounded source; ≥1 anchor in every text file | `outputs[].sha256` of the `distilly --json` receipt, byte-identical to `sha256` in `knowledge/index.json` | Non-zero exit: record command, stderr, remedy; with 0 grounded sources stop and do not enter Derive | +| 2 Derive | `evidence/derived/*.json` | Every derived conclusion carries evidence anchors; two runs are byte-identical | Receipt `outputs[].sha256`; identical sha256 across two runs | Non-zero exit: first repair `knowledge/index.json` integrity; never hand-write derived JSON | +| 3 Read | No new files; produce a restatement of what was read | Per file: file → rows → anchors | Quote `sha256` from `knowledge/index.json`; never compute your own | Missing files or zero anchors: go back to Step 1; never write conclusions from memory | +| 4 Distill | `work.md`, `persona.md`, plus celebrity research/audit/synthesis/validation | Every dimension has anchors or `unknown`; celebrity has an explicit `PASS/FAIL` | Quote the sha256 of cited sources from the ledger | Thin evidence: mark `(insufficient source material)` / candidate and say what material is missing | +| 5 Render | `views/.view.json`, `views/.html`, `evidence/renders/receipt.json` | Receipt sha256 matches the actual html sha256; `distilly doctor` reports the anchor back-reference rate | sha256 in `evidence/renders/receipt.json` | Render failure: keep view.json, do not publish, report the error | -### Step 1: Basic Info Collection +No step may degrade silently: either fix it, or state the failure in the user-facing report and in the receipt's `warnings[]` / `unavailable[]`. -Choose the intake prompt by character family: +### Step 1: Collect -- `colleague` → `prompts/intake.md` -- `relationship` → `prompts/relationship/intake.md` -- `celebrity` → `prompts/celebrity/intake.md` +1. Read `prompts/collectors.md` first and pick the route from its "which command when" table. +2. Zero-credential sources (local files, export bundles, subtitles, documents, archives) go straight to `distilly harvest`, `distilly parse-chat`, `distilly parse-email`, `distilly parse-subtitle`, `distilly parse-doc`, `distilly parse-archive`. +3. Channels needing a key or OAuth (Feishu, Slack, DingTalk, X, Discord, Reddit, Notion, Gmail) require the user's consent first, then `distilly collect `; manage consent scope with `distilly consent `. +4. Browser computer use must follow `prompts/computer-use.md`: ask before acting, read-only whitelist, default ≤20 screens / ≤10 minutes / ≤6 scrolls per minute, every screen persisted with raw text + URL + timestamp + screenshot (screenshots stay local), interruptible; `distilly collect x --mode browser --consent ` must exit without a token — never work around it. +5. When the user can only paste text or screenshots, register the source with `distilly note --from ` (`method:"model-read"`); never pretend it was collected. +6. Transcribe audio/video with `distilly transcribe` before parsing subtitles; never commit a full transcript to the repository. -For `colleague` and `relationship`, ask only 3 questions. -For `celebrity`, use the 4-question intake in `prompts/celebrity/intake.md`; the fourth question must confirm `research_profile`. +**Completion criteria**: `knowledge/index.json` has one ledger entry per grounded source (with `id`, `kind`, `origin`, `fetched_at`, `bytes`, `sha256`, `credentialed`, `method`, `warnings[]`); every `knowledge/text/.md` has at least one anchor; receipt `inputs[]`/`outputs[]` sha256 matches the ledger; unavailable channels appear in `unavailable[]`. +**On failure**: when a command exits non-zero, report the exact command, its stderr, and the remedy (for example which `~/.distilly/*_config.json` must be configured), then stop and wait for instructions; if 0 sources landed, do not enter Step 2. -The default 3 base questions are: +### Step 1.5: Corpus shape check (before Derive) -1. **Alias / Codename** (required) -2. **Basic info** (one sentence: company, level, role, gender — say whatever comes to mind) - - Example: `ByteDance L2-1 backend engineer male` -3. **Personality profile** (one sentence: MBTI, zodiac, traits, corporate culture, impressions) - - Example: `INTJ Capricorn blame-shifter ByteDance-style strict in CR but never explains why` +1. Once collection is done, run `distilly doctor --require-shape` and read `shape[]` in the receipt. +2. On `shape[].verdict === "FAIL"` **stop**, quote `shape[].reasons[]` to the user, and say what to add + (typically: multi-speaker material with only 3% of units attributable — ask for that person's own + output: their interviews/talks, their writing, their mail, not a meeting stream). +3. The thresholds live in `shape[].reasons[]`, so nothing has to be memorised: fewer than 20 citable + units → FAIL; multi-speaker material with under 40% attributable units, or a top speaker under 20% + → FAIL. Missing speaker labels is **not** a failure (a person's own writing or a solo recording is fine). +4. Only continue to Step 2 on `PASS`. -Everything except the alias can be skipped. Summarize and confirm before moving to the next step. +**Why this step exists**: without asking "can this material carry a person at all", the run goes all the +way to Step 4 and produces a portrait of a room instead of a portrait of a person. That happened once. -### Step 2: Source Material Import +### Step 2: Derive -Ask the user how they'd like to provide materials: +1. Do not read `evidence/derived/*` before deriving — run `distilly retrospect` first. +2. `distilly retrospect` is pure derivation: input is `knowledge/**`, output is `evidence/derived/*.json`, and every conclusion carries evidence anchors. +3. To prove determinism, run it twice; the same input must produce identical sha256. -``` -How would you like to provide source materials? +**Completion criteria**: `evidence/derived/*.json` exists; the receipt reports `anchors.total` / `anchors.cited`; `outputs[].sha256` is identical across two runs. +**On failure**: a non-zero exit means the input side is broken — go back to Step 1 and check the ledger and text anchors; never hand-write or hand-edit derived JSON to force a pass. - [A] Lark Auto-Collect (recommended) - Enter name, auto-pull messages + docs + spreadsheets +### Step 3: Read - [B] DingTalk Auto-Collect - Enter name, auto-pull docs + spreadsheets - Messages collected via browser (DingTalk API doesn't support message history) +1. Read in this order: `knowledge/index.json` → `knowledge/text/*.md` → `evidence/derived/*.json`. +2. Restate to the user "which files were read, how many rows each, how many anchors" before writing conclusions. +3. Every conclusion carries `file + anchor` (for example `knowledge/text/feishu.md [k0042]`). +4. Conclusions without evidence are written as `unknown`, together with what material would supply the evidence. +5. Keep facts and candidates apart: only statements backed by a specific anchor are facts; patterns, tendencies, and inferences from derived files stay candidates and never get promoted to conclusions. +6. Full detail rules are in `prompts/retrospection.md`. - [C] Lark Link - Provide doc/Wiki link (browser session or MCP) +**Completion criteria**: every file in the restatement list back-references into the ledger; every cited anchor really exists in `knowledge/text/**`; no conclusion is left without an anchor. +**On failure**: with missing files or zero anchors, go back to Step 1; never fill the gap from memory or general knowledge. - [D] Upload Files - PDF / images / exported JSON / email .eml +### Step 4: Distill - [E] Paste Text - Copy-paste text directly +Resolve the execution matrix for the family confirmed in Step 0: -Can mix and match, or skip entirely (generate from manual info only). -``` +| character | intake | persona analyzer | persona builder | merger | storage root | +|-----------|--------|------------------|-----------------|--------|--------------| +| `colleague` | `prompts/intake.md` | `prompts/persona_analyzer.md` | `prompts/persona_builder.md` | `prompts/merger.md` | `./skills/colleague/{slug}` | +| `relationship` | `prompts/relationship/intake.md` | `prompts/relationship/persona_analyzer.md` | `prompts/relationship/persona_builder.md` | `prompts/relationship/merger.md` | `./skills/relationship/{slug}` | +| `celebrity` | `prompts/celebrity/intake.md` | `prompts/celebrity/persona_analyzer.md` | `prompts/celebrity/persona_builder.md` | `prompts/celebrity/merger.md` | `./skills/celebrity/{slug}` | ---- +Shared across all families: Work analyzer `prompts/work_analyzer.md`, Work builder `prompts/work_builder.md`, Correction handler `prompts/correction_handler.md`. -#### Option A: Lark Auto-Collect (Recommended) +Two tracks: -First-time setup: -```bash -python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" --setup -``` +- **Track A (Work Skill)**: follow `prompts/work_analyzer.md` and extract responsible systems, technical standards, workflow, output preferences, experience. For `celebrity`, interpret `work` as methods, judgment frameworks, and decision patterns. +- **Track B (Persona)**: use the family-specific persona analyzer; for `celebrity` with `research_profile=budget-unfriendly`, switch to `prompts/celebrity/budget_unfriendly/persona_analyzer.md`. Translate the user's tags into concrete behavior rules and extract communication style, decision patterns, and interpersonal behavior from the material. -**Group chat collection** (uses tenant_access_token, bot must be in the group): -```bash -python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" \ - --name "{name}" \ - --output-dir ./knowledge/{slug} \ - --msg-limit 1000 \ - --doc-limit 20 -``` +Never hand-build a `skills/{family}/{slug}` tree: write `meta.json` / `work.md` / `persona.md` to temporary files and call `distilly skill create` (or `distilly skill update`). Install a generated person Skill with `distilly install `. -**Private chat (P2P) collection** (requires user_access_token + p2p chat_id): - -Private messages can only be accessed via user identity (user_access_token). App identity cannot access private chats. - -**Prerequisites**: - -The user needs to provide: -1. **Lark app credentials**: `app_id` and `app_secret` (from the Open Platform) -2. **User scopes**: The app must have these user scopes enabled: - - `im:message` — read/send messages as user - - `im:chat` — read chat list as user -3. **OAuth authorization code**: obtained after user completes OAuth in browser - -If the user is missing any of these, guide them through setup. Don't assume anything is pre-configured. - -**Getting user_access_token**: - -Once the user provides app_id, app_secret, and confirms scopes are enabled: - -1. Generate the OAuth URL for them: - ``` - https://open.feishu.cn/open-apis/authen/v1/authorize?app_id={APP_ID}&redirect_uri=http://www.example.com&scope=im:message%20im:chat - ``` - > ⚠️ The redirect_uri must be added in the app's "Security Settings → Redirect URLs" - -2. User opens URL, logs in, authorizes -3. Page redirects to `http://www.example.com?code=xxx`, user copies the code -4. Exchange code for token: - ```bash - python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" --exchange-code {CODE} - ``` - Or write a Python script to call the same API directly: - ```python - # 1. Get app_access_token - POST https://open.feishu.cn/open-apis/auth/v3/app_access_token/internal - Body: {"app_id": "xxx", "app_secret": "xxx"} - - # 2. Exchange code for user_access_token - POST https://open.feishu.cn/open-apis/authen/v1/oidc/access_token - Header: Authorization: Bearer {app_access_token} - Body: {"grant_type": "authorization_code", "code": "xxx"} - ``` - -**Getting the p2p chat_id**: - -Users typically don't know their chat_id. When the user has a user_access_token but no chat_id, **write a Python script yourself** to obtain it: - -- **Method**: Send a message to the other user's open_id — the response includes the chat_id - ```python - POST https://open.feishu.cn/open-apis/im/v1/messages?receive_id_type=open_id - Header: Authorization: Bearer {user_access_token} - Body: {"receive_id": "{target_open_id}", "msg_type": "text", "content": "{\"text\":\"hello\"}"} - # The chat_id in the response is the p2p chat ID - ``` -- **Important**: `GET /im/v1/chats` does NOT return p2p chats — this is an API limitation, not a permission issue. Do not try to use it for finding private chats. -- If the user doesn't know the target's open_id, use tenant_access_token to search contacts: - ```python - GET https://open.feishu.cn/open-apis/contact/v3/scopes - # Returns open_ids of all users visible to the app - ``` - -**Running collection**: - -Once you have user_access_token and chat_id: -```bash -python3 "{distilly_skill_root}/tools/feishu_auto_collector.py" \ - --open-id {target_open_id} \ - --p2p-chat-id {chat_id} \ - --user-token {user_access_token} \ - --name "{name}" \ - --output-dir ./knowledge/{slug} \ - --msg-limit 1000 -``` +**Completion criteria**: every dimension has anchors or an explicit `(insufficient source material)`; every behavior rule is concrete and executable; celebrity audit / validation returns an explicit `PASS` or `FAIL`; `distilly doctor` can report evidence coverage, unavailable channels, and the anchor back-reference rate. Celebrity research thresholds are in the subflow below. +**On failure**: mark thin dimensions `(insufficient source material, add related documents)` and downgrade them to candidates; when `source_grounding` fails, keep the `FAIL` and explain what is missing instead of padding with generic links. -**Flexibility principle**: The above API calls don't have to go through the collector script. If the script doesn't work or doesn't fit the scenario, write Python scripts directly against the same endpoints. Key API reference: -- Get token: `POST /auth/v3/app_access_token/internal`, `POST /authen/v1/oidc/access_token` -- Send message (get chat_id): `POST /im/v1/messages?receive_id_type=open_id` -- Fetch messages: `GET /im/v1/messages?container_id_type=chat&container_id={chat_id}` -- Search contacts: `GET /contact/v3/scopes`, `GET /contact/v3/users/{user_id}` - -Auto-collected content: -- Group chats: messages sent by them (system messages and stickers filtered) -- Private chats: full conversation with both parties (for context understanding) -- Lark docs and Wikis they created/edited -- Related spreadsheets (if accessible) - -After collection, `Read` the output files: -- `knowledge/{slug}/messages.txt` → messages (group + private) -- `knowledge/{slug}/docs.txt` → document content -- `knowledge/{slug}/collection_summary.json` → collection summary - -If collection fails, diagnose the error and attempt to fix it. Common issues: -- Group chat: bot not added to the group -- Private chat: user_access_token expired (2-hour TTL, refresh with refresh_token) -- Insufficient permissions: guide user to enable scopes and re-authorize -- Or switch to Option B/C +### Step 5: Render ---- +1. Run `distilly view check` first to confirm every anchor back-references to `knowledge/index.json`. +2. Write `views/.view.json`: sections, order, and emphasis only — no facts. +3. `distilly view render` produces the single-file, offline, dual-theme `views/.html` and writes `evidence/renders/receipt.json` (sha256 + bytes + inlined sources). +4. Use `distilly view render --shareable` only for external sharing, and confirm with the user first. +5. Re-check evidence coverage, unavailable channels, anchor back-reference rate, and computer-use share with `distilly doctor`. -#### Option B: DingTalk Auto-Collect +**Completion criteria**: both `views/.html` and `evidence/renders/receipt.json` exist; the receipt sha256 matches the actual html sha256; zero broken internal links. +**On failure**: when rendering fails, keep `views/.view.json`, do not publish the HTML, and report the error and the missing sources to the user. -First-time setup: -```bash -python3 "{distilly_skill_root}/tools/dingtalk_auto_collector.py" --setup -``` +### Step 0 (prerequisite): Confirm the family and run intake -Then enter the name: -```bash -python3 "{distilly_skill_root}/tools/dingtalk_auto_collector.py" \ - --name "{name}" \ - --output-dir ./knowledge/{slug} \ - --msg-limit 500 \ - --doc-limit 20 \ - --show-browser # add this flag on first use to complete DingTalk login -``` +If the user entered `/distilly`, first confirm which family should be distilled: -Collected content: -- DingTalk docs and knowledge bases they created/edited -- Spreadsheets -- Messages (⚠️ DingTalk API doesn't support message history — auto-switches to browser scraping) +1. `colleague` +2. `relationship` +3. `celebrity` -After collection, `Read`: -- `knowledge/{slug}/docs.txt` -- `knowledge/{slug}/bitables.txt` -- `knowledge/{slug}/messages.txt` +If the host already passed an explicit family, lock the character family immediately. -If message collection fails, prompt user to upload chat screenshots. +If the current family is `celebrity`, also confirm the research profile: ---- +1. `budget-friendly` +2. `budget-unfriendly` -#### Option D: Upload Files - -- **PDF / Images**: `Read` tool directly -- **Lark message JSON export**: - ```bash - python3 "{distilly_skill_root}/tools/feishu_parser.py" --file {path} --target "{name}" --output /tmp/feishu_out.txt - ``` - Then `Read /tmp/feishu_out.txt` -- **Email files .eml / .mbox**: - ```bash - python3 "{distilly_skill_root}/tools/email_parser.py" --file {path} --target "{name}" --output /tmp/email_out.txt - ``` - Then `Read /tmp/email_out.txt` -- **Markdown / TXT**: `Read` tool directly +Default to `budget-friendly`. Only switch to `budget-unfriendly` when the user explicitly wants deeper research, higher confidence, or accepts a slower and more expensive distillation pass. ---- +Choose the intake prompt by family: `colleague` → `prompts/intake.md`; `relationship` → `prompts/relationship/intake.md`; `celebrity` → `prompts/celebrity/intake.md`. `colleague` and `relationship` ask only 3 questions; `celebrity` asks 4, and the fourth must confirm `research_profile`. -#### Option C: Lark Link +The default 3 base questions: -When the user provides a Lark doc/Wiki link, ask which method to use: +1. **Alias / Codename** (required) +2. **Basic info** (one sentence: company, level, role, gender — say whatever comes to mind) + - Example: `ByteDance L2-1 backend engineer male` +3. **Personality profile** (one sentence: MBTI, zodiac, traits, corporate culture, impressions) + - Example: `INTJ Capricorn blame-shifter ByteDance-style strict in CR but never explains why` -``` -Lark link detected. Choose read method: - - [1] Browser Method (recommended) - Reuses your local Chrome login session - ✅ Works with internal docs requiring permissions - ✅ No token configuration needed - ⚠️ Requires Chrome + playwright installed locally - - [2] MCP Method - Uses a Lark App Token via the official API - ✅ Stable, no browser dependency - ✅ Can read messages (needs chat ID) - ⚠️ Requires App ID / App Secret setup - ⚠️ Internal docs need admin authorization for the app - -Choose [1/2]: -``` +Everything except the alias can be skipped. Summarize and confirm before entering Collect. -**Option 1 (Browser)**: -```bash -python3 "{distilly_skill_root}/tools/feishu_browser.py" \ - --url "{feishu_url}" \ - --target "{name}" \ - --output /tmp/feishu_doc_out.txt -``` -First use will open a browser window for login (one-time). +--- -**Option 2 (MCP)**: +## Celebrity research subflow (between Step 2 and Step 3) -First-time setup: -```bash -python3 "{distilly_skill_root}/tools/feishu_mcp_client.py" --setup -``` +### budget-friendly -Then read directly: -```bash -python3 "{distilly_skill_root}/tools/feishu_mcp_client.py" \ - --url "{feishu_url}" \ - --output /tmp/feishu_doc_out.txt -``` +1. Read `prompts/celebrity/research.md` and follow its **6-dimension parallel collection strategy**. +2. Collection strategy (fixed during intake): **Local-first** (analyze local material first, search only the gaps) / **Web + local** (full 6-dimension web research cross-validated with local material) / **Web-only**. +3. For video or podcasts: `distilly transcribe ` first, then `distilly parse-subtitle`; never commit a full transcript. +4. Split the raw research notes across **at least 3 files** (2 dimensions each), never one monolithic `research_notes.md`: + - `knowledge/research/raw/01_core_profile.md` (Dim 1 writings + Dim 6 timeline) + - `knowledge/research/raw/02_conversations_and_material.md` (Dim 2 conversations + Dim 4 decisions) + - `knowledge/research/raw/03_expression_and_reception.md` (Dim 3 expression DNA + Dim 5 external views) +5. Taste principles: long-form > snippets, controversy > consensus, change > fixity, firsthand > secondhand. Source blacklist: never cite Zhihu, WeChat official accounts, Baidu Baike, content farms. Source hierarchy: user local material > first-person works > long interviews > decision records > social media > external analysis > secondhand summaries. +6. Confirm `Files scanned >= 3`, `Unique URLs >= 2`, `Potential long quote lines = 0`; every URL must be a specific page actually opened, not a platform root, search page, topic page, or placeholder. +7. **Quality checkpoint (Phase 1.5)**: show the user a structured collection summary (sources per dimension + key findings + contradictions + thin dimensions + cold-figure verdict) and wait for confirmation. +8. **Cold figure detection**: below 10 total sources, limit mental models to 2–3, mark thin models "based on limited information", expand the honest boundaries section, and tell the user what material would improve quality. +9. Analysis input priority: primary material (source weight 1-3) > merged research summary > explicit user notes. -Read messages (needs chat ID, format `oc_xxx`): -```bash -python3 "{distilly_skill_root}/tools/feishu_mcp_client.py" \ - --chat-id "oc_xxx" \ - --target "{name}" \ - --limit 500 \ - --output /tmp/feishu_msg_out.txt -``` +### budget-unfriendly -Both methods output to files, then use `Read` to load results into analysis. +1. Read `prompts/celebrity/budget_unfriendly/research.md` and `references/celebrity_budget_unfriendly_framework.md` first. +2. Write the **six-track research set** as independent files (never merged, never cloned): `01_writings.md` / `02_conversations.md` / `03_expression_dna.md` / `04_decisions.md` / `05_external_views.md` / `06_timeline.md`. +3. Every evidence item carries a source weight (1-7); follow taste principles + source blacklist + source hierarchy. +4. Minimum floor: `Files scanned >= 6`, `Unique URLs >= 8`, `Primary-source markers >= 3`, `Source metadata blocks >= 6`, `Contradiction bullets >= 6`, `Inference bullets >= 6`, `Potential long quote lines = 0`, `Track coverage count = 6`. If short, fill the weak track instead of skipping to review. +5. Write, in order: `knowledge/research/reviews/research_audit.md` (explicit `PASS/FAIL`; checks source hierarchy, primary ratio > 50%, taste principles, cold figure) → `synthesis.md` (triple gate: cross-context recurrence / generative power / exclusivity; extract intellectual genealogy and Agentic Protocol seeds) → `validation.md` per `prompts/celebrity/budget_unfriendly/validation.md` (known-answer ≥2 questions + 1 edge case + 100-word voice check + copyright check + Agentic Protocol check, explicit `PASS/FAIL`). +6. Any `FAIL` means backfill first; never invent URLs, quotes, book titles, or video titles to pass a check. --- -#### Option E: Paste Text +## Evolution Mode: Append Files + +When the user provides new files or text: -User-pasted content is used directly as text material. No tools needed. +1. Collect the new material with the Step 1 flow (`distilly harvest` for local files, `distilly parse-chat` for exports, `distilly note --from -` for pasted text). +2. Run `distilly retrospect` to refresh derivations, then restate "what was read, how many rows, how many anchors" per Step 3. +3. Resolve the base dir for the current family and read the existing `{resolved_base_dir}/{slug}/work.md` and `persona.md`. +4. Analyze the delta with the family-specific merger prompt. +5. Archive the current version with `distilly skill version`. +6. Write the work/persona deltas to temporary patch files and apply them with `distilly skill update`. +7. For `celebrity`, re-check evidence coverage with `distilly doctor` after the update. --- -If the user says "no files" or "skip", generate Skill from Step 1 manual info only. - -### Step 3: Analyze Source Material +## Evolution Mode: Conversation Correction -First resolve the execution matrix for the selected character family: +When the user says "that's wrong" / "he should be": -| character | intake | persona analyzer | persona builder | merger | storage root | -|-----------|--------|------------------|-----------------|--------|--------------| -| `colleague` | `prompts/intake.md` | `prompts/persona_analyzer.md` | `prompts/persona_builder.md` | `prompts/merger.md` | `./skills/colleague/{slug}` | -| `relationship` | `prompts/relationship/intake.md` | `prompts/relationship/persona_analyzer.md` | `prompts/relationship/persona_builder.md` | `prompts/relationship/merger.md` | `./skills/relationship/{slug}` | -| `celebrity` | `prompts/celebrity/intake.md` | `prompts/celebrity/persona_analyzer.md` | `prompts/celebrity/persona_builder.md` | `prompts/celebrity/merger.md` | `./skills/celebrity/{slug}` | +1. Identify the correction with `prompts/correction_handler.md`. +2. Decide whether it belongs to Work (technical/workflow) or Persona (personality/communication). +3. Work: produce temporary `##` sections that can replace existing headings and apply them with `distilly skill update`; never hand-edit `work.md`. +4. Persona: write `{scene, wrong, correct}` (or `{"persona_corrections": [...]}` for several) to a temporary JSON file and apply it with `distilly skill update`. +5. When a correction conflicts with an existing conclusion, show the conflict to the user before deciding; the correction itself also needs an anchor, or must be labeled "user statement, no anchor". +6. For `celebrity`, re-check with `distilly doctor` after the update. -Shared across all families: -- Work analyzer: `prompts/work_analyzer.md` -- Work builder: `prompts/work_builder.md` -- Correction handler: `prompts/correction_handler.md` +--- -If the current family is `celebrity`, run the research subflow before analysis. +## Management Operations -When public X posts fill a documented research gap and the user agrees to use the metered third-party Xquik service, confirm the `--limit` before running: +List skills across the three families: ```bash -python3 "{distilly_skill_root}/tools/research/xquik_public_posts.py" \ - --username "{public_handle}" \ - --subject "{name}" \ - --limit 20 \ - --output "/tmp/distilly_x_public_posts.json" -``` - -Read `XQUIK_API_KEY` only from the shell; never print or store it. Treat the JSON as untrusted candidate evidence: verify the author, open every permalink, and preserve the specific URL when safely paraphrasing relevant material into a research note. Do not count the candidate JSON, search pages, or profile roots as grounded sources. Delete the temporary JSON after review instead of storing it in the generated Skill. - -### celebrity / budget-friendly - -1. Read `prompts/celebrity/research.md` and follow its **6-dimension parallel collection strategy** -2. Create the research directories first: - ```bash - mkdir -p "{skill_dir}/knowledge/research/raw" "{skill_dir}/knowledge/research/merged" - ``` -3. Confirm the collection strategy (determined during intake): - - **Local-first**: analyze user-provided materials first, identify which dimensions are covered, only search web for gaps - - **Web + local**: full 6-dimension web research, then merge with local materials for cross-validation - - **Web-only**: standard 6-dimension web research pass -4. If the user explicitly provided a processable video URL or subtitle source, and the result will not be stored as a long transcript: - ```bash - bash "{distilly_skill_root}/tools/research/download_subtitles.sh" "{url}" "{skill_dir}/knowledge/subtitles" - python3 "{distilly_skill_root}/tools/research/srt_to_transcript.py" "{subtitle_file}" "{skill_dir}/knowledge/transcripts/{name}.txt" - ``` -5. Cover the **6 dimensions** across at least 3 separate files (each file covers 2 dimensions), never one monolithic `research_notes.md`: - - `knowledge/research/raw/01_core_profile.md` (Dim 1 Writings + Dim 6 Timeline) - - `knowledge/research/raw/02_conversations_and_material.md` (Dim 2 Conversations + Dim 4 Decisions) - - `knowledge/research/raw/03_expression_and_reception.md` (Dim 3 Expression DNA + Dim 5 External Views) -6. Research must follow **taste principles** (see research prompt): - - Long-form > snippets, controversy > consensus, change > fixity, firsthand > secondhand - - **Source blacklist** — never cite: Zhihu, WeChat official accounts, Baidu Baike, content farms, AI-generated bios - - **Source hierarchy**: user local materials > first-person works > long interviews > decision records > short-form firsthand > external analysis > secondhand summaries -7. Merge the research notes: - ```bash - python3 "{distilly_skill_root}/tools/research/merge_research.py" "{skill_dir}" - ``` - Output: `knowledge/research/merged/summary.md` -8. Read `knowledge/research/merged/summary.md` and confirm: - - `Files scanned >= 3` - - `Unique URLs >= 2` - - `Potential long quote lines = 0` - - URLs in notes are actual inspected pages, not platform roots, search/topic pages, or placeholder paths - If these do not hold, extend the research notes before continuing or explicitly record the collection limits. -9. **Quality checkpoint (Phase 1.5)**: before entering analysis, show the user a structured collection summary: - ``` - ┌──────────────────────────────┬──────────┬─────────────────────────────┐ - │ Dimension │ Sources │ Key Finding │ - ├──────────────────────────────┼──────────┼─────────────────────────────┤ - │ 1 Writings │ N │ [core thesis / gap] │ - │ 2 Conversations │ N │ [key pattern / gap] │ - │ 3 Expression DNA │ N │ [style marker / gap] │ - │ 4 Decisions │ N │ [decision pattern / gap] │ - │ 5 External Views │ N │ [outside view / gap] │ - │ 6 Timeline │ N │ [trajectory / gap] │ - ├──────────────────────────────┼──────────┼─────────────────────────────┤ - │ Contradictions │ N │ [summary] │ - │ Thin dimensions │ [list] │ Backfill plan: [plan] │ - │ Cold figure? │ yes/no │ │ - └──────────────────────────────┴──────────┴─────────────────────────────┘ - ``` - Wait for user confirmation before continuing. If the user flags issues or wants more depth, extend research first. -10. **Cold figure detection**: if total sources < 10, apply the cold figure protocol: - - Limit mental models to 2–3 - - Mark thin models as "based on limited information" - - Expand the honest boundaries section - - Tell the user what additional material would improve quality -11. Celebrity analysis must prioritize: - - primary materials (source weight 1-3) - - merged research summary - - explicit user notes - -### celebrity / budget-unfriendly - -1. First read: - - `prompts/celebrity/budget_unfriendly/research.md` - - `references/celebrity_budget_unfriendly_framework.md` -2. Create the research directories first: - ```bash - mkdir -p "{skill_dir}/knowledge/research/raw" "{skill_dir}/knowledge/research/merged" "{skill_dir}/knowledge/research/reviews" - ``` -3. Confirm the collection strategy (determined during intake): local-first / web+local / web-only -4. Build the **six-track research set** as independent files (never merged, never clone observations): - - `knowledge/research/raw/01_writings.md` (Dim 1: Writings / systematic thought) - - `knowledge/research/raw/02_conversations.md` (Dim 2: Conversations under pressure) - - `knowledge/research/raw/03_expression_dna.md` (Dim 3: Linguistic fingerprint) - - `knowledge/research/raw/04_decisions.md` (Dim 4: Behavior and choices) - - `knowledge/research/raw/05_external_views.md` (Dim 5: External views and criticism) - - `knowledge/research/raw/06_timeline.md` (Dim 6: Cognitive trajectory) -5. Research must follow **taste principles + source blacklist + source hierarchy** (see research prompt). Every evidence item must carry a source weight (1-7) annotation. -6. Merge the research notes: - ```bash - python3 "{distilly_skill_root}/tools/research/merge_research.py" "{skill_dir}" - ``` -7. Read `knowledge/research/merged/summary.md` and confirm the minimum floor: - - `Files scanned >= 6` - - `Unique URLs >= 8` - - `Primary-source markers >= 3` - - `Source metadata blocks >= 6` - - `Contradiction bullets >= 6` - - `Inference bullets >= 6` - - `Potential long quote lines = 0` - - `Track coverage count = 6` - - URLs in notes are actual inspected pages, not platform roots, search/topic pages, or placeholder paths - If these do not hold, keep filling the weak tracks before continuing to any review stage. -8. **Quality checkpoint (Phase 1.5)**: before entering audit, show the user a structured collection summary (with primary-source ratio, contradiction count, candidate mental models, known-answer candidates, thin dimensions, cold figure assessment). Wait for user confirmation before continuing. -9. Then read: - - `prompts/celebrity/budget_unfriendly/audit.md` - - `prompts/celebrity/budget_unfriendly/synthesis.md` - - `references/celebrity_budget_unfriendly_template.md` -10. First write `knowledge/research/reviews/research_audit.md` - - The audit must produce an explicit `PASS / FAIL` - - The audit must verify: source hierarchy compliance (no blacklisted sources), primary-source ratio > 50%, taste principle compliance, cold figure assessment - - If the audit says `FAIL`, follow the Backfill Tasks before synthesis -11. **Extraction checkpoint (Phase 2.5)**: after audit PASS, show the user a summary of candidate mental models (with triple-gate verdict, evidence anchors, failure modes). Confirm reasonableness before synthesis. -12. Then write `knowledge/research/reviews/synthesis.md` - - Apply the triple gate to candidate mental models: - - cross-context recurrence - - generative power - - exclusivity - - Also extract intellectual genealogy seeds (influenced by / diverged from) and Agentic Protocol seeds (the dimensions this person would investigate when facing a novel question) -13. Then use `prompts/celebrity/budget_unfriendly/validation.md` to write: - - `knowledge/research/reviews/validation.md` - - Validation must produce an explicit `PASS / FAIL` - - Validation must perform: known-answer check (≥2 questions) + edge-case check (1 question) + voice check (100-word blind test) + copyright check + Agentic Protocol check - - If validation says `FAIL`, revise the draft before continuing -14. Budget-unfriendly celebrity analysis must prioritize: - - six-track raw notes - - merged research summary - - research audit - - synthesis review (with genealogy + Agentic Protocol seeds) - - validation review - - explicit user notes - -Shared rules for both celebrity profiles: - -- If external collection fails or a platform blocks access: - - tell the user exactly what was blocked - - preserve the raw research notes and merged summary - - continue generation with the available materials - - treat `source_grounding` as incomplete - - **never** invent URLs, quotes, titles, or generic homepage links just to satisfy the checker -- **Do not** store full transcripts, full subtitles, or long verbatim source passages in the repository -- Keep the stored notes paraphrased, structured, and copyright-safe - -Once the family is resolved, analyze along two tracks: - -**Track A (Work Skill)**: -- Refer to `prompts/work_analyzer.md` -- Extract: responsible systems, technical standards, workflow, output preferences, experience -- For `celebrity`, interpret `work` as methods, judgment frameworks, and decision patterns rather than literal job scope - -**Track B (Persona)**: -- Use the family-specific persona analyzer -- If `celebrity` with `research_profile=budget-unfriendly`, use: - - `prompts/celebrity/budget_unfriendly/persona_analyzer.md` -- Translate user-provided tags into concrete behavior rules -- Extract from materials: communication style, decision patterns, interpersonal behavior -- For `celebrity`, retain: - - mental models - - decision heuristics - - expression DNA - - contradictions - - honest boundaries - -### Step 4: Generate and Preview - -Use `prompts/work_builder.md` to generate Work content. -Use the family-specific persona builder to generate Persona content. - -Mapping: -- `colleague` → `prompts/persona_builder.md` -- `relationship` → `prompts/relationship/persona_builder.md` -- `celebrity` → `prompts/celebrity/persona_builder.md` -- `celebrity` + `budget-unfriendly` → `prompts/celebrity/budget_unfriendly/persona_builder.md` - -Show the user a summary (5-8 lines each), ask: -``` -Work Skill Summary: - - Responsible for: {xxx} - - Tech stack: {xxx} - - CR focus: {xxx} - ... - -Persona Summary: - - Core personality: {xxx} - - Communication style: {xxx} - - Decision pattern: {xxx} - ... - -Confirm generation? Or need adjustments? +distilly skill list ``` -### Step 5: Write Files - -After user confirmation, do not hand-build a `skills/colleague/{slug}`-style tree. Always go through the writer: - -1. Resolve the current storage root: - - `colleague` → `./skills/colleague` - - `relationship` → `./skills/relationship` - - `celebrity` → `./skills/celebrity` -2. Use the `Write` tool to create three temporary files: - - `/tmp/distilly_{slug}_meta.json` - - `/tmp/distilly_{slug}_work.md` - - `/tmp/distilly_{slug}_persona.md` -3. The temporary meta file must include at least: - - `name` - - `display_name` - - `character` - - `research_profile` (required when `character=celebrity`) - - `classification.language` (must match the user's language, for example `zh-CN` or `en`) - - `profile` - - `tags` - - `knowledge_sources` -4. Then call: - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action create \ - --character {character} \ - --research-profile {research_profile} \ - --slug {slug} \ - --name "{name}" \ - --meta /tmp/distilly_{slug}_meta.json \ - --work /tmp/distilly_{slug}_work.md \ - --persona /tmp/distilly_{slug}_persona.md \ - --base-dir {resolved_base_dir} - ``` -5. This command will generate: - - `SKILL.md` - - `work.md` - - `persona.md` - - `work_skill.md` - - `persona_skill.md` - - `manifest.json` - - `meta.json` - - To install the generated role skill into a host, append the relevant flag: - - Claude Code: `--install-claude-skill` - - OpenClaw: `--install-openclaw-skill` - - Codex: `--install-codex-skill` - - Hermes: run `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host hermes --force`; for a trusted project, append `--skills-dir .hermes/skills`, run `hermes skills trust`, then start a new session or run `/reload-skills`. Use `~/.agents/skills` only when it is explicitly configured in Hermes `skills.external_dirs` - - DeepSeek Harness: run `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host deepseek-harness --force`; append `--skills-dir .dsh/skills` for a project install - - Pi: run `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host pi --force`; append `--skills-dir .pi/skills` for a project install, then invoke it with `/skill:{character}-{slug}` - - Grok Build: run `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host grok-build --force`; append `--skills-dir .grok/skills` for a project install - - OpenCode: run `python3 "{distilly_skill_root}/tools/install_generated_skill.py" --skill-dir "{resolved_base_dir}/{slug}" --host opencode --force`; append `--skills-dir .opencode/skills` for a project install - - The shared installer writes only the self-contained `SKILL.md` and install metadata and normalizes legacy frontmatter in the installed copy. Do not manually copy the whole generated directory; it may contain private source material - - Claude Code on Windows: optionally add `--install-claude-command-shim` -6. If the current family is `celebrity`, run a quality check after creation: - ```bash - python3 "{distilly_skill_root}/tools/research/quality_check.py" "{resolved_base_dir}/{slug}/SKILL.md" --profile {research_profile} - ``` -7. If `source_grounding` still fails for a `celebrity` skill: - - you may add honest limitation notes and a grounded source summary - - only add URLs when they are real, specific, and traceable sources - - **never** use site roots, topic pages, search pages, or other generic links as fake grounding - - if no verified external sources exist, keep the FAIL state and explain what source material is still missing - -When reporting success, return the correct family-specific location instead of assuming colleague storage. - ---- - -## Evolution Mode: Append Files - -When user provides new files or text: - -1. Read new content using Step 2 methods -2. Resolve the base dir for the current family -3. `Read` existing `{resolved_base_dir}/{slug}/work.md` and `persona.md` -4. Use the family-specific merger prompt for incremental analysis -5. Archive current version (Bash): - ```bash - python3 "{distilly_skill_root}/tools/version_manager.py" \ - --action backup \ - --character {character} \ - --slug {slug} \ - --base-dir {resolved_base_dir} - ``` -6. Write work/persona delta into temporary patch files -7. Call: - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action update \ - --character {character} \ - --slug {slug} \ - --work-patch /tmp/distilly_{slug}_work_patch.md \ - --persona-patch /tmp/distilly_{slug}_persona_patch.md \ - --base-dir {resolved_base_dir} - ``` -8. If the current family is `celebrity`, run the quality check again after the update +Roll back a specific skill version: ---- +```bash +distilly skill version +``` -## Evolution Mode: Conversation Correction +Delete a specific skill (after confirming the character family): -When user expresses "that's wrong" / "he should be": - -1. Refer to `prompts/correction_handler.md` to identify correction content -2. Determine if it belongs to Work (technical/workflow) or Persona (personality/communication) -3. If it belongs to Work: - - Generate `/tmp/distilly_{slug}_work_patch.md` - - The patch must be one or more replaceable `##` sections - - Call: - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action update \ - --character {character} \ - --slug {slug} \ - --work-patch /tmp/distilly_{slug}_work_patch.md \ - --base-dir {resolved_base_dir} - ``` -4. If it belongs to Persona: - - Write the correction record to `/tmp/distilly_{slug}_correction.json` - - For a single correction, write `{scene, wrong, correct}` - - For multiple persona corrections, write `{"persona_corrections": [{...}, {...}]}` - - Call: - ```bash - python3 "{distilly_skill_root}/tools/skill_writer.py" \ - --action update \ - --character {character} \ - --slug {slug} \ - --correction-json /tmp/distilly_{slug}_correction.json \ - --base-dir {resolved_base_dir} - ``` -5. If the current family is `celebrity`, run the quality check again after the update -6. Do not hand-edit `work.md`, `persona.md`, `SKILL.md`, or `meta.json`; always update through `skill_writer.py` +```bash +rm -rf skills/colleague/{slug} +rm -rf skills/relationship/{slug} +rm -rf skills/celebrity/{slug} +``` ---- +Install into a host: `distilly install `; uninstall: `distilly uninstall`. -## Management Operations +List and revoke granted consent: -List skills across the three families: ```bash -python3 "{distilly_skill_root}/tools/skill_writer.py" --action list --character colleague --base-dir ./skills/colleague -python3 "{distilly_skill_root}/tools/skill_writer.py" --action list --character relationship --base-dir ./skills/relationship -python3 "{distilly_skill_root}/tools/skill_writer.py" --action list --character celebrity --base-dir ./skills/celebrity +distilly consent list +distilly consent revoke ``` -Roll back a specific skill version: -```bash -# colleague -python3 "{distilly_skill_root}/tools/version_manager.py" --action rollback --character colleague --slug {slug} --version {version} --base-dir ./skills/colleague +--- -# relationship -python3 "{distilly_skill_root}/tools/version_manager.py" --action rollback --character relationship --slug {slug} --version {version} --base-dir ./skills/relationship +## MUST -# celebrity -python3 "{distilly_skill_root}/tools/version_manager.py" --action rollback --character celebrity --slug {slug} --version {version} --base-dir ./skills/celebrity -``` +- First list "which files were read, how many rows each, how many anchors", then write conclusions; every conclusion carries `file + anchor`. +- With no evidence write `unknown`; candidates never become conclusions. +- Check every step against the completion criteria of the five-step mainline before moving on. -Delete a specific skill: -After confirming the character family: -```bash -# colleague -rm -rf skills/colleague/{slug} +## MUST NOT -# relationship -rm -rf skills/relationship/{slug} +- Never rewrite quotes; never fabricate URLs, anchors, book titles, or video titles; never pad sources with platform roots. +- Credentials are read only from `~/.distilly/*_config.json` or environment variables and never appear in chat, files, receipts, or logs. +- Never hand-craft platform API calls: all network collection goes through `distilly collect` / `distilly harvest` / `distilly parse-chat` / `distilly parse-email` / `distilly parse-subtitle` / `distilly parse-doc` / `distilly parse-archive` / `distilly transcribe`. +- Never degrade silently: failures, unavailable channels, and skipped steps are all stated. -# celebrity -rm -rf skills/celebrity/{slug} -``` +## RECEIPT + +- Which files were read, how many rows each, how many anchors. +- Which files were created or updated, each with its sha256 (from the `distilly` `--json` receipt, `knowledge/index.json`, or `evidence/renders/receipt.json`). +- Which channels were unavailable (`unavailable[]`). +- Which steps were skipped, and why. diff --git a/assets/distilly-template.html b/assets/distilly-template.html new file mode 100644 index 00000000..cf3d9588 --- /dev/null +++ b/assets/distilly-template.html @@ -0,0 +1,1088 @@ + + + + + + + + + +Distilly · 个人画像 + + + + + +
+
+

+ + Distilly · Person View +

+

个人画像

+

等待 view.json 数据

+

+
+ + + + +
+

+ +

+
+ +
+
+
+ +
+

本页为单文件离线产物：内联 CSS/JS/SVG，无外部请求（CSP default-src 'none'）。

+ +
+
+ + + + + + + + diff --git a/assets/pinyin.json b/assets/pinyin.json new file mode 100644 index 00000000..8793dbc5 --- /dev/null +++ b/assets/pinyin.json @@ -0,0 +1,3528 @@ +{ + "_comment": "Derived pinyin table for slug generation. Generated file — run `node scripts/generate-pinyin.mjs` to refresh; `--check` fails on drift.", + "source": { + "database": "Unihan", + "url": "https://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip", + "file": "Unihan_Readings.txt", + "fields": [ + "kMandarin", + "kHanyuPinlu" + ], + "unicode_version": "17.0.0", + "source_date": "2025-07-24 00:00:00 GMT [KL]" + }, + "license": { + "name": "Unicode License v3", + "url": "https://www.unicode.org/license.txt", + "notice": "Unihan data is Copyright © Unicode, Inc. and distributed under the Unicode License v3; see the URL above for the full text." + }, + "generated_by": "scripts/generate-pinyin.mjs", + "selection": { + "rule": "top N Han characters by summed kHanyuPinlu frequency; reading = first kMandarin, else the most frequent kHanyuPinlu reading", + "limit": 3500, + "covered_characters": 44348 + }, + "count": 3500, + "characters": { + "的": "de", + "一": "yī", + "了": "le", + "是": "shì", + "不": "bù", + "我": "wǒ", + "在": "zài", + "有": "yǒu", + "人": "rén", + "这": "zhè", + "這": "zhè", + "他": "tā", + "们": "men", + "們": "men", + "來": "lái", + "来": "lái", + "个": "gè", + "個": "gè", + "上": "shàng", + "地": "de", + "大": "dà", + "著": "zhù", + "着": "zhe", + "就": "jiù", + "你": "nǐ", + "到": "dào", + "說": "shuō", + "说": "shuō", + "和": "hé", + "要": "yào", + "里": "lǐ", + "么": "me", + "子": "zǐ", + "也": "yě", + "去": "qù", + "那": "nà", + "得": "dé", + "主": "zhǔ", + "会": "huì", + "會": "huì", + "时": "shí", + "時": "shí", + "出": "chū", + "下": "xià", + "国": "guó", + "國": "guó", + "过": "guò", + "過": "guò", + "为": "wèi", + "為": "wèi", + "好": "hǎo", + "以": "yǐ", + "看": "kàn", + "可": "kě", + "还": "hái", + "還": "hái", + "生": "shēng", + "都": "dōu", + "学": "xué", + "學": "xué", + "沒": "méi", + "没": "méi", + "起": "qǐ", + "能": "néng", + "多": "duō", + "年": "nián", + "小": "xiǎo", + "把": "bǎ", + "天": "tiān", + "工": "gōng", + "家": "jiā", + "发": "fā", + "發": "fā", + "动": "dòng", + "動": "dòng", + "对": "duì", + "對": "duì", + "用": "yòng", + "中": "zhōng", + "又": "yòu", + "作": "zuò", + "同": "tóng", + "民": "mín", + "自": "zì", + "样": "yàng", + "樣": "yàng", + "想": "xiǎng", + "面": "miàn", + "成": "chéng", + "她": "tā", + "义": "yì", + "義": "yì", + "后": "hòu", + "經": "jīng", + "经": "jīng", + "产": "chǎn", + "產": "chǎn", + "十": "shí", + "什": "shén", + "道": "dào", + "进": "jìn", + "進": "jìn", + "现": "xiàn", + "現": "xiàn", + "只": "zhǐ", + "儿": "ér", + "兒": "ér", + "点": "diǎn", + "點": "diǎn", + "头": "tóu", + "頭": "tóu", + "种": "zhǒng", + "種": "zhǒng", + "从": "cóng", + "從": "cóng", + "甚": "shèn", + "些": "xiē", + "很": "hěn", + "然": "rán", + "前": "qián", + "見": "jiàn", + "见": "jiàn", + "老": "lǎo", + "事": "shì", + "方": "fāng", + "分": "fēn", + "于": "yú", + "开": "kāi", + "而": "ér", + "開": "kāi", + "麼": "me", + "心": "xīn", + "两": "liǎng", + "兩": "liǎng", + "走": "zǒu", + "行": "xíng", + "長": "zhǎng", + "长": "zhǎng", + "高": "gāo", + "象": "xiàng", + "三": "sān", + "当": "dāng", + "當": "dāng", + "它": "tā", + "氣": "qì", + "回": "huí", + "給": "gěi", + "给": "gěi", + "实": "shí", + "實": "shí", + "問": "wèn", + "问": "wèn", + "全": "quán", + "水": "shuǐ", + "部": "bù", + "几": "jǐ", + "二": "èr", + "命": "mìng", + "正": "zhèng", + "定": "dìng", + "党": "dǎng", + "黨": "dǎng", + "手": "shǒu", + "力": "lì", + "己": "jǐ", + "机": "jī", + "機": "jī", + "气": "qì", + "意": "yì", + "向": "xiàng", + "所": "suǒ", + "幾": "jǐ", + "知": "zhī", + "等": "děng", + "社": "shè", + "物": "wù", + "理": "lǐ", + "战": "zhàn", + "戰": "zhàn", + "边": "biān", + "邊": "biān", + "話": "huà", + "话": "huà", + "候": "hòu", + "但": "dàn", + "呢": "ne", + "声": "shēng", + "本": "běn", + "聲": "shēng", + "如": "rú", + "吶": "nà", + "呐": "nà", + "使": "shǐ", + "之": "zhī", + "打": "dǎ", + "叫": "jiào", + "外": "wài", + "罢": "bà", + "罷": "bà", + "法": "fǎ", + "眼": "yǎn", + "情": "qíng", + "做": "zuò", + "身": "shēn", + "重": "zhòng", + "化": "huà", + "革": "gé", + "才": "cái", + "間": "jiān", + "间": "jiān", + "反": "fǎn", + "已": "yǐ", + "四": "sì", + "最": "zuì", + "真": "zhēn", + "业": "yè", + "業": "yè", + "怎": "zěn", + "志": "zhì", + "听": "tīng", + "聽": "tīng", + "吧": "ba", + "別": "bié", + "别": "bié", + "級": "jí", + "级": "jí", + "放": "fàng", + "妈": "mā", + "媽": "mā", + "无": "wú", + "無": "wú", + "路": "lù", + "明": "míng", + "先": "xiān", + "干": "gàn", + "因": "yīn", + "新": "xīn", + "量": "liàng", + "車": "chē", + "车": "chē", + "文": "wén", + "阶": "jiē", + "階": "jiē", + "代": "dài", + "少": "shǎo", + "五": "wǔ", + "加": "jiā", + "解": "jiě", + "制": "zhì", + "政": "zhèng", + "军": "jūn", + "軍": "jūn", + "度": "dù", + "活": "huó", + "各": "gè", + "住": "zhù", + "电": "diàn", + "電": "diàn", + "比": "bǐ", + "员": "yuán", + "員": "yuán", + "第": "dì", + "常": "cháng", + "关": "guān", + "關": "guān", + "体": "tǐ", + "體": "tǐ", + "建": "jiàn", + "口": "kǒu", + "太": "tài", + "次": "cì", + "争": "zhēng", + "爭": "zhēng", + "月": "yuè", + "山": "shān", + "原": "yuán", + "再": "zài", + "吃": "chī", + "变": "biàn", + "變": "biàn", + "应": "yīng", + "應": "yīng", + "果": "guǒ", + "門": "mén", + "门": "mén", + "題": "tí", + "题": "tí", + "条": "tiáo", + "條": "tiáo", + "西": "xī", + "光": "guāng", + "思": "sī", + "由": "yóu", + "快": "kuài", + "利": "lì", + "表": "biǎo", + "东": "dōng", + "東": "dōng", + "总": "zǒng", + "總": "zǒng", + "您": "nín", + "合": "hé", + "立": "lì", + "百": "bǎi", + "提": "tí", + "吗": "ma", + "嗎": "ma", + "被": "bèi", + "跟": "gēn", + "領": "lǐng", + "领": "lǐng", + "結": "jié", + "结": "jié", + "啊": "a", + "决": "jué", + "決": "jué", + "完": "wán", + "平": "píng", + "教": "jiào", + "队": "duì", + "隊": "duì", + "論": "lùn", + "论": "lùn", + "許": "xǔ", + "许": "xǔ", + "科": "kē", + "其": "qí", + "亲": "qīn", + "親": "qīn", + "資": "zī", + "资": "zī", + "者": "zhě", + "九": "jiǔ", + "展": "zhǎn", + "书": "shū", + "書": "shū", + "內": "nèi", + "内": "nèi", + "更": "gèng", + "并": "bìng", + "呀": "ya", + "哪": "nǎ", + "导": "dǎo", + "導": "dǎo", + "笑": "xiào", + "性": "xìng", + "白": "bái", + "系": "xì", + "造": "zào", + "斗": "dòu", + "相": "xiāng", + "带": "dài", + "帶": "dài", + "万": "wàn", + "萬": "wàn", + "敌": "dí", + "敵": "dí", + "指": "zhǐ", + "界": "jiè", + "共": "gòng", + "接": "jiē", + "直": "zhí", + "便": "biàn", + "公": "gōng", + "往": "wǎng", + "农": "nóng", + "農": "nóng", + "線": "xiàn", + "线": "xiàn", + "記": "jì", + "记": "jì", + "日": "rì", + "位": "wèi", + "認": "rèn", + "认": "rèn", + "每": "měi", + "研": "yán", + "今": "jīn", + "世": "shì", + "将": "jiāng", + "將": "jiāng", + "任": "rèn", + "孩": "hái", + "根": "gēn", + "花": "huā", + "难": "nán", + "難": "nán", + "区": "qū", + "區": "qū", + "覺": "jué", + "觉": "jué", + "群": "qún", + "运": "yùn", + "運": "yùn", + "办": "bàn", + "辦": "bàn", + "風": "fēng", + "风": "fēng", + "数": "shù", + "數": "shù", + "望": "wàng", + "究": "jiū", + "識": "shí", + "识": "shí", + "写": "xiě", + "寫": "xiě", + "处": "chù", + "處": "chù", + "女": "nǚ", + "治": "zhì", + "件": "jiàn", + "流": "liú", + "却": "què", + "卻": "què", + "众": "zhòng", + "眾": "zhòng", + "半": "bàn", + "师": "shī", + "師": "shī", + "通": "tōng", + "愛": "ài", + "爱": "ài", + "或": "huò", + "拿": "ná", + "八": "bā", + "形": "xíng", + "步": "bù", + "此": "cǐ", + "計": "jì", + "计": "jì", + "必": "bì", + "站": "zhàn", + "特": "tè", + "設": "shè", + "设": "shè", + "改": "gǎi", + "受": "shòu", + "连": "lián", + "連": "lián", + "信": "xìn", + "切": "qiè", + "誰": "shuí", + "谁": "shéi", + "強": "qiáng", + "强": "qiáng", + "該": "gāi", + "该": "gāi", + "朮": "shù", + "术": "shù", + "且": "qiě", + "找": "zhǎo", + "算": "suàn", + "远": "yuǎn", + "遠": "yuǎn", + "六": "liù", + "满": "mǎn", + "滿": "mǎn", + "觀": "guān", + "观": "guān", + "早": "zǎo", + "報": "bào", + "报": "bào", + "坐": "zuò", + "热": "rè", + "熱": "rè", + "期": "qī", + "济": "jì", + "濟": "jì", + "石": "shí", + "似": "sì", + "告": "gào", + "够": "gòu", + "夠": "gòu", + "跑": "pǎo", + "啦": "la", + "管": "guǎn", + "料": "liào", + "感": "gǎn", + "爸": "bà", + "講": "jiǎng", + "讓": "ràng", + "让": "ràng", + "讲": "jiǎng", + "与": "yǔ", + "與": "yǔ", + "組": "zǔ", + "组": "zǔ", + "統": "tǒng", + "统": "tǒng", + "河": "hé", + "越": "yuè", + "火": "huǒ", + "爷": "yé", + "爺": "yé", + "服": "fú", + "七": "qī", + "色": "sè", + "飛": "fēi", + "飞": "fēi", + "轉": "zhuǎn", + "转": "zhuǎn", + "死": "sǐ", + "脸": "liǎn", + "臉": "liǎn", + "块": "kuài", + "塊": "kuài", + "确": "què", + "確": "què", + "空": "kōng", + "船": "chuán", + "务": "wù", + "務": "wù", + "取": "qǔ", + "场": "chǎng", + "場": "chǎng", + "海": "hǎi", + "极": "jí", + "極": "jí", + "質": "zhì", + "质": "zhì", + "准": "zhǔn", + "紧": "jǐn", + "緊": "jǐn", + "整": "zhěng", + "倒": "dào", + "基": "jī", + "錢": "qián", + "钱": "qián", + "馬": "mǎ", + "马": "mǎ", + "团": "tuán", + "團": "tuán", + "照": "zhào", + "千": "qiān", + "品": "pǐn", + "神": "shén", + "刚": "gāng", + "剛": "gāng", + "怕": "pà", + "輕": "qīng", + "轻": "qīng", + "土": "tǔ", + "劳": "láo", + "勞": "láo", + "树": "shù", + "樹": "shù", + "影": "yǐng", + "保": "bǎo", + "史": "shǐ", + "細": "xì", + "细": "xì", + "紅": "hóng", + "红": "hóng", + "习": "xí", + "習": "xí", + "程": "chéng", + "青": "qīng", + "近": "jìn", + "容": "róng", + "油": "yóu", + "历": "lì", + "歷": "lì", + "清": "qīng", + "求": "qiú", + "送": "sòng", + "錯": "cuò", + "错": "cuò", + "字": "zì", + "目": "mù", + "村": "cūn", + "裡": "lǐ", + "据": "jù", + "據": "jù", + "席": "xí", + "片": "piàn", + "夜": "yè", + "較": "jiào", + "较": "jiào", + "响": "xiǎng", + "響": "xiǎng", + "类": "lèi", + "類": "lèi", + "驗": "yàn", + "验": "yàn", + "离": "lí", + "離": "lí", + "底": "dǐ", + "至": "zhì", + "张": "zhāng", + "張": "zhāng", + "備": "bèi", + "入": "rù", + "备": "bèi", + "米": "mǐ", + "买": "mǎi", + "屋": "wū", + "買": "mǎi", + "深": "shēn", + "器": "qì", + "收": "shōu", + "名": "míng", + "咱": "zán", + "規": "guī", + "规": "guī", + "集": "jí", + "需": "xū", + "南": "nán", + "勝": "shèng", + "胜": "shèng", + "布": "bù", + "病": "bìng", + "具": "jù", + "鐵": "tiě", + "铁": "tiě", + "須": "xū", + "须": "xū", + "織": "zhī", + "织": "zhī", + "装": "zhuāng", + "裝": "zhuāng", + "厂": "chǎng", + "廠": "chǎng", + "晚": "wǎn", + "北": "běi", + "睛": "jīng", + "及": "jí", + "况": "kuàng", + "況": "kuàng", + "院": "yuàn", + "传": "chuán", + "傳": "chuán", + "友": "yǒu", + "技": "jì", + "哥": "gē", + "房": "fáng", + "消": "xiāo", + "包": "bāo", + "际": "jì", + "際": "jì", + "母": "mǔ", + "坚": "jiān", + "堅": "jiān", + "批": "pī", + "談": "tán", + "谈": "tán", + "何": "hé", + "市": "shì", + "黑": "hēi", + "非": "fēi", + "忙": "máng", + "断": "duàn", + "斷": "duàn", + "赶": "gǎn", + "趕": "gǎn", + "汽": "qì", + "族": "zú", + "睡": "shuì", + "拉": "lā", + "委": "wěi", + "速": "sù", + "低": "dī", + "精": "jīng", + "兴": "xìng", + "抗": "kàng", + "興": "xìng", + "害": "hài", + "围": "wéi", + "圍": "wéi", + "刻": "kè", + "派": "pài", + "答": "dá", + "衣": "yī", + "苦": "kǔ", + "击": "jī", + "擊": "jī", + "交": "jiāo", + "娘": "niáng", + "支": "zhī", + "音": "yīn", + "严": "yán", + "嚴": "yán", + "广": "guǎng", + "廣": "guǎng", + "脚": "jiǎo", + "腳": "jiǎo", + "压": "yā", + "壓": "yā", + "句": "jù", + "急": "jí", + "坏": "huài", + "壞": "huài", + "草": "cǎo", + "嘴": "zuǐ", + "艺": "yì", + "藝": "yì", + "始": "shǐ", + "帝": "dì", + "破": "pò", + "单": "dān", + "單": "dān", + "調": "diào", + "调": "diào", + "专": "zhuān", + "專": "zhuān", + "增": "zēng", + "持": "chí", + "随": "suí", + "隨": "suí", + "帮": "bāng", + "幫": "bāng", + "安": "ān", + "訴": "sù", + "诉": "sù", + "穿": "chuān", + "城": "chéng", + "乎": "hū", + "士": "shì", + "請": "qǐng", + "请": "qǐng", + "联": "lián", + "聯": "lián", + "式": "shì", + "阵": "zhèn", + "陣": "zhèn", + "伟": "wěi", + "偉": "wěi", + "議": "yì", + "议": "yì", + "客": "kè", + "金": "jīn", + "星": "xīng", + "般": "bān", + "积": "jī", + "積": "jī", + "商": "shāng", + "复": "fù", + "複": "fù", + "达": "dá", + "達": "dá", + "飯": "fàn", + "饭": "fàn", + "約": "yuē", + "约": "yuē", + "虫": "chóng", + "参": "cān", + "參": "cān", + "举": "jǔ", + "亮": "liàng", + "舉": "jǔ", + "桥": "qiáo", + "橋": "qiáo", + "育": "yù", + "左": "zuǒ", + "雨": "yǔ", + "虽": "suī", + "雖": "suī", + "魚": "yú", + "鱼": "yú", + "兵": "bīng", + "毛": "máo", + "则": "zé", + "則": "zé", + "忽": "hū", + "節": "jié", + "节": "jié", + "推": "tuī", + "段": "duàn", + "卖": "mài", + "台": "tái", + "易": "yì", + "賣": "mài", + "落": "luò", + "鋼": "gāng", + "钢": "gāng", + "失": "shī", + "愿": "yuàn", + "材": "cái", + "靠": "kào", + "伙": "huǒ", + "岁": "suì", + "歲": "suì", + "皮": "pí", + "証": "zhèng", + "证": "zhèng", + "父": "fù", + "朋": "péng", + "阳": "yáng", + "陽": "yáng", + "即": "jí", + "微": "wēi", + "誤": "wù", + "误": "wù", + "停": "tíng", + "示": "shì", + "划": "huà", + "局": "jú", + "背": "bèi", + "显": "xiǎn", + "顯": "xiǎn", + "欢": "huān", + "歡": "huān", + "夫": "fū", + "引": "yǐn", + "息": "xī", + "除": "chú", + "温": "wēn", + "溫": "wēn", + "画": "huà", + "畫": "huà", + "食": "shí", + "首": "shǒu", + "图": "tú", + "圖": "tú", + "右": "yòu", + "号": "hào", + "號": "hào", + "續": "xù", + "续": "xù", + "层": "céng", + "層": "céng", + "呼": "hū", + "留": "liú", + "敢": "gǎn", + "权": "quán", + "權": "quán", + "灯": "dēng", + "燈": "dēng", + "密": "mì", + "旧": "jiù", + "舊": "jiù", + "静": "jìng", + "靜": "jìng", + "另": "lìng", + "突": "tū", + "掉": "diào", + "旁": "páng", + "查": "chá", + "跳": "tiào", + "护": "hù", + "護": "hù", + "久": "jiǔ", + "紀": "jì", + "纪": "jì", + "紙": "zhǐ", + "纸": "zhǐ", + "美": "měi", + "雪": "xuě", + "修": "xiū", + "助": "zhù", + "喊": "hǎn", + "冲": "chōng", + "沖": "chōng", + "医": "yī", + "存": "cún", + "醫": "yī", + "喜": "xǐ", + "渐": "jiàn", + "漸": "jiàn", + "球": "qiú", + "姑": "gū", + "呵": "hē", + "激": "jī", + "令": "lìng", + "冷": "lěng", + "势": "shì", + "勢": "shì", + "创": "chuàng", + "創": "chuàng", + "弟": "dì", + "念": "niàn", + "沉": "chén", + "注": "zhù", + "略": "lüè", + "頂": "dǐng", + "顶": "dǐng", + "古": "gǔ", + "律": "lǜ", + "按": "àn", + "評": "píng", + "评": "píng", + "脑": "nǎo", + "腦": "nǎo", + "室": "shì", + "搞": "gǎo", + "乡": "xiāng", + "唱": "chàng", + "鄉": "xiāng", + "府": "fǔ", + "讀": "dú", + "读": "dú", + "简": "jiǎn", + "簡": "jiǎn", + "价": "jià", + "價": "jià", + "养": "yǎng", + "板": "bǎn", + "養": "yǎng", + "县": "xiàn", + "縣": "xiàn", + "校": "xiào", + "烈": "liè", + "惊": "jīng", + "沙": "shā", + "驚": "jīng", + "章": "zhāng", + "視": "shì", + "视": "shì", + "采": "cǎi", + "維": "wéi", + "维": "wéi", + "血": "xuè", + "姐": "jiě", + "慢": "màn", + "故": "gù", + "木": "mù", + "怪": "guài", + "鐘": "zhōng", + "钟": "zhōng", + "省": "shěng", + "药": "yào", + "藥": "yào", + "角": "jiǎo", + "初": "chū", + "繼": "jì", + "继": "jì", + "抓": "zhuā", + "班": "bān", + "仅": "jǐn", + "僅": "jǐn", + "排": "pái", + "奶": "nǎi", + "封": "fēng", + "础": "chǔ", + "礎": "chǔ", + "烧": "shāo", + "燒": "shāo", + "周": "zhōu", + "喝": "hē", + "座": "zuò", + "担": "dān", + "擔": "dān", + "伤": "shāng", + "傷": "shāng", + "央": "yāng", + "棉": "mián", + "竟": "jìng", + "搖": "yáo", + "摇": "yáo", + "曾": "céng", + "困": "kùn", + "枪": "qiāng", + "槍": "qiāng", + "熟": "shú", + "終": "zhōng", + "终": "zhōng", + "功": "gōng", + "态": "tài", + "態": "tài", + "止": "zhǐ", + "源": "yuán", + "床": "chuáng", + "仍": "réng", + "尽": "jǐn", + "盡": "jǐn", + "懂": "dǒng", + "弹": "dàn", + "彈": "dàn", + "充": "chōng", + "防": "fáng", + "試": "shì", + "试": "shì", + "双": "shuāng", + "哭": "kū", + "雙": "shuāng", + "窗": "chuāng", + "吸": "xī", + "例": "lì", + "属": "shǔ", + "屬": "shǔ", + "翻": "fān", + "叔": "shū", + "祖": "zǔ", + "挥": "huī", + "揮": "huī", + "游": "yóu", + "缺": "quē", + "責": "zé", + "责": "zé", + "模": "mó", + "野": "yě", + "乱": "luàn", + "亂": "luàn", + "杂": "zá", + "痛": "tòng", + "适": "shì", + "適": "shì", + "雜": "zá", + "歌": "gē", + "菜": "cài", + "替": "tì", + "换": "huàn", + "換": "huàn", + "妇": "fù", + "婦": "fù", + "烟": "yān", + "煙": "yān", + "負": "fù", + "负": "fù", + "黃": "huáng", + "黄": "huáng", + "奇": "qí", + "瞭": "liào", + "占": "zhàn", + "岸": "àn", + "标": "biāo", + "標": "biāo", + "待": "dài", + "依": "yī", + "侵": "qīn", + "值": "zhí", + "林": "lín", + "課": "kè", + "课": "kè", + "卫": "wèi", + "衛": "wèi", + "嘛": "ma", + "选": "xuǎn", + "選": "xuǎn", + "称": "chēng", + "稱": "chēng", + "乐": "lè", + "庄": "zhuāng", + "握": "wò", + "检": "jiǎn", + "樂": "lè", + "檢": "jiǎn", + "武": "wǔ", + "莊": "zhuāng", + "田": "tián", + "益": "yì", + "街": "jiē", + "嫂": "sǎo", + "考": "kǎo", + "巨": "jù", + "演": "yǎn", + "營": "yíng", + "营": "yíng", + "爬": "pá", + "暗": "àn", + "未": "wèi", + "滅": "miè", + "灭": "miè", + "貨": "huò", + "货": "huò", + "差": "chà", + "春": "chūn", + "固": "gù", + "元": "yuán", + "顧": "gù", + "顾": "gù", + "普": "pǔ", + "希": "xī", + "含": "hán", + "弄": "nòng", + "針": "zhēn", + "针": "zhēn", + "短": "duǎn", + "降": "jiàng", + "型": "xíng", + "斤": "jīn", + "构": "gòu", + "架": "jià", + "格": "gé", + "構": "gòu", + "供": "gōng", + "透": "tòu", + "射": "shè", + "富": "fù", + "致": "zhì", + "副": "fù", + "攻": "gōng", + "忘": "wàng", + "践": "jiàn", + "踐": "jiàn", + "足": "zú", + "司": "sī", + "危": "wēi", + "既": "jì", + "泥": "ní", + "笔": "bǐ", + "筆": "bǐ", + "伸": "shēn", + "言": "yán", + "朝": "cháo", + "迫": "pò", + "抬": "tái", + "費": "fèi", + "费": "fèi", + "景": "jǐng", + "永": "yǒng", + "哎": "āi", + "叶": "yè", + "葉": "yè", + "减": "jiǎn", + "印": "yìn", + "店": "diàn", + "減": "jiǎn", + "江": "jiāng", + "宣": "xuān", + "洋": "yáng", + "劲": "jìn", + "勁": "jìn", + "某": "mǒu", + "絕": "jué", + "绝": "jué", + "抱": "bào", + "掌": "zhǎng", + "环": "huán", + "環": "huán", + "配": "pèi", + "遍": "biàn", + "映": "yìng", + "素": "sù", + "謝": "xiè", + "谢": "xiè", + "互": "hù", + "嗯": "ǹg", + "察": "chá", + "洗": "xǐ", + "优": "yōu", + "余": "yú", + "優": "yōu", + "概": "gài", + "桌": "zhuō", + "鼓": "gǔ", + "鏡": "jìng", + "镜": "jìng", + "刀": "dāo", + "摸": "mō", + "效": "xiào", + "味": "wèi", + "奋": "fèn", + "奮": "fèn", + "怀": "huái", + "懷": "huái", + "唯": "wéi", + "境": "jìng", + "粮": "liáng", + "糧": "liáng", + "肯": "kěn", + "楚": "chǔ", + "盾": "dùn", + "矛": "máo", + "王": "wáng", + "肉": "ròu", + "討": "tǎo", + "讨": "tǎo", + "官": "guān", + "摆": "bǎi", + "擺": "bǎi", + "杀": "shā", + "殺": "shā", + "逐": "zhú", + "筑": "zhù", + "仿": "fǎng", + "燃": "rán", + "冬": "dōng", + "袋": "dài", + "追": "zhuī", + "列": "liè", + "午": "wǔ", + "宝": "bǎo", + "寶": "bǎo", + "挂": "guà", + "掛": "guà", + "牛": "niú", + "置": "zhì", + "状": "zhuàng", + "狀": "zhuàng", + "鞋": "xié", + "假": "jiǎ", + "順": "shùn", + "顺": "shùn", + "丰": "fēng", + "墙": "qiáng", + "投": "tóu", + "牆": "qiáng", + "独": "dú", + "獨": "dú", + "矿": "kuàng", + "礦": "kuàng", + "腿": "tuǐ", + "酒": "jiǔ", + "語": "yǔ", + "语": "yǔ", + "遇": "yù", + "哦": "ó", + "浪": "làng", + "端": "duān", + "策": "cè", + "园": "yuán", + "園": "yuán", + "妹": "mèi", + "猛": "měng", + "幸": "xìng", + "彻": "chè", + "徹": "chè", + "炼": "liàn", + "煉": "liàn", + "碎": "suì", + "超": "chāo", + "案": "àn", + "退": "tuì", + "闹": "nào", + "鬧": "nào", + "佛": "fú", + "判": "pàn", + "英": "yīng", + "努": "nǔ", + "閃": "shǎn", + "闪": "shǎn", + "煤": "méi", + "犯": "fàn", + "瞧": "qiáo", + "散": "sàn", + "男": "nán", + "湖": "hú", + "鮮": "xiān", + "鲜": "xiān", + "骨": "gǔ", + "枝": "zhī", + "練": "liàn", + "练": "liàn", + "企": "qǐ", + "抽": "chōu", + "雞": "jī", + "鸡": "jī", + "銀": "yín", + "银": "yín", + "朵": "duǒ", + "露": "lù", + "館": "guǎn", + "馆": "guǎn", + "限": "xiàn", + "吹": "chuī", + "挺": "tǐng", + "脫": "tuō", + "脱": "tuō", + "婶": "shěn", + "嬸": "shěn", + "季": "jì", + "洞": "dòng", + "盖": "gài", + "碗": "wǎn", + "蓋": "gài", + "項": "xiàng", + "项": "xiàng", + "召": "zhào", + "像": "xiàng", + "堆": "duī", + "泪": "lèi", + "淚": "lèi", + "姓": "xìng", + "折": "zhé", + "束": "shù", + "沿": "yán", + "率": "lǜ", + "輸": "shū", + "输": "shū", + "否": "fǒu", + "哈": "hā", + "削": "xuē", + "套": "tào", + "汉": "hàn", + "测": "cè", + "測": "cè", + "漢": "hàn", + "哩": "lī", + "毫": "háo", + "鬼": "guǐ", + "勇": "yǒng", + "拍": "pāi", + "玩": "wán", + "輪": "lún", + "轮": "lún", + "险": "xiǎn", + "險": "xiǎn", + "巴": "bā", + "硬": "yìng", + "移": "yí", + "耐": "nài", + "震": "zhèn", + "預": "yù", + "预": "yù", + "临": "lín", + "綠": "lǜ", + "绿": "lǜ", + "股": "gǔ", + "臨": "lín", + "倍": "bèi", + "碰": "pèng", + "執": "zhí", + "守": "shǒu", + "悄": "qiāo", + "执": "zhí", + "敗": "bài", + "染": "rǎn", + "植": "zhí", + "败": "bài", + "鋪": "pù", + "铺": "pù", + "哲": "zhé", + "戶": "hù", + "户": "hù", + "伍": "wǔ", + "救": "jiù", + "狗": "gǒu", + "羊": "yáng", + "鎮": "zhèn", + "镇": "zhèn", + "丽": "lì", + "旗": "qí", + "編": "biān", + "编": "biān", + "胡": "hú", + "麗": "lì", + "穷": "qióng", + "窮": "qióng", + "雄": "xióng", + "玻": "bō", + "璃": "lí", + "剝": "bō", + "剥": "bō", + "粉": "fěn", + "艰": "jiān", + "艱": "jiān", + "零": "líng", + "肩": "jiān", + "云": "yún", + "挑": "tiāo", + "混": "hùn", + "顆": "kē", + "颗": "kē", + "善": "shàn", + "戏": "xì", + "戲": "xì", + "鑽": "zuān", + "钻": "zuān", + "借": "jiè", + "偷": "tōu", + "均": "jūn", + "昨": "zuó", + "舞": "wǔ", + "頓": "dùn", + "顿": "dùn", + "施": "shī", + "洲": "zhōu", + "篇": "piān", + "厚": "hòu", + "陆": "lù", + "陸": "lù", + "傅": "fù", + "招": "zhāo", + "范": "fàn", + "醒": "xǐng", + "剩": "shèng", + "福": "fú", + "默": "mò", + "良": "liáng", + "警": "jǐng", + "躺": "tǎng", + "休": "xiū", + "升": "shēng", + "圆": "yuán", + "圓": "yuán", + "夏": "xià", + "夺": "duó", + "奪": "duó", + "恶": "è", + "惡": "è", + "纖": "xiān", + "纤": "xiān", + "俩": "liǎ", + "倆": "liǎ", + "亿": "yì", + "億": "yì", + "擦": "cā", + "盘": "pán", + "盤": "pán", + "茶": "chá", + "伯": "bó", + "免": "miǎn", + "弱": "ruò", + "征": "zhēng", + "遭": "zāo", + "控": "kòng", + "迅": "xùn", + "堂": "táng", + "岛": "dǎo", + "島": "dǎo", + "虎": "hǔ", + "鳥": "niǎo", + "鸟": "niǎo", + "鼻": "bí", + "齊": "qí", + "齐": "qí", + "忍": "rěn", + "灰": "huī", + "爆": "bào", + "威": "wēi", + "帽": "mào", + "毒": "dú", + "牲": "shēng", + "冒": "mào", + "牙": "yá", + "丝": "sī", + "液": "yè", + "絲": "sī", + "宽": "kuān", + "寬": "kuān", + "灵": "líng", + "靈": "líng", + "居": "jū", + "松": "sōng", + "訓": "xùn", + "训": "xùn", + "罪": "zuì", + "炮": "pào", + "粗": "cū", + "罵": "mà", + "膀": "bǎng", + "若": "ruò", + "骂": "mà", + "圈": "quān", + "孔": "kǒng", + "貴": "guì", + "贵": "guì", + "扬": "yáng", + "揚": "yáng", + "楼": "lóu", + "樓": "lóu", + "献": "xiàn", + "獻": "xiàn", + "縮": "suō", + "缩": "suō", + "份": "fèn", + "紡": "fǎng", + "纺": "fǎng", + "胸": "xiōng", + "輛": "liàng", + "辆": "liàng", + "途": "tú", + "炉": "lú", + "爐": "lú", + "渡": "dù", + "耳": "ěr", + "倾": "qīng", + "傾": "qīng", + "涂": "tú", + "票": "piào", + "菌": "jūn", + "壮": "zhuàng", + "壯": "zhuàng", + "播": "bō", + "械": "xiè", + "拖": "tuō", + "职": "zhí", + "職": "zhí", + "克": "kè", + "帐": "zhàng", + "帳": "zhàng", + "挤": "jǐ", + "擠": "jǐ", + "秋": "qiū", + "括": "kuò", + "索": "suǒ", + "肚": "dù", + "插": "chā", + "棵": "kē", + "湿": "shī", + "濕": "shī", + "謂": "wèi", + "谓": "wèi", + "麻": "má", + "尾": "wěi", + "阿": "ā", + "尖": "jiān", + "慌": "huāng", + "梁": "liáng", + "涌": "yǒng", + "盆": "pén", + "蛋": "dàn", + "趣": "qù", + "冰": "bīng", + "怒": "nù", + "咬": "yǎo", + "財": "cái", + "财": "cái", + "避": "bì", + "累": "lèi", + "辩": "biàn", + "辯": "biàn", + "曲": "qū", + "磨": "mó", + "逃": "táo", + "餓": "è", + "饿": "è", + "承": "chéng", + "疑": "yí", + "刺": "cì", + "探": "tàn", + "糊": "hú", + "肥": "féi", + "贊": "zàn", + "赞": "zàn", + "弯": "wān", + "彎": "wān", + "徒": "tú", + "香": "xiāng", + "付": "fù", + "腰": "yāo", + "愤": "fèn", + "憤": "fèn", + "扩": "kuò", + "擴": "kuò", + "暖": "nuǎn", + "吨": "dūn", + "噸": "dūn", + "阻": "zǔ", + "介": "jiè", + "柴": "chái", + "獲": "huò", + "紹": "shào", + "绍": "shào", + "获": "huò", + "藏": "cáng", + "緩": "huǎn", + "缓": "huǎn", + "隔": "gé", + "奔": "bēn", + "秘": "mì", + "偏": "piān", + "叹": "tàn", + "嘆": "tàn", + "窝": "wō", + "窩": "wō", + "净": "jìng", + "晨": "chén", + "淨": "jìng", + "稳": "wěn", + "穩": "wěn", + "詩": "shī", + "诗": "shī", + "喂": "wèi", + "暴": "bào", + "殖": "zhí", + "潮": "cháo", + "协": "xié", + "協": "xié", + "登": "dēng", + "迷": "mí", + "壁": "bì", + "毕": "bì", + "畢": "bì", + "浮": "fú", + "紛": "fēn", + "纷": "fēn", + "闊": "kuò", + "阔": "kuò", + "阴": "yīn", + "附": "fù", + "陰": "yīn", + "井": "jǐng", + "哼": "hēng", + "巧": "qiǎo", + "拼": "pīn", + "榮": "róng", + "滚": "gǔn", + "滾": "gǔn", + "荣": "róng", + "厉": "lì", + "厲": "lì", + "异": "yì", + "異": "yì", + "麥": "mài", + "麦": "mài", + "寒": "hán", + "惯": "guàn", + "慣": "guàn", + "谷": "gǔ", + "丟": "diū", + "丢": "diū", + "培": "péi", + "宇": "yǔ", + "泛": "fàn", + "肃": "sù", + "肅": "sù", + "載": "zài", + "载": "zài", + "录": "lù", + "舒": "shū", + "錄": "lù", + "健": "jiàn", + "婆": "pó", + "搬": "bān", + "禁": "jìn", + "寻": "xún", + "尋": "xún", + "灌": "guàn", + "补": "bǔ", + "補": "bǔ", + "駝": "tuó", + "驼": "tuó", + "促": "cù", + "刷": "shuā", + "扑": "pū", + "撲": "pū", + "析": "xī", + "珠": "zhū", + "愈": "yù", + "旅": "lǚ", + "跃": "yuè", + "躍": "yuè", + "凝": "níng", + "彩": "cǎi", + "拔": "bá", + "袖": "xiù", + "幕": "mù", + "庭": "tíng", + "戴": "dài", + "援": "yuán", + "航": "háng", + "呆": "dāi", + "挖": "wā", + "杆": "gān", + "沟": "gōu", + "溝": "gōu", + "猿": "yuán", + "瓜": "guā", + "凡": "fán", + "吓": "xià", + "嚇": "xià", + "迎": "yíng", + "凭": "píng", + "憑": "píng", + "扫": "sǎo", + "掃": "sǎo", + "騎": "qí", + "骑": "qí", + "冻": "dòng", + "凍": "dòng", + "扎": "zhā", + "操": "cāo", + "箱": "xiāng", + "純": "chún", + "纯": "chún", + "聞": "wén", + "闻": "wén", + "仔": "zǐ", + "績": "jī", + "绩": "jì", + "訊": "xùn", + "讯": "xùn", + "踏": "tà", + "顏": "yán", + "颜": "yán", + "序": "xù", + "恨": "hèn", + "抢": "qiǎng", + "搶": "qiǎng", + "横": "héng", + "橫": "héng", + "疯": "fēng", + "瘋": "fēng", + "眉": "méi", + "宙": "zhòu", + "凉": "liáng", + "卷": "juǎn", + "夢": "mèng", + "梦": "mèng", + "氧": "yǎng", + "涼": "liáng", + "繁": "fán", + "距": "jù", + "銅": "tóng", + "铜": "tóng", + "仗": "zhàng", + "割": "gē", + "损": "sǔn", + "損": "sǔn", + "摄": "shè", + "摔": "shuāi", + "攝": "shè", + "瓶": "píng", + "悲": "bēi", + "昏": "hūn", + "疼": "téng", + "繩": "shéng", + "绳": "shéng", + "豆": "dòu", + "烂": "làn", + "烦": "fán", + "煩": "fán", + "爛": "làn", + "蓝": "lán", + "藍": "lán", + "訂": "dìng", + "订": "dìng", + "侧": "cè", + "側": "cè", + "巩": "gǒng", + "慮": "lǜ", + "虑": "lǜ", + "軟": "ruǎn", + "软": "ruǎn", + "鞏": "gǒng", + "匆": "cōng", + "域": "yù", + "尺": "chǐ", + "貼": "tiē", + "賽": "sài", + "贴": "tiē", + "赛": "sài", + "躲": "duǒ", + "剧": "jù", + "劇": "jù", + "役": "yì", + "恰": "qià", + "惟": "wéi", + "狠": "hěn", + "薄": "báo", + "释": "shì", + "釋": "shì", + "駛": "shǐ", + "驶": "shǐ", + "俺": "ǎn", + "兄": "xiōng", + "尊": "zūn", + "幅": "fú", + "拥": "yōng", + "授": "shòu", + "擁": "yōng", + "杯": "bēi", + "謀": "móu", + "谋": "móu", + "劝": "quàn", + "勸": "quàn", + "博": "bó", + "仪": "yí", + "儀": "yí", + "捧": "pěng", + "睁": "zhēng", + "睜": "zhēng", + "網": "wǎng", + "网": "wǎng", + "触": "chù", + "觸": "chù", + "腾": "téng", + "騰": "téng", + "匪": "fěi", + "夹": "jiā", + "夾": "jiā", + "抖": "dǒu", + "揭": "jiē", + "稍": "shāo", + "稼": "jià", + "腐": "fǔ", + "閉": "bì", + "闭": "bì", + "浓": "nóng", + "濃": "nóng", + "胞": "bāo", + "脈": "mài", + "脉": "mài", + "駱": "luò", + "骆": "luò", + "刑": "xíng", + "惜": "xī", + "皱": "zhòu", + "皺": "zhòu", + "监": "jiān", + "監": "jiān", + "脏": "zàng", + "臟": "zàng", + "蒸": "zhēng", + "貧": "pín", + "贫": "pín", + "鍋": "guō", + "锅": "guō", + "波": "bō", + "炸": "zhà", + "礼": "lǐ", + "禮": "lǐ", + "私": "sī", + "繞": "rào", + "绕": "rào", + "塑": "sù", + "磁": "cí", + "违": "wéi", + "違": "wéi", + "丈": "zhàng", + "玉": "yù", + "茫": "máng", + "吐": "tǔ", + "喷": "pēn", + "噴": "pēn", + "废": "fèi", + "廢": "fèi", + "怜": "lián", + "恢": "huī", + "悉": "xī", + "憐": "lián", + "挨": "āi", + "敲": "qiāo", + "淡": "dàn", + "尤": "yóu", + "忆": "yì", + "憶": "yì", + "災": "zāi", + "灾": "zāi", + "蜜": "mì", + "啥": "shá", + "恐": "kǒng", + "述": "shù", + "隐": "yǐn", + "隱": "yǐn", + "残": "cán", + "殘": "cán", + "額": "é", + "额": "é", + "亩": "mǔ", + "旋": "xuán", + "污": "wū", + "甲": "jiǎ", + "畝": "mǔ", + "胆": "dǎn", + "膽": "dǎn", + "蹲": "dūn", + "迟": "chí", + "遲": "chí", + "乘": "chéng", + "伴": "bàn", + "掏": "tāo", + "縫": "fèng", + "缝": "fèng", + "刮": "guā", + "椅": "yǐ", + "串": "chuàn", + "埋": "mái", + "抵": "dǐ", + "捉": "zhuō", + "秒": "miǎo", + "乏": "fá", + "喔": "ō", + "噢": "ō", + "坡": "pō", + "捕": "bǔ", + "添": "tiān", + "牺": "xī", + "犧": "xī", + "粒": "lì", + "舍": "shě", + "允": "yǔn", + "哇": "wa", + "柜": "guì", + "酸": "suān", + "寄": "jì", + "扔": "rēng", + "托": "tuō", + "措": "cuò", + "狂": "kuáng", + "遗": "yí", + "遺": "yí", + "伏": "fú", + "兔": "tù", + "勤": "qín", + "珍": "zhēn", + "糟": "zāo", + "輝": "huī", + "辉": "huī", + "拾": "shí", + "殊": "shū", + "浑": "hún", + "渾": "hún", + "滴": "dī", + "典": "diǎn", + "漠": "mò", + "猜": "cāi", + "障": "zhàng", + "唇": "chún", + "壳": "ké", + "峡": "xiá", + "峽": "xiá", + "德": "dé", + "忿": "fèn", + "撞": "zhuàng", + "棒": "bàng", + "殼": "ké", + "滑": "huá", + "牵": "qiān", + "牽": "qiān", + "盛": "shèng", + "糖": "táng", + "貢": "gòng", + "贡": "gòng", + "哟": "yō", + "喲": "yō", + "宜": "yí", + "敬": "jìng", + "斜": "xié", + "暂": "zàn", + "暫": "zàn", + "歼": "jiān", + "殲": "jiān", + "竹": "zhú", + "笼": "lóng", + "籠": "lóng", + "聪": "cōng", + "聰": "cōng", + "蜂": "fēng", + "騙": "piàn", + "骗": "piàn", + "扭": "niǔ", + "詳": "xiáng", + "详": "xiáng", + "貌": "mào", + "辟": "pì", + "亡": "wáng", + "峰": "fēng", + "励": "lì", + "勵": "lì", + "归": "guī", + "歸": "guī", + "焊": "hàn", + "秀": "xiù", + "唤": "huàn", + "喚": "huàn", + "寸": "cùn", + "毀": "huǐ", + "毁": "huǐ", + "稻": "dào", + "緒": "xù", + "绪": "xù", + "脆": "cuì", + "銷": "xiāo", + "销": "xiāo", + "库": "kù", + "庫": "kù", + "渠": "qú", + "爹": "diē", + "祝": "zhù", + "貫": "guàn", + "贯": "guàn", + "雷": "léi", + "坑": "kēng", + "蒙": "méng", + "辛": "xīn", + "遵": "zūn", + "飄": "piāo", + "飘": "piāo", + "婚": "hūn", + "披": "pī", + "胃": "wèi", + "趟": "tàng", + "逼": "bī", + "閑": "xián", + "闲": "xián", + "嚷": "rǎng", + "垂": "chuí", + "塞": "sāi", + "娃": "wá", + "扯": "chě", + "狼": "láng", + "鍛": "duàn", + "锻": "duàn", + "凳": "dèng", + "卵": "luǎn", + "炕": "kàng", + "箭": "jiàn", + "肤": "fū", + "膚": "fū", + "跡": "jī", + "輩": "bèi", + "辈": "bèi", + "迹": "jì", + "匠": "jiàng", + "巾": "jīn", + "洁": "jié", + "涨": "zhǎng", + "漲": "zhǎng", + "潔": "jié", + "猴": "hóu", + "耗": "hào", + "臂": "bì", + "虚": "xū", + "虛": "xū", + "陷": "xiàn", + "吵": "chǎo", + "咳": "hāi", + "搭": "dā", + "森": "sēn", + "漂": "piào", + "狱": "yù", + "獄": "yù", + "疗": "liáo", + "療": "liáo", + "皇": "huáng", + "翅": "chì", + "脾": "pí", + "鈴": "líng", + "铃": "líng", + "雾": "wù", + "霧": "wù", + "飽": "bǎo", + "饱": "bǎo", + "尚": "shàng", + "拣": "jiǎn", + "振": "zhèn", + "掩": "yǎn", + "揀": "jiǎn", + "歇": "xiē", + "牧": "mù", + "番": "fān", + "符": "fú", + "趁": "chèn", + "挡": "dǎng", + "擋": "dǎng", + "晓": "xiǎo", + "曉": "xiǎo", + "猪": "zhū", + "綱": "gāng", + "纲": "gāng", + "舅": "jiù", + "豬": "zhū", + "迈": "mài", + "递": "dì", + "遞": "dì", + "邁": "mài", + "壤": "rǎng", + "撤": "chè", + "浅": "qiǎn", + "淺": "qiǎn", + "瘦": "shòu", + "肠": "cháng", + "腸": "cháng", + "塘": "táng", + "塵": "chén", + "妙": "miào", + "尘": "chén", + "砍": "kǎn", + "碑": "bēi", + "焦": "jiāo", + "衡": "héng", + "齒": "chǐ", + "齿": "chǐ", + "剂": "jì", + "剑": "jiàn", + "劍": "jiàn", + "劑": "jì", + "匹": "pǐ", + "摘": "zhāi", + "竞": "jìng", + "競": "jìng", + "凶": "xiōng", + "售": "shòu", + "堵": "dǔ", + "康": "kāng", + "拱": "gǒng", + "漆": "qī", + "疲": "pí", + "盒": "hé", + "紗": "shā", + "纱": "shā", + "坦": "tǎn", + "斯": "sī", + "杨": "yáng", + "楊": "yáng", + "泡": "pào", + "盟": "méng", + "瞪": "dèng", + "緣": "yuán", + "缘": "yuán", + "苹": "píng", + "蘋": "píng", + "轟": "hōng", + "轰": "hōng", + "逗": "dòu", + "享": "xiǎng", + "喘": "chuǎn", + "嘿": "hēi", + "挣": "zhēng", + "掙": "zhēng", + "棚": "péng", + "签": "qiān", + "簽": "qiān", + "龍": "lóng", + "龙": "lóng", + "宿": "sù", + "悶": "mèn", + "泼": "pō", + "溉": "gài", + "潑": "pō", + "甜": "tián", + "舱": "cāng", + "艙": "cāng", + "闷": "mèn", + "餅": "bǐng", + "饼": "bǐng", + "愉": "yú", + "捏": "niē", + "棍": "gùn", + "盼": "pàn", + "篮": "lán", + "籃": "lán", + "芦": "lú", + "蘆": "lú", + "鉛": "qiān", + "铅": "qiān", + "匯": "huì", + "奴": "nú", + "宫": "gōng", + "宮": "gōng", + "汇": "huì", + "炭": "tàn", + "版": "bǎn", + "牌": "pái", + "窑": "yáo", + "窯": "yáo", + "聚": "jù", + "脖": "bó", + "訪": "fǎng", + "访": "fǎng", + "隶": "lì", + "隸": "lì", + "咐": "fù", + "摊": "tān", + "攤": "tān", + "昆": "kūn", + "桶": "tǒng", + "池": "chí", + "猎": "liè", + "獵": "liè", + "碍": "ài", + "礙": "ài", + "臭": "chòu", + "詞": "cí", + "词": "cí", + "軌": "guǐ", + "轨": "guǐ", + "釣": "diào", + "钓": "diào", + "顫": "chàn", + "颤": "chàn", + "亏": "kuī", + "仇": "chóu", + "择": "zé", + "擇": "zé", + "智": "zhì", + "苗": "miáo", + "虧": "kuī", + "鋒": "fēng", + "锋": "fēng", + "仰": "yǎng", + "屆": "jiè", + "届": "jiè", + "岗": "gǎng", + "岩": "yán", + "岭": "lǐng", + "崗": "gǎng", + "嶺": "lǐng", + "慰": "wèi", + "抄": "chāo", + "盐": "yán", + "譯": "yì", + "译": "yì", + "鹽": "yán", + "丛": "cóng", + "乌": "wū", + "凑": "còu", + "厘": "lí", + "叢": "cóng", + "奖": "jiǎng", + "妻": "qī", + "径": "jìng", + "徑": "jìng", + "悟": "wù", + "欠": "qiàn", + "湊": "còu", + "烏": "wū", + "獎": "jiǎng", + "苍": "cāng", + "荷": "hé", + "蒼": "cāng", + "輯": "jí", + "辑": "jí", + "陪": "péi", + "叛": "pàn", + "捞": "lāo", + "撈": "lāo", + "撒": "sā", + "柱": "zhù", + "株": "zhū", + "核": "hé", + "润": "rùn", + "漏": "lòu", + "潤": "rùn", + "瓷": "cí", + "糾": "jiū", + "纠": "jiū", + "蛇": "shé", + "鎖": "suǒ", + "锁": "suǒ", + "估": "gū", + "傲": "ào", + "厌": "yàn", + "厭": "yàn", + "宗": "zōng", + "扶": "fú", + "捆": "kǔn", + "荡": "dàng", + "蕩": "dàng", + "蚀": "shí", + "蝕": "shí", + "裂": "liè", + "驕": "jiāo", + "骄": "jiāo", + "幼": "yòu", + "拨": "bō", + "挽": "wǎn", + "掀": "xiān", + "撥": "bō", + "銳": "ruì", + "锐": "ruì", + "鳴": "míng", + "鸣": "míng", + "款": "kuǎn", + "盯": "dīng", + "胳": "gē", + "偶": "ǒu", + "寂": "jì", + "屈": "qū", + "恳": "kěn", + "懇": "kěn", + "晃": "huǎng", + "歪": "wāi", + "眯": "mī", + "瞇": "mī", + "秧": "yāng", + "稿": "gǎo", + "綜": "zōng", + "综": "zōng", + "踩": "cǎi", + "鯨": "jīng", + "鲸": "jīng", + "吼": "hǒu", + "嗓": "sǎng", + "扁": "biǎn", + "朴": "pǔ", + "欣": "xīn", + "莫": "mò", + "傻": "shǎ", + "幻": "huàn", + "扣": "kòu", + "拢": "lǒng", + "掠": "lüè", + "攏": "lǒng", + "榴": "liú", + "溶": "róng", + "滩": "tān", + "灘": "tān", + "牢": "láo", + "猫": "māo", + "腔": "qiāng", + "蚕": "cán", + "蝗": "huáng", + "蠶": "cán", + "裤": "kù", + "褲": "kù", + "貓": "māo", + "跨": "kuà", + "霜": "shuāng", + "冶": "yě", + "咽": "yàn", + "宅": "zhái", + "搜": "sōu", + "晴": "qíng", + "遮": "zhē", + "启": "qǐ", + "啟": "qǐ", + "彼": "bǐ", + "抹": "mǒ", + "搁": "gē", + "擱": "gē", + "敏": "mǐn", + "漫": "màn", + "码": "mǎ", + "碼": "mǎ", + "筋": "jīn", + "鍵": "jiàn", + "键": "jiàn", + "厅": "tīng", + "吊": "diào", + "廳": "tīng", + "拒": "jù", + "旱": "hàn", + "桃": "táo", + "欺": "qī", + "燕": "yàn", + "琴": "qín", + "舌": "shé", + "蔽": "bì", + "袄": "ǎo", + "襖": "ǎo", + "釘": "dīng", + "钉": "dīng", + "駕": "jià", + "驾": "jià", + "丘": "qiū", + "审": "shěn", + "審": "shěn", + "币": "bì", + "幣": "bì", + "愣": "lèng", + "拦": "lán", + "摧": "cuī", + "撕": "sī", + "攔": "lán", + "浇": "jiāo", + "澆": "jiāo", + "賞": "shǎng", + "赏": "shǎng", + "鴉": "yā", + "鸦": "yā", + "伞": "sǎn", + "傘": "sǎn", + "晶": "jīng", + "涉": "shè", + "犹": "yóu", + "猶": "yóu", + "蛙": "wā", + "丫": "yā", + "僚": "liáo", + "哏": "gén", + "嗡": "wēng", + "宪": "xiàn", + "憲": "xiàn", + "描": "miáo", + "朗": "lǎng", + "柔": "róu", + "橘": "jú", + "瞎": "xiā", + "稀": "xī", + "肝": "gān", + "裳": "shang", + "隆": "lóng", + "頑": "wán", + "顽": "wán", + "驢": "lǘ", + "驴": "lǘ", + "倡": "chàng", + "哀": "āi", + "堤": "dī", + "姨": "yí", + "崇": "chóng", + "庙": "miào", + "廟": "miào", + "延": "yán", + "汤": "tāng", + "湯": "tāng", + "碳": "tàn", + "童": "tóng", + "耕": "gēng", + "跪": "guì", + "辫": "biàn", + "辮": "biàn", + "闖": "chuǎng", + "闯": "chuǎng", + "頗": "pō", + "颇": "pō", + "勃": "bó", + "哗": "huā", + "嘩": "huā", + "嫁": "jià", + "孤": "gū", + "拳": "quán", + "晒": "shài", + "栽": "zāi", + "洒": "sǎ", + "耀": "yào", + "胀": "zhàng", + "胁": "xié", + "脅": "xié", + "脹": "zhàng", + "膜": "mó", + "荒": "huāng", + "亭": "tíng", + "咧": "liě", + "填": "tián", + "妥": "tuǒ", + "帘": "lián", + "患": "huàn", + "截": "jié", + "抑": "yì", + "攀": "pān", + "梅": "méi", + "烛": "zhú", + "燭": "zhú", + "督": "dū", + "逢": "féng", + "魔": "mó", + "伐": "fá", + "媳": "xí", + "悬": "xuán", + "懸": "xuán", + "戚": "qī", + "煮": "zhǔ", + "盗": "dào", + "盜": "dào", + "綁": "bǎng", + "绑": "bǎng", + "肺": "fèi", + "侦": "zhēn", + "俗": "sú", + "偵": "zhēn", + "哨": "shào", + "喉": "hóu", + "岂": "qǐ", + "帜": "zhì", + "幟": "zhì", + "庆": "qìng", + "弃": "qì", + "惨": "cǎn", + "慘": "cǎn", + "慶": "qìng", + "抛": "pāo", + "拋": "pāo", + "末": "mò", + "棄": "qì", + "浸": "jìn", + "港": "gǎng", + "眨": "zhǎ", + "租": "zū", + "窜": "cuàn", + "竄": "cuàn", + "誠": "chéng", + "诚": "chéng", + "豈": "qǐ", + "醉": "zuì", + "刊": "kān", + "墨": "mò", + "桩": "zhuāng", + "樁": "zhuāng", + "炎": "yán", + "盏": "zhǎn", + "盞": "zhǎn", + "肖": "xiào", + "踢": "tī", + "錦": "jǐn", + "锦": "jǐn", + "啪": "pā", + "塔": "tǎ", + "惹": "rě", + "柳": "liǔ", + "筐": "kuāng", + "紫": "zǐ", + "罩": "zhào", + "萄": "táo", + "葡": "pú", + "貝": "bèi", + "贝": "bèi", + "辨": "biàn", + "顛": "diān", + "颠": "diān", + "伪": "wěi", + "偽": "wěi", + "冤": "yuān", + "厨": "chú", + "吩": "fēn", + "妄": "wàng", + "姿": "zī", + "屁": "pì", + "廚": "chú", + "愁": "chóu", + "晌": "shǎng", + "渴": "kě", + "溜": "liū", + "甩": "shuǎi", + "眠": "mián", + "粥": "zhōu", + "綸": "lún", + "纶": "lún", + "蝉": "chán", + "蟬": "chán", + "覽": "lǎn", + "览": "lǎn", + "返": "fǎn", + "閥": "fá", + "阀": "fá", + "鞭": "biān", + "頻": "pín", + "频": "pín", + "仓": "cāng", + "倉": "cāng", + "傍": "bàng", + "壶": "hú", + "壺": "hú", + "怨": "yuàn", + "汗": "hàn", + "泉": "quán", + "窄": "zhǎi", + "紋": "wén", + "纹": "wén", + "跌": "diē", + "喽": "lóu", + "嘍": "lóu", + "坟": "fén", + "墳": "fén", + "扛": "káng", + "扮": "bàn", + "洪": "hóng", + "瓦": "wǎ", + "秩": "zhì", + "脂": "zhī", + "虾": "xiā", + "蝦": "xiā", + "衬": "chèn", + "袭": "xí", + "裹": "guǒ", + "襯": "chèn", + "襲": "xí", + "諒": "liàng", + "谅": "liàng", + "魂": "hún", + "乙": "yǐ", + "倘": "tǎng", + "卧": "wò", + "矮": "ǎi", + "筒": "tǒng", + "膊": "bó", + "臥": "wò", + "揉": "róu", + "昂": "áng", + "栏": "lán", + "欄": "lán", + "疾": "jí", + "痕": "hén", + "砖": "zhuān", + "磚": "zhuān", + "膨": "péng", + "餐": "cān", + "兜": "dōu", + "夸": "kuā", + "崖": "yá", + "拆": "chāi", + "斧": "fǔ", + "欲": "yù", + "沫": "mò", + "涡": "wō", + "渦": "wō", + "縱": "zòng", + "纵": "zòng", + "肌": "jī", + "胖": "pàng", + "趴": "pā", + "飲": "yǐn", + "饮": "yǐn", + "齡": "líng", + "龄": "líng", + "丹": "dān", + "勾": "gōu", + "嘻": "xī", + "御": "yù", + "戒": "jiè", + "拴": "shuān", + "撐": "chēng", + "撑": "chēng", + "朽": "xiǔ", + "甘": "gān", + "袜": "wà", + "袱": "fú", + "裁": "cái", + "襪": "wà", + "譬": "pì", + "鉤": "gōu", + "鋁": "lǚ", + "钩": "gōu", + "铝": "lǚ", + "鼠": "shǔ", + "催": "cuī", + "咦": "yí", + "拧": "níng", + "搅": "jiǎo", + "擰": "níng", + "攪": "jiǎo", + "淹": "yān", + "渔": "yú", + "漁": "yú", + "熊": "xióng", + "盲": "máng", + "筷": "kuài", + "緯": "wěi", + "纬": "wěi", + "購": "gòu", + "购": "gòu", + "鴨": "yā", + "鸭": "yā", + "予": "yǔ", + "兼": "jiān", + "兽": "shòu", + "呈": "chéng", + "哄": "hōng", + "娶": "qǔ", + "恆": "héng", + "恒": "héng", + "慧": "huì", + "梯": "tī", + "殿": "diàn", + "氏": "shì", + "淋": "lín", + "溪": "xī", + "獸": "shòu", + "罐": "guàn", + "蚁": "yǐ", + "蚂": "mǎ", + "蜡": "là", + "螞": "mǎ", + "蟻": "yǐ", + "蠟": "là", + "誕": "dàn", + "诞": "dàn", + "逮": "dǎi", + "飾": "shì", + "饰": "shì", + "剪": "jiǎn", + "叠": "dié", + "嗽": "sòu", + "悔": "huǐ", + "槽": "cáo", + "疊": "dié", + "碧": "bì", + "繪": "huì", + "绘": "huì", + "耸": "sǒng", + "聳": "sǒng", + "蝇": "yíng", + "蠅": "yíng", + "豫": "yù", + "蹬": "dēng", + "軸": "zhóu", + "轴": "zhóu", + "叮": "dīng", + "嘗": "cháng", + "圾": "jī", + "垃": "lā", + "垮": "kuǎ", + "尝": "cháng", + "慎": "shèn", + "沾": "zhān", + "潛": "qián", + "潜": "qián", + "皂": "zào", + "窃": "qiè", + "竊": "qiè", + "缸": "gāng", + "肢": "zhī", + "胎": "tāi", + "脊": "jǐ", + "膝": "xī", + "艳": "yàn", + "艷": "yàn", + "詫": "chà", + "诧": "chà", + "酷": "kù", + "雕": "diāo", + "霉": "méi", + "冈": "gāng", + "勉": "miǎn", + "吆": "yāo", + "嫌": "xián", + "岡": "gāng", + "巷": "xiàng", + "愧": "kuì", + "拌": "bàn", + "揪": "jiū", + "晰": "xī", + "泊": "pō", + "灿": "càn", + "燦": "càn", + "瓣": "bàn", + "症": "zhèng", + "胶": "jiāo", + "膠": "jiāo", + "豁": "huō", + "踱": "duó", + "閨": "guī", + "闺": "guī", + "隙": "xì", + "飢": "jī", + "饅": "mán", + "饥": "jī", + "馒": "mán", + "债": "zhài", + "債": "zhài", + "唰": "shuā", + "墩": "dūn", + "弓": "gōng", + "恥": "chǐ", + "旦": "dàn", + "李": "lǐ", + "烤": "kǎo", + "熄": "xī", + "砸": "zá", + "粪": "fèn", + "糞": "fèn", + "耻": "chǐ", + "誉": "yù", + "譽": "yù", + "貿": "mào", + "贸": "mào", + "酱": "jiàng", + "醬": "jiàng", + "鑄": "zhù", + "铸": "zhù", + "飼": "sì", + "饲": "sì", + "亦": "yì", + "仙": "xiān", + "哧": "chī", + "嘱": "zhǔ", + "囑": "zhǔ", + "妨": "fáng", + "婴": "yīng", + "嬰": "yīng", + "寞": "mò", + "押": "yā", + "斥": "chì", + "框": "kuàng", + "爽": "shuǎng", + "甭": "béng", + "畜": "chù", + "癌": "ái", + "硫": "liú", + "笨": "bèn", + "籍": "jí", + "芒": "máng", + "蝴": "hú", + "蝶": "dié", + "袍": "páo", + "豪": "háo", + "邻": "lín", + "鄰": "lín", + "頁": "yè", + "页": "yè", + "馳": "chí", + "驰": "chí", + "倚": "yǐ", + "僵": "jiāng", + "凿": "záo", + "勻": "yún", + "匀": "yún", + "君": "jūn", + "宴": "yàn", + "宵": "xiāo", + "崭": "zhǎn", + "嶄": "zhǎn", + "扇": "shàn", + "枕": "zhěn", + "枯": "kū", + "渗": "shèn", + "滲": "shèn", + "焰": "yàn", + "瞅": "chǒu", + "縛": "fù", + "缚": "fù", + "蛛": "zhū", + "蜘": "zhī", + "赤": "chì", + "迁": "qiān", + "遷": "qiān", + "鑿": "záo", + "埃": "āi", + "慨": "kǎi", + "挫": "cuò", + "淘": "táo", + "渣": "zhā", + "砂": "shā", + "耽": "dān", + "苏": "sū", + "蔬": "shū", + "蘇": "sū", + "訝": "yà", + "讶": "yà", + "躁": "zào", + "鉴": "jiàn", + "鑒": "jiàn", + "雀": "què", + "駐": "zhù", + "驻": "zhù", + "侮": "wǔ", + "吁": "xū", + "呸": "pēi", + "啾": "jiū", + "塌": "tā", + "循": "xún", + "怖": "bù", + "扰": "rǎo", + "擾": "rǎo", + "朦": "méng", + "朧": "lóng", + "燥": "zào", + "瞒": "mán", + "瞞": "mán", + "纏": "chán", + "缠": "chán", + "胧": "lóng", + "苇": "wěi", + "葦": "wěi", + "診": "zhěn", + "诊": "zhěn", + "辽": "liáo", + "遼": "liáo", + "邮": "yóu", + "郵": "yóu", + "陌": "mò", + "丑": "chǒu", + "俘": "fú", + "凸": "tū", + "凹": "āo", + "刹": "shā", + "剎": "shā", + "嗨": "hāi", + "宏": "hóng", + "懒": "lǎn", + "懶": "lǎn", + "扒": "bā", + "梢": "shāo", + "狭": "xiá", + "狹": "xiá", + "碱": "jiǎn", + "芽": "yá", + "虏": "lǔ", + "虜": "lǔ", + "賊": "zéi", + "贼": "zéi", + "輔": "fǔ", + "輻": "fú", + "辅": "fǔ", + "辐": "fú", + "逝": "shì", + "陡": "dǒu", + "陵": "líng", + "頌": "sòng", + "颂": "sòng", + "鹼": "jiǎn", + "黏": "nián", + "丙": "bǐng", + "吞": "tūn", + "哆": "duō", + "嗦": "suō", + "夕": "xī", + "屿": "yǔ", + "嶼": "yǔ", + "州": "zhōu", + "梳": "shū", + "汹": "xiōng", + "洶": "xiōng", + "浊": "zhuó", + "淀": "diàn", + "澱": "diàn", + "濁": "zhuó", + "烫": "tàng", + "燙": "tàng", + "爪": "zhǎo", + "竖": "shù", + "竿": "gān", + "繃": "běng", + "绷": "bēng", + "翘": "qiào", + "翹": "qiào", + "艘": "sōu", + "蚊": "wén", + "誒": "éi", + "誼": "yì", + "诶": "éi", + "谊": "yì", + "豎": "shù", + "貪": "tān", + "贪": "tān", + "遣": "qiǎn", + "邀": "yāo", + "黎": "lí", + "乒": "pīng", + "乓": "pāng", + "佩": "pèi", + "华": "huá", + "堪": "kān", + "昼": "zhòu", + "晝": "zhòu", + "柄": "bǐng", + "橡": "xiàng", + "毯": "tǎn", + "潭": "tán", + "烁": "shuò", + "爍": "shuò", + "矩": "jǔ", + "耍": "shuǎ", + "聊": "liáo", + "膛": "táng", + "茂": "mào", + "華": "huá", + "賤": "jiàn", + "贱": "jiàn", + "鏟": "chǎn", + "铲": "chǎn", + "倦": "juàn", + "卡": "kǎ", + "卸": "xiè", + "嘲": "cháo", + "囪": "cōng", + "囱": "cōng", + "圣": "shèng", + "垄": "lǒng", + "垒": "lěi", + "壕": "háo", + "壘": "lěi", + "壟": "lǒng", + "寡": "guǎ", + "巢": "cháo", + "氛": "fēn", + "沥": "lì", + "瀝": "lì", + "炯": "jiǒng", + "甸": "diàn", + "聖": "shèng", + "荔": "lì", + "葫": "hú", + "賴": "lài", + "赖": "lài", + "蹦": "bèng", + "鑼": "luó", + "锣": "luó", + "鵲": "què", + "鹊": "què", + "乃": "nǎi", + "忠": "zhōng", + "恼": "nǎo", + "惱": "nǎo", + "拜": "bài", + "搀": "chān", + "攙": "chān", + "旺": "wàng", + "昧": "mèi", + "滋": "zī", + "磷": "lín", + "竭": "jié", + "絡": "luò", + "絨": "róng", + "绒": "róng", + "络": "luò", + "署": "shǔ", + "脯": "pú", + "覆": "fù", + "訟": "sòng", + "讼": "sòng", + "轎": "jiào", + "轿": "jiào", + "霸": "bà", + "馱": "tuó", + "驮": "tuó", + "亚": "yà", + "亞": "yà", + "劣": "liè", + "咙": "lóng", + "喃": "nán", + "嚨": "lóng", + "奏": "zòu", + "奠": "diàn", + "嫩": "nèn", + "尔": "ěr", + "徐": "xú", + "捡": "jiǎn", + "搏": "bó", + "撿": "jiǎn", + "枉": "wǎng", + "煌": "huáng", + "爾": "ěr", + "猩": "xīng", + "盔": "kuī", + "窟": "kū", + "窿": "lóng", + "粘": "zhān", + "紐": "niǔ", + "纽": "niǔ", + "罚": "fá", + "罰": "fá", + "衫": "shān", + "謙": "qiān", + "謬": "miù", + "谦": "qiān", + "谬": "miù", + "郊": "jiāo", + "頃": "qǐng", + "顷": "qǐng", + "駁": "bó", + "驳": "bó", + "储": "chǔ", + "僱": "gù", + "儲": "chǔ", + "剿": "jiǎo", + "卜": "bo", + "叭": "bā", + "喇": "lǎ", + "奉": "fèng", + "弥": "mí", + "彌": "mí", + "揍": "zòu", + "搂": "lǒu", + "摟": "lǒu", + "旬": "xún", + "杰": "jié", + "毅": "yì", + "氓": "máng", + "沸": "fèi", + "涤": "dí", + "滌": "dí", + "灶": "zào", + "磺": "huáng", + "萝": "luó", + "蘿": "luó", + "融": "róng", + "誓": "shì", + "賺": "zhuàn", + "赚": "zhuàn", + "辣": "là", + "鈔": "chāo", + "錫": "xī", + "钞": "chāo", + "锡": "xī", + "闡": "chǎn", + "阐": "chǎn", + "雇": "gù", + "乳": "rǔ", + "伺": "cì", + "剖": "pōu", + "吟": "yín", + "址": "zhǐ", + "坝": "bà", + "坯": "pī", + "垫": "diàn", + "堡": "bǎo", + "墊": "diàn", + "壩": "bà", + "怦": "pēng", + "恍": "huǎng", + "惭": "cán", + "慕": "mù", + "慚": "cán", + "拐": "guǎi", + "砌": "qì", + "綢": "chóu", + "绸": "chóu", + "羡": "xiàn", + "羨": "xiàn", + "肿": "zhǒng", + "腫": "zhǒng", + "腹": "fù", + "蓬": "péng", + "詢": "xún", + "諸": "zhū", + "询": "xún", + "诸": "zhū", + "賠": "péi", + "赔": "péi", + "踌": "chóu", + "躇": "chú", + "躊": "chóu", + "逻": "luó", + "邏": "luó", + "陈": "chén", + "陳": "chén", + "鵝": "é", + "鹅": "é", + "匾": "biǎn", + "墓": "mù", + "忧": "yōu", + "悅": "yuè", + "悦": "yuè", + "惑": "huò", + "憂": "yōu", + "捣": "dǎo", + "搓": "cuō", + "搗": "dǎo", + "档": "dàng", + "檔": "dàng", + "歉": "qiàn", + "泌": "mì", + "溅": "jiàn", + "澡": "zǎo", + "濺": "jiàn", + "磕": "kē", + "稅": "shuì", + "税": "shuì", + "篷": "péng", + "翼": "yì", + "蟀": "shuài", + "蟋": "xī", + "辞": "cí", + "辭": "cí", + "遙": "yáo", + "遥": "yáo", + "陶": "táo", + "饒": "ráo", + "饶": "ráo", + "丧": "sàng", + "俯": "fǔ", + "呗": "bei", + "唄": "bei", + "喪": "sàng", + "奈": "nài", + "宁": "níng", + "寧": "níng", + "帆": "fān", + "廓": "kuò", + "拟": "nǐ", + "捂": "wǔ", + "擬": "nǐ", + "氨": "ān", + "汞": "gǒng", + "淌": "tǎng", + "炒": "chǎo", + "煎": "jiān", + "繡": "xiù", + "绣": "xiù", + "艇": "tǐng", + "躬": "gōng", + "辱": "rǔ", + "酬": "chóu", + "醋": "cù", + "鋤": "chú", + "鏽": "xiù", + "锄": "chú", + "锈": "xiù", + "閱": "yuè", + "阅": "yuè", + "隧": "suì", + "雌": "cí", + "鞠": "jū", + "骼": "gé", + "佳": "jiā", + "冊": "cè", + "册": "cè", + "啸": "xiào", + "嘯": "xiào", + "姻": "yīn", + "孵": "fū", + "憾": "hàn", + "扳": "bān", + "敷": "fū", + "棋": "qí", + "涛": "tāo", + "濤": "tāo", + "熔": "róng", + "熬": "áo", + "狮": "shī", + "獅": "shī", + "畔": "pàn", + "疏": "shū", + "納": "nà", + "纳": "nà", + "腈": "jīng", + "膏": "gāo", + "舰": "jiàn", + "艦": "jiàn", + "蜓": "tíng", + "蜻": "qīng", + "謊": "huǎng", + "谎": "huǎng", + "趋": "qū", + "趨": "qū", + "軋": "yà", + "轧": "yà", + "頒": "bān", + "颁": "bān", + "騾": "luó", + "骡": "luó", + "鵪": "ān", + "鶉": "chún", + "鹌": "ān", + "鹑": "chún", + "侍": "shì", + "壽": "shòu", + "奸": "jiān", + "寿": "shòu", + "慈": "cí", + "捷": "jié", + "枣": "zǎo", + "棗": "zǎo", + "榜": "bǎng", + "毡": "zhān", + "氈": "zhān", + "汪": "wāng", + "津": "jīn", + "滔": "tāo", + "狡": "jiǎo", + "猾": "huá", + "申": "shēn", + "畏": "wèi", + "祥": "xiáng", + "穗": "suì", + "簇": "cù", + "翁": "wēng", + "茸": "róng", + "蚩": "chī", + "蠕": "rú", + "衙": "yá", + "謹": "jǐn", + "谨": "jǐn", + "閘": "zhá", + "闸": "zhá", + "驟": "zhòu", + "骤": "zhòu", + "乖": "guāi", + "咆": "páo", + "哮": "xiào", + "孙": "sūn", + "孫": "sūn", + "寇": "kòu", + "崩": "bēng", + "庞": "páng", + "憋": "biē", + "捎": "shāo", + "敞": "chǎng", + "晕": "yūn", + "暈": "yūn", + "柏": "bǎi", + "瀑": "pù", + "烘": "hōng", + "熏": "xūn", + "舀": "yǎo", + "荐": "jiàn", + "蔼": "ǎi", + "藹": "ǎi", + "衰": "shuāi", + "謠": "yáo", + "譏": "jī", + "讥": "jī", + "谣": "yáo", + "跺": "duò", + "逛": "guàng", + "鷹": "yīng", + "鹰": "yīng", + "龐": "páng", + "俱": "jù", + "厢": "xiāng", + "叙": "xù", + "咂": "zā", + "屹": "yì", + "廂": "xiāng", + "怯": "qiè", + "拘": "jū", + "携": "xié", + "摩": "mó", + "攜": "xié", + "敘": "xù", + "暢": "chàng", + "梗": "gěng", + "沃": "wò", + "滥": "làn", + "濫": "làn", + "狐": "hú", + "狸": "lí", + "琢": "zuó", + "畅": "chàng", + "眶": "kuàng", + "簸": "bǒ", + "粜": "tiào", + "糕": "gāo", + "糶": "tiào", + "絹": "juàn", + "綿": "mián", + "縷": "lǚ", + "绢": "juàn", + "绵": "mián", + "缕": "lǚ", + "菩": "pú", + "萤": "yíng", + "萨": "sà", + "薩": "sà", + "螢": "yíng", + "鍍": "dù", + "镀": "dù", + "劈": "pī", + "厦": "shà", + "咋": "zǎ", + "啃": "kěn", + "屉": "tì", + "屜": "tì", + "嵌": "qiàn", + "廈": "shà", + "徊": "huái", + "徘": "pái", + "捍": "hàn", + "撼": "hàn", + "斃": "bì", + "杉": "shān", + "毙": "bì", + "泳": "yǒng", + "浆": "jiāng", + "湾": "wān", + "漾": "yàng", + "漿": "jiāng", + "灣": "wān", + "煞": "shā", + "疙": "gē", + "瘩": "da", + "碌": "lù", + "磅": "bàng", + "粹": "cuì", + "繳": "jiǎo", + "缰": "jiāng", + "缴": "jiǎo", + "舶": "bó", + "茅": "máo", + "薪": "xīn", + "裕": "yù", + "鉗": "qián", + "鑲": "xiāng", + "钳": "qián", + "镶": "xiāng", + "韁": "jiāng", + "丁": "dīng", + "偎": "wēi", + "凄": "qī", + "凤": "fèng", + "凰": "huáng", + "叼": "diāo", + "姆": "mǔ", + "尿": "niào", + "弦": "xián", + "惕": "tì", + "惧": "jù", + "懼": "jù", + "挎": "kuà", + "撅": "juē", + "杜": "dù", + "桨": "jiǎng", + "槳": "jiǎng", + "樟": "zhāng", + "欧": "ōu", + "歐": "ōu", + "淒": "qī", + "淳": "chún", + "渺": "miǎo", + "珊": "shān", + "瑚": "hú", + "痒": "yǎng", + "瞥": "piē", + "砰": "pēng", + "硝": "xiāo", + "祸": "huò", + "禍": "huò", + "稚": "zhì", + "糙": "cāo", + "紳": "shēn", + "绅": "shēn", + "羽": "yǔ", + "舔": "tiǎn", + "葵": "kuí", + "蚓": "yǐn", + "蚯": "qiū", + "豺": "chái", + "郑": "zhèng", + "鄭": "zhèng", + "鐺": "dāng", + "铛": "dāng", + "鳳": "fèng", + "鵑": "juān", + "鹃": "juān", + "伶": "líng", + "咀": "jǔ", + "噪": "zào", + "嚼": "jué", + "娛": "yú", + "娱": "yú", + "屠": "tú", + "怔": "zhēng", + "惩": "chéng", + "懲": "chéng", + "捐": "juān", + "捶": "chuí", + "撩": "liāo", + "枚": "méi", + "枢": "shū", + "槛": "kǎn", + "樞": "shū", + "檻": "kǎn", + "漩": "xuán", + "碟": "dié", + "秤": "chèng", + "竽": "yú", + "笆": "bā", + "篱": "lí", + "籬": "lí", + "臣": "chén", + "茎": "jīng", + "莖": "jīng", + "蚜": "yá", + "蝙": "biān", + "蝠": "fú", + "褂": "guà", + "詭": "guǐ", + "诡": "guǐ", + "豹": "bào", + "賀": "hè", + "贺": "hè", + "蹄": "tí", + "鈾": "yóu", + "錘": "chuí", + "鍬": "qiāo", + "铀": "yóu", + "锤": "chuí", + "锹": "qiāo", + "雹": "báo", + "霞": "xiá", + "呃": "è", + "咕": "gū", + "哑": "yǎ", + "哺": "bǔ", + "唬": "hǔ", + "唾": "tuò", + "啞": "yǎ", + "嗅": "xiù", + "嗐": "hài", + "嘀": "dí", + "嘶": "sī", + "尸": "shī", + "屎": "shǐ", + "悠": "yōu", + "惋": "wǎn", + "愕": "è", + "抿": "mǐn", + "掷": "zhì", + "搔": "sāo", + "搪": "táng", + "擲": "zhì", + "旷": "kuàng", + "曠": "kuàng", + "桅": "wéi", + "梨": "lí", + "榕": "róng", + "泣": "qì", + "泵": "bèng", + "涕": "tì", + "猬": "wèi", + "瑩": "yíng", + "疆": "jiāng", + "矗": "chù", + "硅": "guī", + "綴": "zhuì", + "繭": "jiǎn", + "缀": "zhuì", + "腮": "sāi", + "芯": "xīn", + "茧": "jiǎn", + "莹": "yíng", + "蕉": "jiāo", + "蕴": "yùn", + "藤": "téng", + "蘊": "yùn", + "蝟": "wèi", + "誣": "wū", + "誦": "sòng", + "諷": "fěng", + "讽": "fěng", + "诬": "wū", + "诵": "sòng", + "販": "fàn", + "贩": "fàn", + "蹭": "cèng", + "鈷": "gǔ", + "鋸": "jù", + "鎂": "měi", + "鐮": "lián", + "钴": "gǔ", + "锯": "jù", + "镁": "měi", + "镰": "lián", + "隘": "ài", + "髦": "máo", + "鱷": "è", + "鳄": "è", + "鸝": "lí", + "鹂": "lí", + "丸": "wán", + "伊": "yī", + "勒": "lēi", + "勘": "kān", + "匙": "shi", + "卑": "bēi", + "吉": "jí" + } +} diff --git a/assets/template.source.html b/assets/template.source.html new file mode 100644 index 00000000..d9e2b74f --- /dev/null +++ b/assets/template.source.html @@ -0,0 +1,246 @@ + + + + + + + + +Distilly · 个人画像 + + + + + +
+
+

+ + Distilly · Person View +

+

个人画像

+

等待 view.json 数据

+

+
+ + + + +
+

+ +

+
+ +
+
+
+ +
+

本页为单文件离线产物：内联 CSS/JS/SVG，无外部请求（CSP default-src 'none'）。

+ +
+
+ + + + + + + + diff --git a/bin/distilly.mjs b/bin/distilly.mjs index 7460a406..f5330ab8 100755 --- a/bin/distilly.mjs +++ b/bin/distilly.mjs @@ -1,204 +1,226 @@ #!/usr/bin/env node - -import { - cpSync, - existsSync, - mkdirSync, - readFileSync, - renameSync, - rmSync, -} from "node:fs"; -import { homedir } from "node:os"; -import { basename, dirname, join, parse, resolve } from "node:path"; +/** + * Distilly entry point. + * + * Contract: `docs/v2/CONTRACT.md` §1 — this is the only user-facing entry. It + * parses global flags, resolves a subcommand through the registry in + * `src/commands/index.mjs`, prints a bilingual help screen, and always answers + * with the receipt shape from §3 when `--json` is set. + * + * Adding a command: create `src/commands/.mjs`, call `register(...)` from + * it, and import that module below. See `docs/v2/NODE-CORE.md`. + */ + +import { existsSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -const packageRoot = fileURLToPath(new URL("..", import.meta.url)); -const packageMetadata = JSON.parse( - readFileSync(join(packageRoot, "package.json"), "utf8"), -); - -const payloadEntries = [ +import { ArgError, wantsHelp } from "../src/cli/args.mjs"; +import { isEntryPoint } from "../src/cli/entry.mjs"; +import { CliError, createReceipt, createReporter } from "../src/cli/receipt.mjs"; +// Importing the registry also registers every built-in command module. +import { + lookup, + missingCommandError, + renderCommandHelp, + renderHelp, + resolveCommand, +} from "../src/commands/index.mjs"; + +export const packageRoot = fileURLToPath(new URL("..", import.meta.url)); +const packageMetadata = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")); +const version = packageMetadata.version; +const binary = "distilly"; + +/** + * Every path the published package must contain. `src/` and `assets/` are not + * optional: this file imports `../src/cli/args.mjs` at startup and the viewer + * template is read from `assets/`. Kept in sync with `package.json`'s `files` + * by `validatePayload`, which refuses to pack when they disagree. + */ +export const payloadEntries = [ "SKILL.md", "prompts", "references", - "tools", - "requirements.txt", + "bin", + "src", + "assets", + "scripts", + "package.json", "INSTALL.md", "INSTALL_EN.md", "LICENSE", "CITATION.cff", ]; -const hosts = { - "claude-code": () => join(homedir(), ".claude", "skills", "distilly"), - openclaw: () => - join(homedir(), ".openclaw", "workspace", "skills", "distilly"), - hermes: () => - join(homedir(), ".hermes", "skills", "openclaw-imports", "distilly"), - codex: () => join(homedir(), ".agents", "skills", "distilly"), - "deepseek-harness": () => - join(process.env.DSH_HOME || join(homedir(), ".dsh"), "skills", "distilly"), - pi: () => join(homedir(), ".pi", "agent", "skills", "distilly"), - "grok-build": () => join(homedir(), ".grok", "skills", "distilly"), - opencode: () => - join(homedir(), ".config", "opencode", "skills", "distilly"), -}; - -const aliases = { - claude: "claude-code", - deepseek: "deepseek-harness", - grok: "grok-build", -}; - -function printHelp() { - console.log(`Distilly ${packageMetadata.version} - -Install the Distilly creator Skill into a supported agent host. - -Usage: - distilly install [--force] - distilly install --path [--force] - -Hosts: - claude-code, openclaw, hermes, codex, deepseek-harness, - pi, grok-build, opencode - -Options: - --force Preserve an existing install as a timestamped backup, then install - --path Install to a custom path whose final directory is named distilly - --version Print the package version - --help Show this help -`); -} - -function fail(message) { - console.error(`Error: ${message}`); - process.exit(1); +/** npm always includes these whatever `files` says, so they need no pattern. */ +const ALWAYS_PACKED = new Set(["package.json", "README.md", "LICENSE", "LICENCE"]); + +/** + * Does at least one `files` pattern put `entry` into the tarball? + * + * npm's `files` accepts bare names (`SKILL.md`), directories (`src/`) and globs + * (`prompts/**`); a directory pattern covers everything below it. Only the + * shapes this manifest actually uses are supported — an unrecognised pattern is + * reported as "does not cover", never silently treated as a match. + */ +function packs(entry, patterns) { + if (ALWAYS_PACKED.has(entry)) return true; + return patterns.some((raw) => { + const pattern = String(raw).replace(/\/+$/, ""); + if (pattern === entry) return true; + if (entry.startsWith(`${pattern}/`)) return true; + if (!pattern.includes("*")) return false; + const source = pattern + .split("**") + .map((part) => + part + .split("*") + .map((literal) => literal.replace(/[.+?^${}()|[\]\\]/g, "\\$&")) + .join("[^/]*"), + ) + .join(".*"); + return new RegExp(`^${source}$`).test(entry); + }); } -function validatePayload() { - const missing = payloadEntries.filter( - (entry) => !existsSync(join(packageRoot, entry)), - ); +/** + * Prepack guard: the published payload must be complete and version-consistent. + * + * This runs from `prepack`, so it must judge the tarball npm is about to build, + * not the working tree. Checking only `root` is what let a broken package ship: + * `files` still listed the Python-era `tools/` and `requirements.txt` and had + * dropped `src/` and `assets/`, `--check-package` printed "payload is valid" + * because the repo tree had everything, and the extracted tarball could not even + * start. So the manifest is checked too. + */ +export function validatePayload(root = packageRoot) { + const missing = payloadEntries.filter((entry) => !existsSync(join(root, entry))); if (missing.length > 0) { - fail(`package payload is missing: ${missing.join(", ")}`); + throw new CliError(`package payload is missing: ${missing.join(", ")}`, { + code: "payload-incomplete", + remedy: "restore the missing paths or update payloadEntries in bin/distilly.mjs.", + }); } - const skill = readFileSync(join(packageRoot, "SKILL.md"), "utf8"); - if (!skill.includes(`version: "${packageMetadata.version}"`)) { - fail("package.json version does not match SKILL.md"); + const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); + const patterns = Array.isArray(manifest.files) ? manifest.files : []; + if (patterns.length === 0) { + throw new CliError("package.json declares no `files`, so the tarball would be unpredictable", { + code: "payload-manifest", + remedy: "add a `files` array listing bin/, src/, assets/, scripts/ and the documents.", + }); } -} -function expandHome(inputPath) { - if (inputPath === "~") return homedir(); - if (inputPath.startsWith("~/")) return join(homedir(), inputPath.slice(2)); - return inputPath; -} - -function validateTarget(inputPath) { - const target = resolve(expandHome(inputPath)); - const parsed = parse(target); - if (target === parsed.root || target === resolve(homedir())) { - fail("refusing to install into a filesystem root or home directory"); - } - if (basename(target) !== "distilly") { - fail("the install path must end with a directory named distilly"); + const absent = patterns + .map((raw) => String(raw).replace(/\/+$/, "")) + .filter((pattern) => !existsSync(join(root, pattern))); + if (absent.length > 0) { + throw new CliError(`package.json \`files\` names paths that do not exist: ${absent.join(", ")}`, { + code: "payload-manifest", + remedy: "remove the stale entries (or restore the paths) so the manifest describes this tree.", + }); } - return target; -} -function parseInstallArgs(args) { - let host; - let customPath; - let force = false; - - for (let index = 0; index < args.length; index += 1) { - const arg = args[index]; - if (arg === "--force") { - force = true; - } else if (arg === "--path") { - customPath = args[index + 1]; - if (!customPath) fail("--path requires a value"); - index += 1; - } else if (arg.startsWith("--")) { - fail(`unknown option: ${arg}`); - } else if (!host) { - host = aliases[arg] || arg; - } else { - fail(`unexpected argument: ${arg}`); - } + const omitted = payloadEntries.filter((entry) => !packs(entry, patterns)); + if (omitted.length > 0) { + throw new CliError(`package.json \`files\` would omit required paths: ${omitted.join(", ")}`, { + code: "payload-incomplete", + remedy: `add ${omitted.map((entry) => `"${entry}/"`).join(", ")} to \`files\`; the package cannot run without them.`, + }); } - if (customPath) return { target: validateTarget(customPath), force }; - if (!host) fail("choose a host or pass --path"); - if (!hosts[host]) fail(`unsupported host: ${host}`); - return { target: validateTarget(hosts[host]()), force }; + const skill = readFileSync(join(root, "SKILL.md"), "utf8"); + if (!skill.includes(`version: "${version}"`)) { + throw new CliError("package.json version does not match SKILL.md", { + code: "version-mismatch", + remedy: `set SKILL.md frontmatter version to "${version}" (or bump package.json).`, + }); + } } -function timestamp() { - return new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-"); +function failureReceipt(command, error) { + return createReceipt(command, { + ok: false, + error: { + code: error.code ?? "error", + message: error.message, + ...(error.remedy ? { remedy: error.remedy } : {}), + }, + warnings: [error.message], + }); } -function install(target, force) { - validatePayload(); +async function main(argv) { + // `--json` is global (CONTRACT §1): every subcommand answers with a receipt. + const json = argv.includes("--json"); + const args = argv.filter((arg) => arg !== "--json"); + const reporter = createReporter(json); + + if (args.includes("--check-package")) { + validatePayload(); + // A validation diagnostic, not command output: `prepack` shares stdout with + // `npm pack --json`, which must stay parseable. + process.stderr.write("Distilly package payload is valid.\n"); + return 0; + } - if (existsSync(target) && !force) { - fail(`${target} already exists; rerun with --force to preserve and replace it`); + // Global flags only count before a command name: `skill version rollback + // --version v1` must reach the version manager, not print the CLI version. + if (args[0] === "--version") { + reporter.line(version); + return 0; } - const parent = dirname(target); - const staging = join(parent, `.distilly-install-${process.pid}`); - let backup; + const { name, rest } = resolveCommand(args); + const command = lookup(name); - mkdirSync(parent, { recursive: true }); - if (existsSync(staging)) { - fail(`temporary install path already exists: ${staging}`); + if (args.length === 0 || args[0] === "help" || (wantsHelp(args) && !command)) { + process.stdout.write(renderHelp({ version, binary })); + return 0; } - try { - mkdirSync(staging); - for (const entry of payloadEntries) { - cpSync(join(packageRoot, entry), join(staging, entry), { - recursive: true, - preserveTimestamps: true, - }); - } - - if (!existsSync(join(staging, "SKILL.md"))) { - throw new Error("staged install does not contain SKILL.md"); - } - - if (existsSync(target)) { - backup = `${target}.backup-${timestamp()}`; - renameSync(target, backup); - } - renameSync(staging, target); - } catch (error) { - if (existsSync(staging)) { - rmSync(staging, { recursive: true, force: true }); - } - if (backup && !existsSync(target) && existsSync(backup)) { - renameSync(backup, target); - } - throw error; + if (command === null) { + throw missingCommandError(name); } - console.log(`Distilly ${packageMetadata.version} installed at ${target}`); - if (backup) console.log(`Previous install preserved at ${backup}`); + if (wantsHelp(args)) { + process.stdout.write(`${renderCommandHelp(command, { binary })}\n`); + return 0; + } + + const result = (await command.run({ + argv: rest, + json, + reporter, + ctx: { packageRoot, version, binary }, + })) ?? {}; + + const receipt = result.receipt ?? createReceipt(command.name); + reporter.finish(receipt); + if (result.exitCode !== undefined) return result.exitCode; + return receipt.ok === false ? 1 : 0; } -const args = process.argv.slice(2); -if (args.includes("--check-package")) { - validatePayload(); - console.log("Distilly package payload is valid."); -} else if (args.includes("--version")) { - console.log(packageMetadata.version); -} else if (args.length === 0 || args.includes("--help") || args[0] === "help") { - printHelp(); -} else if (args[0] === "install") { - const { target, force } = parseInstallArgs(args.slice(1)); - install(target, force); -} else { - fail(`unknown command: ${args[0]}`); +// Dispatch only when this file is the process entry point. Importing it (tests do, +// to reach `validatePayload` and `payloadEntries`) must not run a command with the +// importer's argv. `isEntryPoint` resolves symlinks, so this still fires when the +// CLI is reached through an npm `bin` shim or any other symlinked path. +if (isEntryPoint(import.meta.url)) { + try { + process.exitCode = await main(process.argv.slice(2)); + } catch (error) { + const json = process.argv.includes("--json"); + const command = process.argv.slice(2).find((arg) => !arg.startsWith("-")) ?? null; + const reporter = createReporter(json); + const failure = + error instanceof CliError || error instanceof ArgError + ? error + : new CliError(error?.message ?? String(error), { code: "unexpected" }); + + reporter.warn(`Error: ${failure.message}`); + if (failure.remedy) reporter.warn(`Remedy: ${failure.remedy}`); + reporter.finish(failureReceipt(command, failure)); + process.exitCode = failure.exitCode ?? 1; + } } diff --git a/docs/NUWA_CONTRAST.md b/docs/NUWA_CONTRAST.md new file mode 100644 index 00000000..dde160b5 --- /dev/null +++ b/docs/NUWA_CONTRAST.md @@ -0,0 +1,149 @@ +# 对照：nuwa-skill 怎么做的，我们该抄什么 + +> 起因：讨论「语料形状」与「效果怎么量」时，被要求去看 `alchaincyf/nuwa-skill` 的做法。 +> 本文只记录**跑过、读过**的事实，不转述宣传语。抓取日期 2026-09-15，仓库 `main` 分支。 + +## 0. 它是什么 + +- 形态：**纯 Skill**（`SKILL.md` + `references/` + `scripts/` + `examples/`），MIT，靠 Agent Skills 协议跑在 + Claude Code / Codex / Cursor / OpenClaw / Hermes 等 runtime。**没有 CLI、没有依赖、没有机械门禁。** +- 定位：输入一个名字 → 自动调研 → 提炼这个人的**思维框架**（不是角色扮演），产出一个 `*-perspective` Skill。 +- 它的 README 开头写明师承：[同事.skill](https://github.com/titanwings/colleague-skill)「证明了蒸馏一个人是可行的」—— + 也就是我们的前身。它把问题从「蒸馏同事」推到「蒸馏任何公众人物」。 + +## 1. 它的流程（6 个 Phase，带两个人工检查点） + +| Phase | 做什么 | 关键设计 | +|---|---|---| +| 0 分流 | 明确人名 → 直接路径；模糊需求 → 诊断路径（10 个需求维度反推人选） | 没想好蒸馏谁时，先做需求定位再推荐 | +| 0A 澄清 | 确认人名/聚焦方向/用途/新建或更新，然后问一句：**「你手上有没有这个人的一手素材？书籍 PDF、演讲/访谈 transcript、视频字幕、博客导出」** | 这一问决定采集模式 | +| 0.5 建目录 | 先建 `references/research/01..06` + `sources/` | 调研必须落盘；「不存文件的调研等于没做」 | +| 1 采集 | **6 个并行 subagent**：著作 / 对话 / 表达 / 他者批评 / 决策 / 时间线，各写一个 md，标注一手 vs 二手，矛盾保留 | 三种模式：纯网络搜索 / 本地语料优先 / 纯本地语料；本地优先时只对缺失维度补搜 | +| 1.5 检查点 | 展示来源数量表 + 矛盾点 + 信息不足维度，**等用户确认** | 「垃圾进垃圾出，在这里拦截比在 Phase 4 返工便宜」 | +| 2 提炼 | 心智模型 3-7 个 + 决策启发式 5-10 条 + 表达 DNA + 价值观与反模式 + 内在张力 + 诚实边界 | 见 §2 的三重验证 | +| 2.5 检查点 | 展示提炼摘要，**等用户确认** | 「提炼是主观判断最重的环节」 | +| 3 构建 | 按模板填 `SKILL.md`，并生成「回答工作流（Agentic Protocol）」 | 从心智模型**反推**研究维度，遇到需要事实的问题强制先联网核查 | +| 4 验证 | 子 agent 跑 3 道已知立场题 + 1 道超范围题 + 1 道风格题 | 独立于主 agent，避免自评偏差 | +| 5 精炼 | 双 agent（结构评估 + 触发条件评审）产出改进建议 | Phase 2→4 最多循环 2 次，不无限打磨 | + +## 2. 它凭什么说「这是心智模型」——三重验证 + +一个观点要被收录为心智模型，必须同时通过： + +1. **跨域复现**：在此人讨论的 ≥2 个不同领域出现； +2. **有生成力**：能用它推断此人对新问题的立场； +3. **有排他性**：不是所有聪明人都会这样想。 + +只过 1 重 → **降级**为「决策启发式」；0 重 → 丢弃。 +表达层另有量化口径：平均句长、疑问句比例、类比密度、第一人称使用率、确定性语气比例、转折频率。 + +## 3. 它怎么处理「说话人」——不处理，绕开 + +这是本次对照对我们最要紧的一条。 + +它的字幕工具链只有两个脚本（`scripts/download_subtitles.sh` 55 行、`scripts/srt_to_transcript.py` 108 行， +sha256 见 §7）。清洗逻辑是：**去掉序号行、时间戳行、HTML 标签，相邻重复行去重，短句合并成段**。 +说话人标签**原样留在正文里**，交给读文本的模型自己理解。 + +它能这么干，是因为它的语料形状天然是**一个人的一手产出**（著作、本人演讲/访谈、本人社媒）。 +多人会议根本不是它的输入。**它不是解决了说话人归属问题，是产品设计上绕开了它。** + +用我们的公开语料实跑它的脚本（`us-house-floor-2009-07-29/transcript.srt`，C-SPAN 真实字幕）： + +``` +$ python3 srt_to_transcript.py us-house.srt us-house.nuwa.txt +✅ 转换完成 字数: 36399 段落数: 825 + +原始 SRT : 84860 字符, 5430 行 +它的 transcript : 36399 字符, 825 段 +说话人标签 : 84 处 → 产物里作为普通文本保留 84 处（无结构） +产物里还有时间戳吗 : 否（时间戳去干净了） +``` + +而它的产物开头是这样的： + +``` +starttime 1248896221.592 QUORUM IS NOT PRESENT AND MAKES WITH THE UNITED STATES HOUSE OF REPRESENTATIVES. +... +>> MR. S SUSPEND THE RULES AND PASS THE , A BILL TO RESTORE THE HIGHWAY TRUST FUND ... +``` + +两处泄漏值得记下来：**字幕头字段 `starttime 1248896221.592` 被当成正文写进了语料**； +**`>>` 换人标记没被剥掉**。我们的 `parse` 层对同一份文件会把这些头字段收进 warnings 并在产物里 +报出来，而不是混进正文。我们这边同一份语料的产物是 `knowledge/text/*.md`：`[k00NN] <时间> <说话人>：文本`， +每行可回指到原文的字节区间，说话人从 84 处标签里认出 82 条记录。 + +**结论**：它的路线要的是「喂给模型读的干净语料」，我们要的是「每条结论能回指的审计链」。 +两者的取舍不同，但「一个人」这个输入形状是对的，我们此前在会议记录上优化解析器是错的。 + +## 4. 它怎么量效果——保真度评分卡 + +`references/fidelity-scorecard.md`，100 分五维： + +| 维度 | 分 | 测法 | +|---|---|---| +| 立场一致性 | 30 | 3 道人物公开表态过的问题，对比回答方向 | +| 风格辨识度 | 20 | 不看名字盲读，能否认出是谁（还是通用 AI 腔） | +| 边缘诚实度 | 20 | 1 道超范围题：标注「这是推断」=满分，伪装成本人断言 = 0 | +| 来源透明度 | 15 | **静态检查**：有来源章节、一手占比 >50%、关键引语有出处 | +| 结构完整度 | 15 | **静态检查**：心智模型 3-7 个、诚实边界 ≥3 条、内在张力 ≥2 对、反模式清单 | + +等级 A ≥85 / B 70-84 / C 55-69 / D <55。铁律：**答题 agent 与评分 agent 必须是两个独立 agent， +绝不自评自证**（其文档引用 SkillLens 论文称 LLM 自评准确率 46.4%、接近随机；该论文我们未独立核对）。 +反作弊四条：答题者不知道被测什么维度、评分者不参与答题、出题避开 skill 内已有示例对话、 +重要结论双评分 agent 且分差 >10 人工复核。15 个官方 Skill 已全部跑完并公开分数。 + +**对我们最关键的判断**：这套评分卡的「立场一致性 30 分」需要**人物的公开已知立场**作为真值， +因此**只适用于公众人物**。我们的 A/B holdout（按时间码切语料，A 给蒸馏者、B 只给裁判） +不依赖真值，是给**私人语料**（同事 / 伴侣 / 自己）设计的。两者不是替代关系： +**公开人物用评分卡，私人语料用 A/B holdout。** + +## 5. 逐项对照 + +| 维度 | nuwa-skill | 我们（dot-skill / distilly） | 谁更强 | +|---|---|---|---| +| 输入形状 | 一个人的一手产出（著作/访谈/社媒），多源 | 通道采集（chat/邮件/文档/字幕）+ 锚点账本 | 形状它更对；覆盖面我们更广 | +| 说话人 | 不处理，标签留成普通文本 | 解析层认领说话人（VTT voice / `Name:` / 全角冒号 / 句中换人），锚点带说话人 | 我们 | +| 字幕清洗 | 108 行 Python，去时间戳/序号/标签/重复行 | 结构化解析 + 字节级锚点 + 头字段进 warnings | 我们（可回指、不泄漏） | +| 提炼层 | 心智模型三重验证、决策启发式、表达 DNA、反模式、内在张力 | `derive/` 七个维度是**行为统计**（句长/标点/口头禅 n-gram/回应频次/时间线/边界/转向/冲突） | 它（认知层我们还没有） | +| 效果测量 | 评分卡五维 + 独立双 agent + 公开 FIDELITY.md | `blind-test.mjs` A/B holdout（prepare/finalize/control/score） | 各有适用域；**它已经跑出数字，我们还没跑** | +| 可验证性 | 「来源透明度」是静态检查（有没有章节、占比） | 锚点回指可机械验证到字节区间 | 我们 | +| 工程门禁 | 无 | acceptance 12 项 / 目标审计 15 行 / visual-check 8 项 / prompt-lint / 模板防漂移 / CI | 我们 | +| 分发 | 50+ runtime，一行 `npx skills add`，已产出 14 个成品 | 双入口（CLI + Skill），19 个 `ds/*` PR | 它更成熟；我们更可审计 | + +## 6. 可采纳项（按性价比排序，尚未动手） + +1. **换语料形状**（最优先、最便宜）：选一个「有自己的访谈/演讲字幕」的人，而不是从会议里切人。 + 这一步同时解锁评分卡（公开立场可查）。**与本仓库既有结论一致：单人语料的正确来源是单人产出。** +2. **移植评分卡，并加一条它做不到的机械维度**：`scripts/fidelity.mjs` —— + 来源透明度 / 结构完整度 / **锚点回指率（机械可算）** 自动出分； + 立场一致性 / 风格辨识度 / 边缘诚实度 出 rubric + 独立裁判 prompt；产出 `FIDELITY.md`。 + 两种模式并存：公众人物走评分卡，私人语料走 A/B holdout。 +3. **把三重验证做成 `derive` 的筛选门槛**：跨域复现 / 生成力 / 排他性。 + 现状反证：`voice.catchphrases` 现在输出「二级缓」「级缓存」这类话题三元组—— + 正是「没有排他性过滤」的典型症状。 +4. **抄信源纪律**：知乎 / 微信公众号 / 百度百科永远排除；中文只用权威媒体 + B站原始视频 + 小宇宙原始音频； + 「本地一手素材权重最高」；来源不足（<10 条）时提前降期望并把诚实边界写长。 + +**不采纳**：把字幕洗成纯文本（我们要付锚点可回指的代价）；去掉机械门禁（那是我们比它强的地方）。 + +## 7. 来源与复现 + +- 仓库：（MIT） +- `SKILL.md`： + （同一 skill 的镜像，字段与 alchaincyf 版一致） +- 方法论：`references/extraction-framework.md`、`references/fidelity-scorecard.md` +- 成品样例：`examples/steve-jobs-perspective/SKILL.md`（27 KB，6 心智模型 / 8 决策启发式 / 五条诚实边界， + 含「幸存者偏差」自我批评） +- 本次实跑的两个脚本（原样下载，仅存于 `/tmp`，未入库）： + `srt_to_transcript.py` sha256 `fe9de1d39243920f1c28fee519faa45c18230c67e41d1c34d4af83cdbc4f6b8b`、 + `download_subtitles.sh` sha256 `6b729f2117c552ca7a35013c0c7b2cac849db7057b7e5b6285d9a89c6ce93598` + +复现清洗对比： + +```bash +curl -sSL -o /tmp/nuwa/srt_to_transcript.py \ + https://raw.githubusercontent.com/alchaincyf/nuwa-skill/main/scripts/srt_to_transcript.py +cp tests/fixtures/public-corpus/us-house-floor-2009-07-29/transcript.srt /tmp/nuwa/us-house.srt +python3 /tmp/nuwa/srt_to_transcript.py /tmp/nuwa/us-house.srt /tmp/nuwa/us-house.nuwa.txt +``` diff --git a/docs/PRD.md b/docs/PRD.md index 2cab2d06..95f2ede4 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -1,5 +1,11 @@ # 同事.skill —— 产品需求文档 v2.0 +> ⚠️ **这份文件描述的是 v1（Python）时代的形状**：目录是 `colleagues/{slug}`、入口是 +> `tools/*.py`、实现是 Python。v2 已迁移到 Node 单栈：目录是 `skills//`、 +> 入口是 `bin/distilly.mjs`、性格层是 Layer 0–5、schema 是 v4。 +> **产品意图仍然有效，文件路径与实现细节请以 `docs/v2/CONTRACT.md` 与 `docs/v2/STATUS.md` 为准。** +> 迁移对照表见 `SKILL.md` 与 `docs/v2/MIGRATION.md`。 + --- ## 一、产品概述 diff --git a/docs/SKILL_TYPE_ABSTRACTION_DESIGN.md b/docs/SKILL_TYPE_ABSTRACTION_DESIGN.md index 0ec0c056..9924055b 100644 --- a/docs/SKILL_TYPE_ABSTRACTION_DESIGN.md +++ b/docs/SKILL_TYPE_ABSTRACTION_DESIGN.md @@ -1,5 +1,12 @@ # Skill Type Abstraction Design +> ⚠️ **This document describes the v1 (Python-era) shape**: directories are +> `colleagues/{slug}`, the entrypoint is `tools/*.py`, and the implementation is Python. +> v2 is a single Node stack: `skills//`, entrypoint `bin/distilly.mjs`, a +> Layer 0–5 persona, and schema v4. **The product intent still holds; for paths and +> implementation details follow `docs/v2/CONTRACT.md` and `docs/v2/STATUS.md`.** +> The migration table is in `SKILL.md` and `docs/v2/MIGRATION.md`. + Last updated: 2026-04-16 ## 1. Background diff --git a/docs/SKILL_TYPE_ABSTRACTION_DESIGN_ZH.md b/docs/SKILL_TYPE_ABSTRACTION_DESIGN_ZH.md index db1f1ff4..41ec53b4 100644 --- a/docs/SKILL_TYPE_ABSTRACTION_DESIGN_ZH.md +++ b/docs/SKILL_TYPE_ABSTRACTION_DESIGN_ZH.md @@ -1,5 +1,11 @@ # Skill 类型抽象设计 +> ⚠️ **这份文件描述的是 v1（Python）时代的形状**：目录是 `colleagues/{slug}`、入口是 +> `tools/*.py`、实现是 Python。v2 已迁移到 Node 单栈：目录是 `skills//`、 +> 入口是 `bin/distilly.mjs`、性格层是 Layer 0–5、schema 是 v4。 +> **产品意图仍然有效，文件路径与实现细节请以 `docs/v2/CONTRACT.md` 与 `docs/v2/STATUS.md` 为准。** +> 迁移对照表见 `SKILL.md` 与 `docs/v2/MIGRATION.md`。 + 最后更新：2026-04-16 ## 1. 背景 diff --git a/docs/evidence/pr-01-node-core.md b/docs/evidence/pr-01-node-core.md new file mode 100644 index 00000000..6797d69e --- /dev/null +++ b/docs/evidence/pr-01-node-core.md @@ -0,0 +1,53 @@ +# PR-01 · Node 单栈基座：入口 CLI + Skill 内核 + 安装器 + 拼音 + 测试移植 + +- 分支：`ds/01-node-core`（12 个提交，已本地合并进 `dot-skill-test`） +- 依赖：无。这一条是并行工作的基座：契约、命令注册表、验收脚本都由它落下来 +- 交付：38 个文件 / +11 050 行 + +## 1. 变更 + +| # | 提交 | 内容 | +| --- | --- | --- | +| 1 | `c383fca` | `src/skill/writer.mjs` + `src/skill/slug.mjs`：Python `skill_writer.py` 的 Node 移植，含拼音 slug | +| 2 | `3337c39` | `src/skill/versions.mjs`：版本归档（list / backup / rollback / cleanup），归档时间戳按 UTC 固定 | +| 3 | `579b159` | 归档列表确定性：同一天两次 `version list` 输出一致 | +| 4 | `f94675d` | `scripts/parity.mjs`：与迁移前 Python 的**逐字节 parity** 证据（按 rev 跑，不进日常门禁） | +| 5 | `441ceaf` | `src/commands/skill.mjs`：`skill create\|update\|list\|version` 接进注册表 | +| 6 | `9ebfa6e` | `src/install/hosts.mjs`：8 个宿主安装器合并成一个模块（路径矩阵单一出处） | +| 7 | `3436730` | 安装器测试移植到 `node --test`（claude / codex / openclaw / hermes） | +| 8 | `43c837e` | 已注册命令的 `--help` 打印双语两段 | +| 9 | `f6dcf87` | `listCommands()` 暴露命令名（供 doctor / prompt-lint / 审计共用） | +| 10 | `f97bc0c` | `install` / `uninstall` / `doctor` / `legacy` 适配器（旧 `python3 tools/*.py` 调用转发 + deprecation 警告） | +| 11 | `28b0c32` | `assets/pinyin.json`：从 Unihan 生成，去掉 `pypinyin` 运行时依赖 | +| 12 | `1da31aa` | 其余 Python 测试套件移植为 `node --test` | + +新增文件（节选）：`src/commands/{index,skill,install,doctor,legacy}.mjs`、`src/cli/{args,receipt}.mjs`、 +`src/skill/{writer,presets,schema,slug,versions}.mjs`、`src/install/hosts.mjs`、`src/hosts/agents.mjs`、 +`assets/pinyin.json`、`scripts/{generate-pinyin,parity,acceptance}.mjs`、`docs/v2/{CONTRACT,ACCEPTANCE,STATUS}.md`、 +`tests/{dispatcher,commands,cli-lifecycle,skill-writer,pinyin-slug,install-*}.test.mjs`、 +公开语料夹具 `tests/fixtures/public-corpus/synthetic-interview/**`。 + +## 2. 验收（当前树，可复算） + +```bash +node --test tests/dispatcher.test.mjs tests/commands.test.mjs tests/cli-lifecycle.test.mjs \ + tests/skill-writer.test.mjs tests/pinyin-slug.test.mjs tests/install-*.test.mjs +# 33 个测试文件、330 个 test() 块：node --test tests/*.test.mjs → 340 pass / 0 fail +node bin/distilly.mjs --help # 22 个命令名，全部有中英两段 +node bin/distilly.mjs doctor # 宿主矩阵 8 个宿主，逐个报告是否已安装 +node scripts/parity.mjs # 历史 parity 证据（需要旧 rev） +``` + +要点：**零运行时依赖**（`package.json` 无 dependencies）；入口唯一（`bin/distilly.mjs`）； +`--json` 在任何命令上只输出一个对象（`tests/dispatcher.test.mjs` 断言）； +命令注册表是唯一注册点，两段式命令名优先（`skill create` 赢过 `skill`）。 + +## 3. 回滚 + +- 逐提交可 revert；`src/commands/legacy.mjs` 单独 revert 会让旧 `tools/*.py` 调用直接报未知命令。 +- `assets/pinyin.json` 是生成物：`node scripts/generate-pinyin.mjs` 可重现（`--check` 防漂移）。 + +## 4. 已知缺口 + +- `scripts/parity.mjs` 需要一份迁移前的 rev 才能跑：parity 是历史证据，不是日常门禁。 +- 这一条只交付 skill/install/doctor 内核；`harvest` / `parse-*` / `view` / `collect` 由 #02/#03/#07 交付。 diff --git a/docs/evidence/pr-02-parse-zero-cred.md b/docs/evidence/pr-02-parse-zero-cred.md new file mode 100644 index 00000000..b962cd52 --- /dev/null +++ b/docs/evidence/pr-02-parse-zero-cred.md @@ -0,0 +1,59 @@ +# PR-02 · 零凭据解析：`knowledge/` 账本 + 锚点 + chat/subtitle/archive 解析 + +- 分支：`ds/02-parse-zero-cred`（6 个提交，已本地合并进 `dot-skill-test`） +- 依赖：契约（`docs/v2/CONTRACT.md`，由 #01 冻结） +- 交付：31 个文件 / +5 905 行 + +## 1. 变更 + +| # | 提交 | 内容 | +| --- | --- | --- | +| 1 | `05ff594` | 冻结 v2 契约：磁盘布局、回执形状、密钥纪律、computer-use 同意 | +| 2 | `6f95373` | `src/knowledge/store.mjs`：`knowledge/raw/` 字节保险库（逐字落盘 + 读回校验 + 路径逃逸拒绝） | +| 3 | `66bc96f` | `src/knowledge/{anchors,ledger}.mjs`：段落锚点分配、只增账本、`units`/`anchor_detail`、去重（sha256 + origin） | +| 4 | `34e20a3` | `src/parse/subtitle.mjs`：`.srt`/`.vtt` → 一条 cue 一个锚点单元（含说话人、时间码、字节区间） | +| 5 | `1fd53af` | `src/parse/common.mjs`：零依赖读共享 zip 容器（中央目录、CRC 校验、成员流式解压） | +| 6 | `6c42d88` | `src/parse/chat.mjs`：ChatGPT / Claude / Slack / Telegram / Discord / Instagram 导出，其余按名拒绝 | + +新增文件：`src/knowledge/{store,anchors,ledger}.mjs`、`src/parse/{common,chat,subtitle}.mjs`、 +`tests/{knowledge-store,knowledge-anchors,knowledge-ledger,parse-chat,parse-subtitle}.test.mjs`、 +`tests/fixtures/parse/{chat,subtitle}/**`、`.gitattributes`（夹具按字节保真，禁换行转换）。 + +## 2. 磁盘契约（这一条定下来，后面所有分支都按它写） + +``` +skills///knowledge/ + raw//… 原样字节，只增不改 + text/.md 归一化正文，段落锚点 [k0012] / [k0012:t3] + index.json 账本：{id,kind,origin,fetched_at,bytes,sha256,credentialed,method,warnings[]} +``` + +- **锚点必须能回指**：`resolveLedgerAnchor(ledger, anchor)` 返回文本与字节区间； + 容器里读出来的文本（OOXML 成员、去标签的 HTML）标 `synthetic` 且**报 null 字节区间**，不伪造偏移。 +- **幂等**：同一份字节（sha256 相同且 origin 相同）重复导入不新增条目。 +- **响亮拒绝**：不认识的格式按文件名进 warnings，不静默跳过。 + +## 3. 验收（当前树，可复算） + +```bash +node --test tests/knowledge-store.test.mjs tests/knowledge-anchors.test.mjs \ + tests/knowledge-ledger.test.mjs tests/parse-chat.test.mjs tests/parse-subtitle.test.mjs +node scripts/acceptance.mjs # harvest / 账本 / 幂等 / 锚点回指四个阶段 +``` + +`scripts/acceptance.mjs` 里与本条直接相关的断言：回执形状（sha256 + 字节数）、 +重复 harvest 幂等、账本里有锚点、派生结论锚点全部可回指。 + +## 4. 后续修复（合并后由集成轮补上，见后续 PR 文档） + +- `assignAnchorsToText` 曾把段落渲染成 `k0012 text`（无方括号），而本条的读者要求 + `[k0012] text`：派生层因此恒为空。修法与前后对比见 + `docs/evidence/pr-09-evidence-spine-blind-test.md`。 +- 归一化正文曾丢掉说话人与时间（本条的 parser 有元数据，但没进正文）：修法与前后对比见 + `docs/evidence/pr-11-attribution.md`。 + +## 5. 回滚 + +- `src/knowledge/**` 与 `src/parse/**` 是后面所有解析命令的地基：回滚本条会让 + `harvest` / `parse-*` / `retrospect` 全部失效，需回滚到 #01 的 CLI 骨架状态。 +- `.gitattributes` 的字节保真规则不要单独 revert：夹具在 CRLF 平台上会被改写，测试随之漂移。 diff --git a/docs/evidence/pr-03-render.md b/docs/evidence/pr-03-render.md new file mode 100644 index 00000000..9a42a79c --- /dev/null +++ b/docs/evidence/pr-03-render.md @@ -0,0 +1,194 @@ +# PR-03 渲染层证据（`ds/03-render`） + +纯文字证据。截图/PNG **不入库**，见 §6；本文件只记录数值、命令与结果。 + +## 1. 变更摘要 + +单文件 HTML 渲染层：模板是生成物、八段页面、双主题、默认私有、`view check` 结构化诊断、 +`view render` 确定性产物 + 回执、Playwright visual-check 八项。 + +| commit | 内容 | +|---|---| +| `feat(view): add the page template fragments and viewer runtime` | `assets/template.source.html`、`viewer/{sections,theme,export,focus}.js` | +| `feat(view): generate the single-file template from fragments` | `scripts/generate-template.mjs` + 生成物 `assets/distilly-template.html` | +| `feat(view): validate view.json against the v2 contract` | `src/views/schema.mjs`（八段/kind/锚点/confidence/12 字原文 + 诊断形状 + 宽容归一化） | +| `feat(view): render a view.json into a single-file HTML page` | `src/views/render.mjs`（payload 注入、回执、确定性、离线校验） | +| `feat(cli): register view check and view render` | `bin/distilly.mjs` 增加 `view` 子命令（最小注册，见 §8 缺口 1） | +| `chore(view): add the playwright visual check script` | `scripts/visual-check.mjs` | +| `test(view): cover schema diagnostics, rendering and template drift` | `tests/views.test.mjs`、`tests/template.test.mjs` | +| `docs(view): document the render layer contract` | `docs/v2/RENDER.md` | +| `docs(evidence): record the pr-03 render evidence` | 本文件 | + +未触碰 `src/parse/**`、`src/knowledge/**`、`src/skill/**`、`src/install/**`、`SKILL.md`、`prompts/**`； +未修改 `docs/v2/CONTRACT.md`。 + +## 2. 测试命令与结果 + +``` +$ node --test tests/views.test.mjs tests/template.test.mjs +# tests 44 +# pass 44 +# fail 0 +# duration_ms 710.41 +``` + +- `tests/views.test.mjs`（35 项）：schema 正/反例、诊断形状逐字段断言、12 字原文边界（11 字不算、12 字算）、 + `--shareable` 放行、锚点格式/回指、宽容形状归一化（含幂等）、确定性（两次渲染字节相同）、 + 私有产物不含引文、`--shareable` 产物含引文且回执记录 `inlined_sources[]`、 + CLI `view check/render --person --json` 与退出码。 +- `tests/template.test.mjs`（9 项）：模板 = 碎片构建结果（字节相同）、`--check` 干净退出 0、 + **碎片改了但模板没重生成 → 非零退出**（在临时目录里改 `viewer/sections.js` 后断言 exit 1， + 重新生成后回到 0）、marker 缺失/重复报错、`` 碎片被拒、产物单文件 + CSP + 无 `http(s)://`、 + 两次渲染字节相同。 + +## 3. 产物大小与 sha256 + +| 文件 | 字节 | sha256 | +|---|---|---| +| `assets/distilly-template.html`（生成物） | 41673 | `cb88a172694f31eb970d8f323ffefc1a13319d8aa17c84705ead959d1c738798` | +| `assets/template.source.html` | 10955 | `483189a41f196c7434aa359881a5c34a265db9d25fea3744aa788a776225f1db` | +| `viewer/sections.js` | 15585 | `80967985aee069058589ac5dff5ce4b5aada3b9af40cf6b5039753700017aaa7` | +| `viewer/theme.js` | 4073 | `3f9e73ee72a3dfbf4ca22e5b14551458b1b7251750e6740ab2db04a99bfc474a` | +| `viewer/export.js` | 6323 | `5eab79b4ceb5f3dabb5a4841c63c1f736bc3a64afdc0528b39fdb6900bccb06f` | +| `viewer/focus.js` | 4592 | `5a17e146cb6a1044ae94d9928cb5bb7198bcd1e2c672dd5d33e656e1a9456c91` | + +示例人 `zhang-san`（fixture 在 `/tmp/dst-fixture`，不入库；8 个锚点、18 条结论）： + +| 产物 | 字节 | sha256 | +|---|---|---| +| `views/zhang-san.html`（默认私有） | 48338 | `2a0ace948a6027dfec12c77e0052c5c0f035fa3742a39fedb7ec47c9e9f46db0` | +| `views/zhang-san.shareable.html`（`--shareable`） | 49244 | `782118192556b49188bfd97da733e389947dc6be0e8bb6b010f9002581b5b9e1` | +| `evidence/renders/receipt.json`（私有运行） | 1505 | `d4aea409510c0535258266b46dd0726890cce5eac6a95109c8e4cdd1ceac911d` | +| `evidence/renders/receipt.json`（shareable 运行） | 3450 | `651cdd69f131d3c6130698211a8934a9456ce8bf7340df3ed770e27901906ba4` | +| 验收语料形状 `lin-gong.html`（无 `meta.slug`/无 `evidence[]`） | 46402 | `cce9cdb8c850da34917b430ec462e8c370f6adb0ce605733f933ae1477bdba4e` | + +确定性：同一输入连续两次 `view render`，HTML 与 receipt 的 sha256 均不变（单元测试与 §4 手工各验一次）。 +私有 HTML 中检索三条引文均**不存在**；`--shareable` HTML 中三条全部存在，且 +`receipt.inlined_sources[]` 长度 8（每条含 `anchor/source/kind/path/quote_sha256/quote_bytes`）。 + +## 4. visual-check 八项（手工跑） + +命令： + +``` +DISTILLY_PLAYWRIGHT_ROOT=<含 node_modules 的目录> \ + node scripts/visual-check.mjs --out /tmp/dst-evidence/pr-03/ +``` + +对三个页面各跑一次，均 **exit 0、8/8 通过**；下面逐条列出私有示例页的结果（`--json` 回执在 `/tmp/vc-private.json`）。 + +| # | 项 | 结果 | 数值 | +|---|---|---|---| +| 1 | console 无 error/warning | ✅ | 0 条 console / 0 个 pageerror / 0 个失败请求 | +| 2 | 八段非空 | ✅ | 8 段 / 26 条 / 附录 8 锚点 | +| 3 | 无横向溢出 | ✅ | 最大溢出 0px @ 1280/768/375px | +| 4 | 双主题对比度 | ✅ | 最小对比度 6.00:1；背景亮度 light 1.0 → dark 0.0118；`light→dark` 手动切换生效 | +| 5 | 锚点可定位到附录 | ✅ | 8/8 个锚点可聚焦（含回指链接），正文引用无悬空 | +| 6 | 零网络请求（离线 + CSP） | ✅ | 0 个外部请求 / 静态外链 0 个 / CSP 存在 | +| 7 | `@media print` 不裁切 | ✅ | 8 段可见 / 正文 1450 字符（与屏幕一致）/ 溢出 0px | +| 8 | 出 PNG | ✅ | 4 张 → `/tmp/dst-evidence/pr-03` | + +另两个页面（同一套断言，逐条全绿）： + +- `legacy`（`lin-gong`，验收脚本的 `view.template.json` 形状 + 无 `evidence[]`）：8 段 / 14 条 / 7 锚点， + 最小对比度 6.00:1，0 外部请求。 +- `shareable`（`zhang-san`）：8 段 / 26 条 / 8 锚点，附录内联 8 条引文，最小对比度 6.00:1。 + +**负向对照**（证明断言真的会失败，不是恒真；输出在 `/tmp/dst-tamper-out/`）： + +| 篡改 | 退出码 | 失败项 | +|---|---|---| +| 删除 CSP meta | 1 | 零网络请求（离线 + CSP） | +| 注入 `https://` 外链 CSS | 1 | console 无 error/warning、零网络请求 | +| payload 删除时间线段 | 1 | 八段非空、锚点可定位到附录、print 不裁切 | +| 亮色 `muted` 改成近背景色 | 1 | 双主题对比度 | +| 暗色 `muted` 改成近背景色 | 1 | 双主题对比度 | +| 手动暗色覆盖失效（`color-scheme` 打回 light） | 1 | 双主题对比度 | +| 正文引用一个没有附录行的锚点 | 1 | 锚点可定位到附录 | +| 原始页面（对照组） | 0 | — | + +其中「亮/暗 muted 低对比度」与「手动覆盖失效」两组对照是发现真实缺陷后补的： +最初 `:root { color-scheme: light }` 把 `light-dark()` 钉死在亮色，导致深色页面根本没变色， +而对比度断言仍然全绿；现在 theme 断言额外比较**实际背景亮度**（light 1.0 / dark 0.0118）， +并断言手动切换必须覆盖系统偏好。 + +## 5. 其他手工验证 + +``` +$ node scripts/generate-template.mjs --check +template is up to date: sha256 cb88a172… (41673 bytes, 4 fragments) # exit 0 + +$ node bin/distilly.mjs view check zhang-san --root /tmp/dst-fixture +view check ok: 7/8 segments, 18 items, 8/8 anchors cited, 0 warning(s) + +$ node bin/distilly.mjs view check --person lin-gong --root /tmp/dst-legacy --json # 验收调用形式 +{"command":"view check","person":"lin-gong","ok":true,...,"anchors":{"total":7,"cited":7}} +``` + +- 指纹/大小一致性：`receipt.outputs[0].sha256` 与实际 HTML 文件 sha256 相同（`tests/views.test.mjs` 断言）。 +- 产物检索 `http://` / `https://` / ``); + fragments.push({ token, path: relative, sha256: sha256(body), bytes: Buffer.byteLength(body, "utf8") }); + } + + const dataMarkers = output.split(DATA_TOKEN).length - 1; + if (dataMarkers !== 1) fail(`marker ${DATA_TOKEN} must appear exactly once in template.source.html (found ${dataMarkers})`); + + // The view payload marker stays in the committed template: src/views/render.mjs is the + // only writer that replaces it. Without a payload the page shows its empty state. + const leftover = (output.replace(DATA_TOKEN, "")).match(/@@DISTILLY:[A-Z_]+@@/g); + if (leftover) fail(`unsubstituted markers remain: ${[...new Set(leftover)].join(", ")}`); + + const header = + "\n`; + output = output.replace("\n", `\n${header}`); + + validateOutput(output); + return { output, fragments }; +} + +function validateOutput(output) { + if (!output.includes(CSP)) fail("the generated template lost the frozen CSP meta tag"); + if (!output.includes('')) fail("the generated template lost "); + if ((output.match(/ element"); + const dataTag = '`)) { + fail(`the view data marker ${DATA_TOKEN} must be the whole payload placeholder`); + } + for (const pattern of ["http://", "https://", "src=\"//", "@import", "url(http"]) { + if (output.includes(pattern)) fail(`the generated template must stay offline but contains ${pattern}`); + } +} + +function parseArgs(argv) { + const options = { check: false, json: false, quiet: false }; + for (const arg of argv) { + if (arg === "--check") options.check = true; + else if (arg === "--json") options.json = true; + else if (arg === "--quiet") options.quiet = true; + else if (arg === "--help" || arg === "-h") options.help = true; + else fail(`unknown option: ${arg}`); + } + return options; +} + +function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + console.log(`Usage: node scripts/generate-template.mjs [--check] [--json] [--quiet] + + (no flags) write assets/distilly-template.html from template.source.html + viewer/*.js + --check compare the committed template with a fresh build; exit 1 on drift + --json print a receipt instead of prose`); + return 0; + } + + const { output, fragments } = buildTemplate(); + const bytes = Buffer.byteLength(output, "utf8"); + const digest = sha256(output); + const committed = existsSync(TEMPLATE_PATH) ? readFileSync(TEMPLATE_PATH, "utf8") : null; + + if (options.check) { + const drift = committed !== output; + const receipt = { + command: "template --check", + ok: !drift, + template: { path: "assets/distilly-template.html", sha256: digest, bytes, committed_sha256: committed === null ? null : sha256(committed) }, + fragments, + warnings: [], + supported_fixes: drift + ? ["run: node scripts/generate-template.mjs", "commit assets/distilly-template.html with the fragment change"] + : [], + }; + if (options.json) console.log(JSON.stringify(receipt, null, 2)); + else if (drift) { + console.error("template drift: assets/distilly-template.html is not the build of the current fragments."); + console.error(` built sha256 ${digest} (${bytes} bytes)`); + console.error(` on disk sha256 ${committed === null ? "missing" : sha256(committed)}`); + console.error(" fix: node scripts/generate-template.mjs && git add assets/distilly-template.html"); + } else if (!options.quiet) { + console.log(`template is up to date: sha256 ${digest} (${bytes} bytes, ${fragments.length} fragments)`); + } + return drift ? 1 : 0; + } + + if (committed === output) { + if (!options.quiet) console.log(`template unchanged: sha256 ${digest} (${bytes} bytes)`); + return 0; + } + + writeFileSync(TEMPLATE_PATH, output, "utf8"); + if (options.json) { + console.log( + JSON.stringify( + { + command: "template", + ok: true, + outputs: [{ path: "assets/distilly-template.html", sha256: digest, bytes }], + fragments, + warnings: [], + }, + null, + 2, + ), + ); + } else if (!options.quiet) { + console.log(`wrote assets/distilly-template.html: sha256 ${digest} (${bytes} bytes)`); + for (const entry of fragments) console.log(` ${entry.token} <- ${entry.path} (${entry.bytes} bytes)`); + } + return 0; +} + +if (process.argv[1] && isMain()) { + try { + process.exitCode = main(); + } catch (error) { + console.error(`Error: ${error.message}`); + process.exitCode = 1; + } +} + +/** True when this file is the process entry point (/tmp vs /private/tmp safe). */ +function isMain() { + try { + return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); + } catch (error) { + return false; + } +} diff --git a/scripts/parity.mjs b/scripts/parity.mjs new file mode 100644 index 00000000..5bdfbe36 --- /dev/null +++ b/scripts/parity.mjs @@ -0,0 +1,663 @@ +#!/usr/bin/env node +/** + * Byte-parity harness: pinned Python implementation vs. the Node core. + * + * The Python original is exported from git (it is deleted on this branch), so + * the comparison always runs against the frozen reference rather than a + * leftover working copy: + * + * node scripts/parity.mjs [--rev ] [--python ] [--keep] + * [--report ] + * + * Default revision: `git merge-base HEAD dot-skill-test` (the pre-port tree). + * Everything is compared byte for byte: + * A. library level — create/update/list/version operations writing the six + * artifacts plus meta.json into identical sandboxes; + * B. CLI level — `python3 tools/*.py` vs `node bin/distilly.mjs …` stdout, + * stderr and exit codes for the same command lines; + * C. pure helpers — slugify / normalize_command_slug / patch merging. + * + * The clock is frozen through `DISTILLY_PARITY_NOW` so timestamps cannot mask a + * real difference; the Python driver patches `now_iso` to the same value, and + * archived-version mtimes are pinned before they are listed. + */ + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(here, ".."); +const FROZEN_NOW = "2024-01-02T03:04:05.678901+00:00"; +const FROZEN_MTIME = Math.floor(Date.parse("2024-01-02T03:04:05Z") / 1000); + +function arg(name, fallback) { + const index = process.argv.indexOf(`--${name}`); + return index === -1 ? fallback : process.argv[index + 1]; +} +const has = (name) => process.argv.includes(`--${name}`); + +const keep = has("keep"); +const pythonExe = arg("python", process.env.PARITY_PYTHON ?? "python3"); +const reportPath = arg("report", null); + +function git(args) { + const result = spawnSync("git", args, { cwd: repoRoot, encoding: "utf8" }); + if (result.status !== 0) { + throw new Error(`git ${args.join(" ")} failed: ${result.stderr ?? result.stdout}`); + } + return result.stdout.trim(); +} + +const rev = arg("rev", null) ?? process.env.PARITY_PY_REV ?? git(["merge-base", "HEAD", "dot-skill-test"]); + +const results = []; +let failures = 0; + +function record(section, name, ok, detail = "") { + results.push({ section, name, ok, detail }); + if (!ok) failures += 1; + console.log(` ${ok ? "PASS" : "DIFF"} ${section} :: ${name}${detail ? ` — ${detail}` : ""}`); +} + +function sha256(buffer) { + return createHash("sha256").update(buffer).digest("hex"); +} + +function walk(root, base = root, into = new Map()) { + for (const entry of readdirSync(root).sort()) { + const full = join(root, entry); + if (statSync(full).isDirectory()) walk(full, base, into); + else into.set(relative(base, full), sha256(readFileSync(full))); + } + return into; +} + +function compareTrees(label, leftRoot, rightRoot) { + const left = existsSync(leftRoot) ? walk(leftRoot) : new Map(); + const right = existsSync(rightRoot) ? walk(rightRoot) : new Map(); + const names = [...new Set([...left.keys(), ...right.keys()])].sort(); + const differences = []; + for (const name of names) { + const a = left.get(name); + const b = right.get(name); + if (a === b) continue; + differences.push(`${name} [${a ? a.slice(0, 10) : "missing"} vs ${b ? b.slice(0, 10) : "missing"}]`); + } + record( + label, + `${names.length} files byte-identical`, + differences.length === 0, + differences.slice(0, 5).join("; ") + (differences.length > 5 ? ` (+${differences.length - 5} more)` : ""), + ); + return { total: names.length, differences }; +} + +function pinMtimes(root, epochSeconds = FROZEN_MTIME) { + if (!existsSync(root)) return; + for (const entry of readdirSync(root)) { + const full = join(root, entry); + if (statSync(full).isDirectory()) { + utimesSync(full, epochSeconds, epochSeconds); + pinMtimes(full, epochSeconds); + } + } +} + +const sandbox = mkdtempSync(join(tmpdir(), "dst-parity-")); +const pyRoot = join(sandbox, "python"); +const nodeRoot = join(sandbox, "node"); +mkdirSync(pyRoot, { recursive: true }); +mkdirSync(nodeRoot, { recursive: true }); + +const archive = join(sandbox, "pinned-tools.tar"); +const archiveResult = spawnSync("git", ["archive", "--format=tar", "-o", archive, rev, "tools"], { + cwd: repoRoot, + encoding: "utf8", +}); +if (archiveResult.status !== 0) throw new Error(`cannot export tools/ at ${rev}: ${archiveResult.stderr}`); +const untar = spawnSync("tar", ["-xf", archive, "-C", pyRoot], { encoding: "utf8" }); +if (untar.status !== 0) throw new Error(`cannot extract ${archive}: ${untar.stderr}`); + +console.log(`parity: pinned rev ${rev}`); +console.log(`parity: python ${pythonExe}`); +console.log(`parity: sandbox ${sandbox}\n`); + +/* ------------------------------------------------------------------ * + * Shared scenario script. Both drivers implement exactly these steps * + * and write the same files; nothing is sorted output-only. * + * ------------------------------------------------------------------ */ + +const PY_DRIVER = String.raw` +import contextlib, io, json, os, sys +from pathlib import Path + +TOOLS = Path(__file__).resolve().parent +sys.path.insert(0, str(TOOLS)) + +FIXED = os.environ["DISTILLY_PARITY_NOW"] +import skill_schema, skill_writer, version_manager # noqa: E402 + +skill_schema.now_iso = lambda: FIXED +skill_writer.now_iso = lambda: FIXED +version_manager.now_iso = lambda: FIXED + +OUT = Path(sys.argv[1]).resolve() +OUT.mkdir(parents=True, exist_ok=True) +REPORT = OUT.parent / "report" +REPORT.mkdir(parents=True, exist_ok=True) + + +def emit(name, value): + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, indent=2) + (REPORT / name).write_text(text, encoding="utf-8") + + +def capture(fn, *args, **kwargs): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer): + try: + result = fn(*args, **kwargs) + except Exception as error: + print("EXC %s: %s" % (type(error).__name__, error)) + result = None + return buffer.getvalue() + "\n<>" % (result,) + + +def new_base(name, character): + base = OUT / name / "skills" / character + base.mkdir(parents=True, exist_ok=True) + return base + + +WORK_BODY = "## mental models\n- First-principles reasoning\n- Skeptical framing\n\n## limitations\n- Avoids operational detail\n\nSources:\nhttps://example.com/a\nhttps://example.com/b\n" +PERSONA_BODY = "## expression DNA\n- Sentence rhythm is clipped.\n- Uses metaphor when disagreeing.\n\n## honest boundaries\n- States what they do not know.\n" +ZH_WORK = "## \u5de5\u4f5c\u80fd\u529b\u4f7f\u7528\u8bf4\u660e\n\n\u5f53\u7528\u6237\u8981\u6c42\u4f60\u5b8c\u6210\u4ee5\u4e0b\u4efb\u52a1\u65f6\uff0c\u4e25\u683c\u6309\u7167\u4e0a\u8ff0\u89c4\u8303\u6267\u884c\u3002\n\n\u5982\u679c\u88ab\u95ee\u5230\u804c\u8d23\u8303\u56f4\u5916\u7684\u95ee\u9898\uff0c\u4ee5\u8be5\u540c\u4e8b\u7684\u65b9\u5f0f\u56de\u5e94\uff08\u53c2\u89c1 Persona \u90e8\u5206\uff09\u3002\n" +EN_WORK = "## Scope rule\n\nIf you are asked a question outside your recorded responsibilities, respond in this colleague's style (see the Persona section).\n\n## Persona naming note\n\nKeep this documentation sentence.\n" + +# s1: colleague with rich metadata +base = new_base("s1", "colleague") +skill_writer.create_skill(base, "eulalie", { + "character": "colleague", + "display_name": "Eulalie", + "classification": {"language": "en"}, + "profile": {"company": "ByteDance", "level": "L2-1", "role": "Backend Engineer", "mbti": "INTJ"}, + "tags": {"personality": ["direct", "data-driven"], "culture": ["byte-dance-style"]}, + "knowledge_sources": ["manual-notes"], +}, WORK_BODY, PERSONA_BODY) + +# s2: relationship, Chinese chrome +base = new_base("s2", "relationship") +skill_writer.create_skill(base, "mireille", { + "character": "relationship", + "name": "Mireille", + "classification": {"language": "zh-CN"}, + "profile": {"role": "Designer"}, +}, ZH_WORK, PERSONA_BODY) + +# s3: celebrity with tags list and research dirs +base = new_base("s3", "celebrity") +skill_writer.create_skill(base, "zadie-smith", { + "character": "celebrity", + "name": "Zadie Smith", + "profile": {"identity": "Novelist", "known_for": "Essay and criticism"}, + "tags": ["literature", "essay", "public-intellectual"], + "knowledge_sources": ["interview", "essay"], +}, EN_WORK, PERSONA_BODY) + +# s4: celebrity, deep research profile, Chinese, string profile +base = new_base("s4", "celebrity") +skill_writer.create_skill(base, "xu-zhisheng", { + "character": "celebrity", + "research_profile": "budget-unfriendly", + "name": "Xu Zhisheng", + "classification": {"language": "zh-CN"}, + "profile": "\u4e2d\u56fd\u8131\u53e3\u79c0\u6f14\u5458\u3002", +}, ZH_WORK, PERSONA_BODY) + +# s5: legacy dot-skill identifiers survive +base = new_base("s5", "colleague") +skill_writer.create_skill(base, "legacy", { + "name": "Legacy", + "preset": "dot.colleague.v1", + "engine": {"name": "dot-skill"}, + "generation": {"engine": "dot-skill"}, + "artifacts": { + "combined_name": "colleague_legacy", + "work_name": "colleague_legacy_work", + "persona_name": "colleague_legacy_persona", + }, +}, "Work body\n", "Persona body\n") + +# s6: updates on s1 +skill_dir = OUT / "s1" / "skills" / "colleague" / "eulalie" +emit("update-work-patch.txt", capture(skill_writer.update_skill, skill_dir, "## new evidence\n- Adds a later example.\n")) +emit("update-correction.txt", capture(skill_writer.update_skill, skill_dir, None, None, {"scene": "disagreement", "wrong": "flatten disagreement", "correct": "surface it directly"})) +emit("update-replace-sections.txt", capture(skill_writer.update_skill, skill_dir, "## mental models\n- Replaced wholesale\n", "## expression DNA\n- Rewritten section\n")) +emit("update-multi-corrections.txt", capture(skill_writer.update_skill, skill_dir, None, None, {"persona_corrections": [ + {"scene": "\u94fa\u9648\u5904\u5883\u65f6", "wrong": "\u4e00\u4e0a\u6765\u5c31\u4e0b\u5224\u65ad", "correct": "\u5148\u628a\u5904\u5883\u8bb2\u5f97\u5f88\u666e\u901a"}, + {"scene": "\u8868\u8fbe\u7acb\u573a\u65f6", "wrong": "\u5199\u6210\u660e\u663e\u81ea\u5632\u578b", "correct": "\u548c\u89c2\u4f17\u4e00\u8d77\u627f\u8ba4"}, +]})) + +# s7: listing +emit("list-s1.txt", skill_writer.list_skills(OUT / "s1" / "skills" / "colleague")) +emit("list-missing.txt", skill_writer.list_skills(OUT / "nope" / "skills")) + +# s8: version flow +emit("version-backup.txt", capture(version_manager.backup_current_version, skill_dir)) +emit("version-list-before.txt", version_manager.list_versions(skill_dir)) +emit("version-rollback-ok.txt", capture(version_manager.rollback, skill_dir, "v1")) +emit("version-rollback-missing.txt", capture(version_manager.rollback, skill_dir, "v99")) +emit("version-rollback-traversal.txt", capture(version_manager.rollback, skill_dir, "../v1")) +emit("version-list-after.txt", version_manager.list_versions(skill_dir)) +emit("version-cleanup.txt", capture(version_manager.cleanup_old_versions, skill_dir, 2)) +emit("version-cleanup-again.txt", capture(version_manager.cleanup_old_versions, skill_dir, 10)) + +# s9: helpers and slug behaviour +for index, value in enumerate(["Zadie Smith", "\u00c9lodie", "A/B", " --A--B-- ", "!!!", "\u5468\u5947\u58a8"]): + emit("normalize-command-slug-%d.txt" % index, skill_schema.normalize_command_slug(value)) +emit("merge-append.txt", skill_writer.merge_markdown_patch("intro\n", "no headings here\n")) +emit("merge-replace.txt", skill_writer.merge_markdown_patch("intro\n\n## A\n\nold\n\n## B\n\nkeep\n", "## A\n\nnew\n")) +emit("merge-unknown-section.txt", skill_writer.merge_markdown_patch("intro\n\n## A\n\nold\n", "## Z\n\nadded\n")) +emit("work-only-zh.txt", skill_writer.work_only_content(ZH_WORK, chinese=True)) +emit("work-only-en.txt", skill_writer.work_only_content(EN_WORK, chinese=False)) +emit("validate-segments.txt", [skill_schema.validate_path_segment(value) for value in ["Zadie Smith", "\u00c9lodie"]]) +emit("validate-rejects.txt", [ + capture(skill_schema.validate_path_segment, value).strip() + for value in ["C:", "foo:bar", "CON", "nul.txt", "trailing.", "trailing ", "", ".."] +]) +`; + +const NODE_DRIVER = String.raw` +import { mkdirSync, writeFileSync, existsSync } from "node:fs"; +import { join, resolve } from "node:path"; + +import * as skillSchema from "__REPO__/src/skill/schema.mjs"; +import * as skillWriter from "__REPO__/src/skill/writer.mjs"; +import * as versionManager from "__REPO__/src/skill/versions.mjs"; + +const OUT = resolve(process.argv[2]); +mkdirSync(OUT, { recursive: true }); +const REPORT = join(OUT, "..", "report"); +mkdirSync(REPORT, { recursive: true }); + +const emit = (name, value) => { + const text = + typeof value === "string" ? value : JSON.stringify(value, null, 2); + writeFileSync(join(REPORT, name), text, "utf8"); +}; + +const capture = (fn, ...args) => { + const chunks = []; + const originalOut = process.stdout.write.bind(process.stdout); + const originalErr = process.stderr.write.bind(process.stderr); + let result; + const sink = (chunk) => { + chunks.push(String(chunk)); + return true; + }; + process.stdout.write = sink; + process.stderr.write = sink; + try { + result = fn(...args); + } catch (error) { + chunks.push("EXC " + error.name + ": " + error.message + "\n"); + result = undefined; + } finally { + process.stdout.write = originalOut; + process.stderr.write = originalErr; + } + const printed = chunks.join(""); + return printed + "\n<>"; +}; + +function require_repr(value) { + if (value === undefined) return "None"; + if (value === null) return "None"; + if (typeof value === "boolean") return value ? "True" : "False"; + if (typeof value === "number") return String(value); + return JSON.stringify(value); +} + +const newBase = (name, character) => { + const base = join(OUT, name, "skills", character); + mkdirSync(base, { recursive: true }); + return base; +}; + +const WORK_BODY = "## mental models\n- First-principles reasoning\n- Skeptical framing\n\n## limitations\n- Avoids operational detail\n\nSources:\nhttps://example.com/a\nhttps://example.com/b\n"; +const PERSONA_BODY = "## expression DNA\n- Sentence rhythm is clipped.\n- Uses metaphor when disagreeing.\n\n## honest boundaries\n- States what they do not know.\n"; +const ZH_WORK = "## \u5de5\u4f5c\u80fd\u529b\u4f7f\u7528\u8bf4\u660e\n\n\u5f53\u7528\u6237\u8981\u6c42\u4f60\u5b8c\u6210\u4ee5\u4e0b\u4efb\u52a1\u65f6\uff0c\u4e25\u683c\u6309\u7167\u4e0a\u8ff0\u89c4\u8303\u6267\u884c\u3002\n\n\u5982\u679c\u88ab\u95ee\u5230\u804c\u8d23\u8303\u56f4\u5916\u7684\u95ee\u9898\uff0c\u4ee5\u8be5\u540c\u4e8b\u7684\u65b9\u5f0f\u56de\u5e94\uff08\u53c2\u89c1 Persona \u90e8\u5206\uff09\u3002\n"; +const EN_WORK = "## Scope rule\n\nIf you are asked a question outside your recorded responsibilities, respond in this colleague's style (see the Persona section).\n\n## Persona naming note\n\nKeep this documentation sentence.\n"; + +// s1 +let base = newBase("s1", "colleague"); +skillWriter.createSkill(base, "eulalie", { + character: "colleague", + display_name: "Eulalie", + classification: { language: "en" }, + profile: { company: "ByteDance", level: "L2-1", role: "Backend Engineer", mbti: "INTJ" }, + tags: { personality: ["direct", "data-driven"], culture: ["byte-dance-style"] }, + knowledge_sources: ["manual-notes"], +}, WORK_BODY, PERSONA_BODY); + +// s2 +base = newBase("s2", "relationship"); +skillWriter.createSkill(base, "mireille", { + character: "relationship", + name: "Mireille", + classification: { language: "zh-CN" }, + profile: { role: "Designer" }, +}, ZH_WORK, PERSONA_BODY); + +// s3 +base = newBase("s3", "celebrity"); +skillWriter.createSkill(base, "zadie-smith", { + character: "celebrity", + name: "Zadie Smith", + profile: { identity: "Novelist", known_for: "Essay and criticism" }, + tags: ["literature", "essay", "public-intellectual"], + knowledge_sources: ["interview", "essay"], +}, EN_WORK, PERSONA_BODY); + +// s4 +base = newBase("s4", "celebrity"); +skillWriter.createSkill(base, "xu-zhisheng", { + character: "celebrity", + research_profile: "budget-unfriendly", + name: "Xu Zhisheng", + classification: { language: "zh-CN" }, + profile: "\u4e2d\u56fd\u8131\u53e3\u79c0\u6f14\u5458\u3002", +}, ZH_WORK, PERSONA_BODY); + +// s5 +base = newBase("s5", "colleague"); +skillWriter.createSkill(base, "legacy", { + name: "Legacy", + preset: "dot.colleague.v1", + engine: { name: "dot-skill" }, + generation: { engine: "dot-skill" }, + artifacts: { + combined_name: "colleague_legacy", + work_name: "colleague_legacy_work", + persona_name: "colleague_legacy_persona", + }, +}, "Work body\n", "Persona body\n"); + +// s6 +const skillDir = join(OUT, "s1", "skills", "colleague", "eulalie"); +emit("update-work-patch.txt", capture(skillWriter.updateSkill, skillDir, "## new evidence\n- Adds a later example.\n")); +emit("update-correction.txt", capture(skillWriter.updateSkill, skillDir, null, null, { scene: "disagreement", wrong: "flatten disagreement", correct: "surface it directly" })); +emit("update-replace-sections.txt", capture(skillWriter.updateSkill, skillDir, "## mental models\n- Replaced wholesale\n", "## expression DNA\n- Rewritten section\n")); +emit("update-multi-corrections.txt", capture(skillWriter.updateSkill, skillDir, null, null, { persona_corrections: [ + { scene: "\u94fa\u9648\u5904\u5883\u65f6", wrong: "\u4e00\u4e0a\u6765\u5c31\u4e0b\u5224\u65ad", correct: "\u5148\u628a\u5904\u5883\u8bb2\u5f97\u5f88\u666e\u901a" }, + { scene: "\u8868\u8fbe\u7acb\u573a\u65f6", wrong: "\u5199\u6210\u660e\u663e\u81ea\u5632\u578b", correct: "\u548c\u89c2\u4f17\u4e00\u8d77\u627f\u8ba4" }, +] })); + +// s7 +emit("list-s1.txt", skillWriter.listSkills(join(OUT, "s1", "skills", "colleague"))); +emit("list-missing.txt", skillWriter.listSkills(join(OUT, "nope", "skills"))); + +// s8 +emit("version-backup.txt", capture(versionManager.backupCurrentVersion, skillDir)); +emit("version-list-before.txt", versionManager.listVersions(skillDir)); +emit("version-rollback-ok.txt", capture(versionManager.rollback, skillDir, "v1")); +emit("version-rollback-missing.txt", capture(versionManager.rollback, skillDir, "v99")); +emit("version-rollback-traversal.txt", capture(versionManager.rollback, skillDir, "../v1")); +emit("version-list-after.txt", versionManager.listVersions(skillDir)); +emit("version-cleanup.txt", capture(versionManager.cleanupOldVersions, skillDir, 2)); +emit("version-cleanup-again.txt", capture(versionManager.cleanupOldVersions, skillDir, 10)); + +// s9 +["Zadie Smith", "\u00c9lodie", "A/B", " --A--B-- ", "!!!", "\u5468\u5947\u58a8"].forEach((value, index) => { + emit("normalize-command-slug-" + index + ".txt", skillSchema.normalizeCommandSlug(value)); +}); +emit("merge-append.txt", skillWriter.mergeMarkdownPatch("intro\n", "no headings here\n")); +emit("merge-replace.txt", skillWriter.mergeMarkdownPatch("intro\n\n## A\n\nold\n\n## B\n\nkeep\n", "## A\n\nnew\n")); +emit("merge-unknown-section.txt", skillWriter.mergeMarkdownPatch("intro\n\n## A\n\nold\n", "## Z\n\nadded\n")); +emit("work-only-zh.txt", skillWriter.workOnlyContent(ZH_WORK, { chinese: true })); +emit("work-only-en.txt", skillWriter.workOnlyContent(EN_WORK, { chinese: false })); +emit("validate-segments.txt", ["Zadie Smith", "\u00c9lodie"].map((value) => skillSchema.validatePathSegment(value))); +emit("validate-rejects.txt", ["C:", "foo:bar", "CON", "nul.txt", "trailing.", "trailing ", "", ".."].map((value) => + capture(skillSchema.validatePathSegment, value).trim(), +)); +`; + +/* ---------------------------- phase A ---------------------------- */ + +const pyDriverPath = join(pyRoot, "tools", "parity_driver.py"); +writeFileSync(pyDriverPath, PY_DRIVER, "utf8"); + +const nodeDriverPath = join(nodeRoot, "parity_driver.mjs"); +writeFileSync(nodeDriverPath, NODE_DRIVER.replaceAll("__REPO__", repoRoot), "utf8"); + +const env = { ...process.env, DISTILLY_PARITY_NOW: FROZEN_NOW, PYTHONDONTWRITEBYTECODE: "1" }; + +const pythonRun = spawnSync(pythonExe, [pyDriverPath, join(pyRoot, "out")], { + cwd: pyRoot, + encoding: "utf8", + env, +}); +if (pythonRun.status !== 0) { + console.error(pythonRun.stdout); + console.error(pythonRun.stderr); + throw new Error(`python driver failed with status ${pythonRun.status}`); +} + +const nodeRun = spawnSync(process.execPath, [nodeDriverPath, join(nodeRoot, "out")], { + cwd: nodeRoot, + encoding: "utf8", + env, +}); +if (nodeRun.status !== 0) { + console.error(nodeRun.stdout); + console.error(nodeRun.stderr); + throw new Error(`node driver failed with status ${nodeRun.status}`); +} + +pinMtimes(join(pyRoot, "out")); +pinMtimes(join(nodeRoot, "out")); + +compareTrees("A library", join(pyRoot, "out"), join(nodeRoot, "out")); + +/* ---------------------------- phase B ---------------------------- */ + +const CLI_STEPS = [ + { + name: "create", + python: ["tools/skill_writer.py", "--action", "create", "--character", "colleague", "--slug", "eulalie", "--name", "Eulalie", "--meta", "meta.json", "--work", "work.md", "--persona", "persona.md", "--skills-dir", "skills/colleague"], + node: ["skill", "create", "--character", "colleague", "--slug", "eulalie", "--name", "Eulalie", "--meta", "meta.json", "--work", "work.md", "--persona", "persona.md", "--skills-dir", "skills/colleague"], + }, + { + name: "create-pinyin-name", + python: ["tools/skill_writer.py", "--action", "create", "--character", "colleague", "--name", "Zadie Smith", "--skills-dir", "skills/colleague"], + node: ["skill", "create", "--character", "colleague", "--name", "Zadie Smith", "--skills-dir", "skills/colleague"], + }, + { + name: "list", + python: ["tools/skill_writer.py", "--action", "list", "--character", "colleague", "--skills-dir", "skills/colleague"], + node: ["skill", "list", "--character", "colleague", "--skills-dir", "skills/colleague"], + }, + { + name: "update", + python: ["tools/skill_writer.py", "--action", "update", "--character", "colleague", "--slug", "eulalie", "--skills-dir", "skills/colleague", "--work-patch", "patch.md", "--correction-json", "correction.json"], + node: ["skill", "update", "--character", "colleague", "--slug", "eulalie", "--skills-dir", "skills/colleague", "--work-patch", "patch.md", "--correction-json", "correction.json"], + }, + { name: "version-list", python: ["tools/version_manager.py", "--action", "list", "--slug", "eulalie", "--skills-dir", "skills/colleague"], node: ["skill", "version", "list", "--slug", "eulalie", "--skills-dir", "skills/colleague"] }, + { name: "version-backup", python: ["tools/version_manager.py", "--action", "backup", "--slug", "eulalie", "--skills-dir", "skills/colleague"], node: ["skill", "version", "backup", "--slug", "eulalie", "--skills-dir", "skills/colleague"] }, + { name: "version-rollback", python: ["tools/version_manager.py", "--action", "rollback", "--slug", "eulalie", "--version", "v1", "--skills-dir", "skills/colleague"], node: ["skill", "version", "rollback", "--slug", "eulalie", "--version", "v1", "--skills-dir", "skills/colleague"] }, + { name: "version-cleanup", python: ["tools/version_manager.py", "--action", "cleanup", "--slug", "eulalie", "--skills-dir", "skills/colleague"], node: ["skill", "version", "cleanup", "--slug", "eulalie", "--skills-dir", "skills/colleague"] }, +]; + +function seedCliSandbox(root) { + mkdirSync(join(root, "skills", "colleague"), { recursive: true }); + writeFileSync(join(root, "meta.json"), JSON.stringify({ character: "colleague", display_name: "Eulalie", classification: { language: "en" }, profile: { role: "Backend Engineer" } }, null, 2), "utf8"); + writeFileSync(join(root, "work.md"), "Work body\n", "utf8"); + writeFileSync(join(root, "persona.md"), "Persona body\n", "utf8"); + writeFileSync(join(root, "patch.md"), "## Update\n\nPatched section.\n", "utf8"); + writeFileSync(join(root, "correction.json"), JSON.stringify({ scene: "review", wrong: "hedge", correct: "state the risk plainly" }), "utf8"); +} + +const cliPy = join(sandbox, "cli-python"); +const cliNode = join(sandbox, "cli-node"); +for (const root of [cliPy, cliNode]) { + rmSync(root, { recursive: true, force: true }); + mkdirSync(root, { recursive: true }); + seedCliSandbox(root); +} + +const cliEnv = { ...env, DISTILLY_AUTO_INSTALL_CLAUDE: "0", DOT_SKILL_AUTO_INSTALL_CLAUDE: "0" }; +const cliTranscript = { python: [], node: [] }; + +for (const step of CLI_STEPS) { + // Archive mtimes are minute-precision in the listing; pin them before listing. + if (step.name === "version-list" || step.name === "version-cleanup") { + pinMtimes(join(cliPy, "skills", "colleague", "eulalie", "versions")); + pinMtimes(join(cliNode, "skills", "colleague", "eulalie", "versions")); + } + const py = spawnSync(pythonExe, step.python, { cwd: cliPy, encoding: "utf8", env: cliEnv }); + const js = spawnSync(process.execPath, [join(repoRoot, "bin", "distilly.mjs"), ...step.node], { + cwd: cliNode, + encoding: "utf8", + env: cliEnv, + }); + const pyOut = `status=${py.status}\n--- stdout ---\n${py.stdout}--- stderr ---\n${py.stderr}`; + const jsOut = `status=${js.status}\n--- stdout ---\n${js.stdout}--- stderr ---\n${js.stderr}`; + cliTranscript.python.push(`### ${step.name}\n${pyOut}`); + cliTranscript.node.push(`### ${step.name}\n${jsOut}`); + record( + "B cli", + step.name, + pyOut === jsOut, + pyOut === jsOut ? "" : `exit ${py.status}/${js.status}; first diff at ${firstDifference(pyOut, jsOut)}`, + ); +} + +record( + "B cli", + "installed tree byte-identical", + JSON.stringify([...walk(join(cliPy, "skills")).keys()].sort()) === + JSON.stringify([...walk(join(cliNode, "skills")).keys()].sort()), +); +compareTrees("B cli", join(cliPy, "skills"), join(cliNode, "skills")); + +function firstDifference(left, right) { + const limit = Math.min(left.length, right.length); + for (let index = 0; index < limit; index += 1) { + if (left[index] !== right[index]) { + return `char ${index}: ${JSON.stringify(left.slice(Math.max(0, index - 20), index + 20))} vs ${JSON.stringify(right.slice(Math.max(0, index - 20), index + 20))}`; + } + } + return left.length === right.length ? "identical" : `length ${left.length} vs ${right.length}`; +} + +/* ---------------------------- phase C ---------------------------- */ + +const slugNames = ["Zadie Smith", "\u00C9lodie", "A/B", "Zhou Qimo", "Mireille"]; +const pySlug = spawnSync( + pythonExe, + [ + "-c", + [ + "import sys, json", + `sys.path.insert(0, ${JSON.stringify(join(pyRoot, "tools"))})`, + "import skill_writer", + "names = json.loads(sys.argv[1])", + "out = {}", + "for name in names:", + " try:", + " out[name] = skill_writer.slugify(name)", + " except Exception as error:", + " out[name] = 'EXC ' + type(error).__name__", + "print(json.dumps(out, ensure_ascii=False, sort_keys=True))", + ].join("\n"), + JSON.stringify(slugNames), + ], + { cwd: pyRoot, encoding: "utf8", env }, +); + +let pypinyinAvailable = false; +const pypinyinProbe = spawnSync(pythonExe, ["-c", "import pypinyin"], { encoding: "utf8", env }); +pypinyinAvailable = pypinyinProbe.status === 0; + +if (pySlug.status === 0) { + const pythonSlugs = JSON.parse(pySlug.stdout); + const { slugify } = await import(join(repoRoot, "src/skill/writer.mjs")); + const nodeSlugs = {}; + for (const name of slugNames) { + try { + nodeSlugs[name] = slugify(name); + } catch (error) { + nodeSlugs[name] = `EXC ${error.name}`; + } + } + for (const name of slugNames) { + record( + "C slugify", + `${name} → ${pythonSlugs[name]}`, + pythonSlugs[name] === nodeSlugs[name] || !pypinyinAvailable, + pythonSlugs[name] === nodeSlugs[name] + ? "" + : `node: ${nodeSlugs[name]}${pypinyinAvailable ? "" : " (python ran without pypinyin)"}`, + ); + } +} else { + record("C slugify", "python slugify probe", false, pySlug.stderr.trim().split("\n")[0]); +} + +/* ---------------------------- report ---------------------------- */ + +const summary = { + rev, + python: pythonExe, + pypinyinAvailable, + frozenNow: FROZEN_NOW, + checks: results.length, + failures, + results, +}; + +if (reportPath) { + writeFileSync( + reportPath, + [ + `# parity report`, + ``, + `- pinned rev: \`${rev}\``, + `- python: \`${pythonExe}\` (pypinyin available: ${pypinyinAvailable})`, + `- frozen clock: \`${FROZEN_NOW}\``, + `- checks: ${results.length}, failures: ${failures}`, + ``, + `| section | check | result | detail |`, + `| --- | --- | --- | --- |`, + ...results.map((r) => `| ${r.section} | ${r.name} | ${r.ok ? "OK" : "DIFF"} | ${r.detail.replaceAll("|", "\\|")} |`), + ``, + `Raw transcript (${keep ? sandbox : "sandbox removed"}):`, + ``, + "```text", + ...results.map((r) => `${r.ok ? "PASS" : "DIFF"} ${r.section} :: ${r.name} ${r.detail}`), + "```", + "", + ].join("\n"), + "utf8", + ); +} + +if (keep) console.log(`\nsandbox kept: ${sandbox}`); +else rmSync(sandbox, { recursive: true, force: true }); + +console.log(`\nparity: ${results.length - failures}/${results.length} checks passed`); +process.exitCode = failures === 0 ? 0 : 1; diff --git a/scripts/prompt-lint.mjs b/scripts/prompt-lint.mjs new file mode 100644 index 00000000..594ba5a9 --- /dev/null +++ b/scripts/prompt-lint.mjs @@ -0,0 +1,362 @@ +#!/usr/bin/env node +/** + * Prompt contract lint (zero dependencies). + * + * Scans SKILL.md and prompts/**.md and enforces the prompt-layer contract + * documented in docs/v2/PROMPTS.md: + * + * command every `distilly ` must exist in docs/v2/CONTRACT.md §1 + * sections every prompt (and SKILL.md) has 必须/禁止/回执 + MUST/MUST NOT/RECEIPT + * bilingual a `## English` half exists and both halves name the same commands + * anchor anchors are [k00NN] / [k00NN:tM] + * forbidden no shell HTTP client, no Python HTTP library usage, + * no literal credential assignment, no `sk-` shaped key + * deprecated every tools/**\/*.py|sh reference is marked deprecated nearby + * + * Usage: + * node scripts/prompt-lint.mjs [--root ] [--json] + * + * Exit codes: 0 = clean, 1 = findings, 2 = the lint itself could not run + * (missing contract / unparsable command block) — never silently pass. + */ + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { join, relative, resolve, sep } from "node:path"; + +const RULE_ORDER = [ + "command", + "sections", + "bilingual", + "anchor", + "forbidden", + "deprecated", +]; + +const ANCHOR_TEMPLATES = new Set(["[k00NN]", "[k00NN:tM]"]); +const ANCHOR_VALID = /^\[k\d{4}(:t\d+)?\]$/; +const ANCHOR_CANDIDATE = /\[[kK][0-9:tNMT]+\]/g; +const COMMAND_USE = /distilly[ \t]+([A-Za-z][A-Za-z0-9-]*)/g; + +const FORBIDDEN = [ + { + name: "shell-http-client", + pattern: /\bcurl\b/i, + message: "shell HTTP client must not be used; collect through `distilly collect`", + }, + { + name: "python-http-library", + pattern: + /\bimport\s+requests\b|\bfrom\s+requests\b|\brequests\.(get|post|put|patch|delete|head|options|request|Session)\b/, + message: "hand-written API calls are forbidden; use `distilly collect`", + }, + { + name: "literal-credential", + pattern: + /(?:api[_-]?key|apikey|app[_-]?secret|client[_-]?secret|access[_-]?token|refresh[_-]?token|auth[_-]?token|password|passwd|secret|token)["'`]?\s*[:=]\s*["'`][^"'`\s]{3,}["'`]/i, + message: "a credential literal is written down; read it from ~/.distilly/*_config.json or env", + }, + { + name: "key-prefix", + pattern: + /\bsk-(?:proj-|ant-|live-|test-|or-)?[A-Za-z0-9]{20,}\b|\bsk-(?:x{3,}|\.{3,}|<[^>]+>|\$\{[^}]+\}|YOUR|your|PLACEHOLDER|placeholder|redacted|REDACTED|abc|1234)/, + message: "a key-shaped literal is written down; never put credentials in prompts", + }, +]; + +const ZERO_DEP_MARKER = "distilly"; + +function fail(message) { + process.stderr.write(`prompt-lint: ${message}\n`); + process.exit(2); +} + +function parseArgs(argv) { + const options = { root: null, json: false, help: false }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--root") { + options.root = argv[index + 1]; + if (!options.root) fail("--root requires a directory"); + index += 1; + } else if (arg === "--json") { + options.json = true; + } else if (arg === "--help" || arg === "-h") { + options.help = true; + } else { + fail(`unknown option: ${arg}`); + } + } + return options; +} + +/** Command names come from the frozen contract, never from a second list. */ +export function parseContractCommands(contractText) { + const lines = contractText.split("\n"); + const headingIndex = lines.findIndex((line) => /^##\s+1\./.test(line)); + if (headingIndex < 0) return null; + let fenceStart = -1; + for (let index = headingIndex; index < lines.length; index += 1) { + if (/^```/.test(lines[index])) { + fenceStart = index; + break; + } + } + if (fenceStart < 0) return null; + let fenceEnd = -1; + for (let index = fenceStart + 1; index < lines.length; index += 1) { + if (/^```/.test(lines[index])) { + fenceEnd = index; + break; + } + } + if (fenceEnd < 0) return null; + + const commands = new Set(); + for (const raw of lines.slice(fenceStart + 1, fenceEnd)) { + const line = raw.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const first = line.split(/\s+/)[0]; + if (/^[a-z][a-z0-9-]*$/.test(first)) commands.add(first); + // `install | uninstall` and `view check | view render` keep the + // second bare word; angle-bracket groups are placeholders, not commands. + const withoutPlaceholders = line.replace(/<[^>]*>/g, " "); + for (const match of withoutPlaceholders.matchAll(/\|\s*([a-z][a-z0-9-]*)\b/g)) { + commands.add(match[1]); + } + } + return commands.size > 0 ? commands : null; +} + +function englishMarkerIndex(lines) { + return lines.findIndex((line) => /^##\s+English\b/.test(line)); +} + +function headingMissing(text, marker) { + const pattern = new RegExp(`^#{2,3}\\s+${marker}\\s*$`, "m"); + return !pattern.test(text); +} + +export function lintText({ path, text, commands, isPrompt }) { + const findings = []; + const lines = text.split("\n"); + const add = (line, rule, message) => + findings.push({ file: path, line: line + 1, rule, message }); + + // --- command names ------------------------------------------------------- + lines.forEach((line, index) => { + for (const match of line.matchAll(COMMAND_USE)) { + const command = match[1]; + if (!commands.has(command)) { + add( + index, + "command", + `unknown command \`distilly ${command}\`; not in docs/v2/CONTRACT.md §1 (${[...commands].sort().join(", ")})`, + ); + } + } + }); + + // --- forbidden tokens ---------------------------------------------------- + lines.forEach((line, index) => { + for (const rule of FORBIDDEN) { + if (rule.pattern.test(line)) { + add(index, "forbidden", `${rule.name}: ${rule.message}`); + } + } + }); + + // --- anchors ------------------------------------------------------------- + lines.forEach((line, index) => { + for (const match of line.matchAll(ANCHOR_CANDIDATE)) { + const token = match[0]; + if (ANCHOR_VALID.test(token) || ANCHOR_TEMPLATES.has(token)) continue; + add( + index, + "anchor", + `malformed anchor ${token}; use [k00NN] or [k00NN:tM]`, + ); + } + }); + + // --- deprecated legacy tooling ------------------------------------------ + const nonEmpty = lines.map((line) => line.trim().length > 0); + lines.forEach((line, index) => { + if (!/tools\/[\w./-]+\.(py|sh)/.test(line)) return; + let start = index; + while (start > 0 && nonEmpty[start - 1]) start -= 1; + let end = index; + while (end < lines.length - 1 && nonEmpty[end + 1]) end += 1; + const paragraph = lines.slice(start, end + 1).join("\n"); + if (!/deprecated/i.test(paragraph)) { + add( + index, + "deprecated", + "legacy tools/... reference is not marked deprecated in its paragraph", + ); + } + }); + + // --- bilingual split ----------------------------------------------------- + const marker = englishMarkerIndex(lines); + const zhText = marker < 0 ? text : lines.slice(0, marker).join("\n"); + const enText = marker < 0 ? "" : lines.slice(marker).join("\n"); + if (marker < 0) { + add(0, "bilingual", "missing `## English` section (中文段 → --- → ## English)"); + } else { + const separator = lines + .slice(Math.max(0, marker - 4), marker) + .some((line) => /^---\s*$/.test(line)); + if (!separator) { + add(marker, "bilingual", "`## English` must be preceded by a `---` separator"); + } + const zhCommands = new Set( + [...zhText.matchAll(COMMAND_USE)].map((match) => match[1]), + ); + const enCommands = new Set( + [...enText.matchAll(COMMAND_USE)].map((match) => match[1]), + ); + for (const command of [...zhCommands].sort()) { + if (!enCommands.has(command)) { + add( + marker, + "bilingual", + `command \`distilly ${command}\` appears only in the Chinese half`, + ); + } + } + for (const command of [...enCommands].sort()) { + if (!zhCommands.has(command)) { + add( + marker, + "bilingual", + `command \`distilly ${command}\` appears only in the English half`, + ); + } + } + } + + // --- required sections --------------------------------------------------- + if (isPrompt || path === "SKILL.md") { + const at = marker < 0 ? 0 : marker; + for (const markerText of ["必须", "禁止", "回执"]) { + if (headingMissing(zhText, markerText)) { + add(at, "sections", `missing Chinese \`## ${markerText}\` section`); + } + } + for (const markerText of ["MUST", "MUST NOT", "RECEIPT"]) { + if (headingMissing(enText, markerText)) { + add(at, "sections", `missing English \`## ${markerText}\` section`); + } + } + } + + return findings; +} + +function collectTargets(root) { + const targets = []; + const skill = join(root, "SKILL.md"); + if (existsSync(skill)) targets.push({ path: "SKILL.md", isPrompt: false }); + + const promptsRoot = join(root, "prompts"); + const walk = (dir) => { + if (!existsSync(dir)) return; + for (const entry of readdirSync(dir).sort()) { + const full = join(dir, entry); + const stats = statSync(full); + if (stats.isDirectory()) { + walk(full); + } else if (entry.endsWith(".md")) { + targets.push({ + path: relative(root, full).split(sep).join("/"), + isPrompt: true, + }); + } + } + }; + walk(promptsRoot); + return targets; +} + +export function runLint(root) { + const contractPath = join(root, "docs", "v2", "CONTRACT.md"); + if (!existsSync(contractPath)) { + fail(`contract not found at ${contractPath}`); + } + const commands = parseContractCommands(readFileSync(contractPath, "utf8")); + if (!commands) { + fail(`could not parse the §1 command block from ${contractPath}`); + } + + const targets = collectTargets(root); + if (targets.length === 0) { + fail(`no SKILL.md or prompts/**/*.md found under ${root}`); + } + + const findings = []; + for (const target of targets) { + const text = readFileSync(join(root, target.path), "utf8"); + findings.push( + ...lintText({ path: target.path, text, commands, isPrompt: target.isPrompt }), + ); + } + + findings.sort( + (left, right) => + left.file.localeCompare(right.file) || + left.line - right.line || + RULE_ORDER.indexOf(left.rule) - RULE_ORDER.indexOf(right.rule) || + left.message.localeCompare(right.message), + ); + + return { + commands: [...commands].sort(), + files: targets.map((target) => target.path), + findings, + }; +} + +function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + process.stdout.write( + [ + "prompt-lint — prompt contract lint (zero dependencies)", + "", + "Usage: node scripts/prompt-lint.mjs [--root ] [--json]", + "", + `Contract commands: parsed from docs/v2/CONTRACT.md §1 at run time (${ZERO_DEP_MARKER}).`, + "", + ].join("\n"), + ); + return; + } + + const root = resolve( + options.root ?? fileURLToPath(new URL("..", import.meta.url)), + ); + const result = runLint(root); + + if (options.json) { + process.stdout.write( + `${JSON.stringify({ root, ok: result.findings.length === 0, ...result }, null, 2)}\n`, + ); + } else { + for (const finding of result.findings) { + process.stdout.write( + `${finding.file}:${finding.line} ${finding.rule} ${finding.message}\n`, + ); + } + const filesWithFindings = new Set(result.findings.map((f) => f.file)).size; + process.stdout.write( + `prompt-lint: ${result.findings.length} finding(s) in ${filesWithFindings} file(s) across ${result.files.length} file(s) scanned (contract commands: ${result.commands.length})\n`, + ); + } + + if (result.findings.length > 0) process.exit(1); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/scripts/skill-artifacts.mjs b/scripts/skill-artifacts.mjs new file mode 100644 index 00000000..ec197d01 --- /dev/null +++ b/scripts/skill-artifacts.mjs @@ -0,0 +1,89 @@ +/** + * The mechanical checks an acceptance run applies to a **generated Skill**. + * + * They live in their own module, and take plain strings, so they can be falsified by + * a unit test: a gate nobody has ever seen go red is not a gate. That was the state + * of this repo until now — the end-to-end acceptance covered harvest → retrospect → + * view → render and never touched `skill create`, so the artifact this product + * actually delivers could be missing, or could contradict itself, with every gate + * green. + * + * The three failure modes these checks exist for, all observed for real: + * + * - `SKILL.md` absent, or missing PART A / PART B / its operating rules; + * - a persona without the Layer 0–5 structure `prompts/persona_builder.md` defines — + * the shipped operating rules promise "Layer 0 rules always take priority" while + * nothing generated or checked that a Layer 0 existed; + * - an anchor the artifact cites that the ledger does not declare (a citation that + * cannot be followed back to the corpus, which is the product's whole claim). + * + * Distillation *quality* is deliberately not here: it needs a judge, and lives in the + * effect layer (`scripts/blind-test.mjs`). + */ + +/** The artifact set `distilly skill create` promises for a character family. */ +export const REQUIRED_ARTIFACTS = [ + 'SKILL.md', + 'work.md', + 'persona.md', + 'work_skill.md', + 'persona_skill.md', + 'manifest.json', + 'meta.json', +]; + +/** Sections the assembled `SKILL.md` must carry. */ +export const REQUIRED_SECTIONS = ['PART A', 'PART B', 'Operating Rules']; + +/** Layer headings the persona builder defines. */ +export const REQUIRED_LAYERS = [0, 1, 2, 3, 4, 5].map((index) => `Layer ${index}`); + +/** + * Whether `body` carries `Layer N` as a **heading**. + * + * Not a substring test: the operating rules that ship with every generated Skill + * contain the sentence "Layer 0 rules in PART B always take priority", so + * `body.includes("Layer 0")` is true even when PART B has no Layer 0 at all — which + * is exactly the shape that shipped green. Requiring `##` makes the promise text + * unable to satisfy the check it promises. + */ +function hasLayerHeading(body, index) { + return new RegExp(`^##\\s*Layer\\s*${index}\\b`, "m").test(body); +} + +/** Anchors an artifact cites, as `k00NN` / `k00NN:tM`, deduplicated. */ +export function citedAnchors(body) { + const found = new Set(); + for (const match of String(body ?? '').matchAll(/\[(k\d{4}(?::t\d+)?)\]/g)) found.add(match[1]); + return found; +} + +/** + * Inspect a generated Skill. + * + * @param {Record} files artifact name → body (`null` when absent) + * @param {Set} knownAnchors anchors the ledger declares + */ +export function inspectSkillArtifacts(files, knownAnchors = new Set()) { + const missingArtifacts = REQUIRED_ARTIFACTS.filter((name) => files[name] === undefined || files[name] === null); + + const skillBody = files['SKILL.md'] ?? ''; + const missingSections = REQUIRED_SECTIONS.filter((section) => !skillBody.includes(section)); + const missingLayers = REQUIRED_LAYERS.filter((_layer, index) => !hasLayerHeading(skillBody, index)); + + // Layer 0 must say something: the heading alone is what a non-compliant persona + // produces when the builder prompt is skipped, and it is exactly the shape that + // shipped once already. + const layer0Body = skillBody.split('## Layer 0')[1]?.split('## Layer 1')[0] ?? ''; + const layer0Rules = layer0Body + .split('\n') + .filter((row) => row.trim().startsWith('- ') && row.trim().length > 4).length; + + const cited = new Set(); + for (const name of ['SKILL.md', 'work.md', 'persona.md']) { + for (const anchor of citedAnchors(files[name])) cited.add(anchor); + } + const dangling = [...cited].filter((anchor) => !knownAnchors.has(anchor)); + + return { missingArtifacts, missingSections, missingLayers, layer0Rules, cited: [...cited], dangling }; +} diff --git a/scripts/split-corpus.mjs b/scripts/split-corpus.mjs new file mode 100644 index 00000000..9c7fe1d4 --- /dev/null +++ b/scripts/split-corpus.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +/** + * Split a corpus by time into the A (distillation) and B (held-out) halves that + * `docs/v2/ACCEPTANCE.md` requires, and write a receipt so the split cannot be + * quietly re-rolled after seeing the results. + * + * node scripts/split-corpus.mjs --in --out [--ratio 0.7] + * + * Subtitles are split on the cue timeline (B starts where A stops). Anything else + * is split by paragraph count, and the receipt records `by: "paragraphs"` so the + * weaker guarantee is visible. + */ + +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { basename, extname, join } from "node:path"; + +import { SourceFile } from "../src/parse/common.mjs"; +import { detectSubtitleFormat, formatTimecode, splitCues } from "../src/parse/subtitle.mjs"; + +function arg(name, fallback) { + const index = process.argv.indexOf(`--${name}`); + return index === -1 ? fallback : process.argv[index + 1]; +} + +const input = arg("in"); +const outDir = arg("out"); +const ratio = Number(arg("ratio", "0.7")); +if (!input || !outDir) { + console.error("usage: node scripts/split-corpus.mjs --in --out [--ratio 0.7]"); + process.exit(2); +} +if (!(ratio > 0.1 && ratio < 0.95)) { + console.error(`ratio must be between 0.1 and 0.95 (got ${ratio})`); + process.exit(2); +} + +const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex"); +const extension = extname(input).toLowerCase(); +const raw = readFileSync(input); +const source = new SourceFile({ path: basename(input), raw }); + +mkdirSync(outDir, { recursive: true }); + +let by; +let aText; +let bText; +let aMeta; +let bMeta; +let cut = null; + +if (extension === ".srt" || extension === ".vtt") { + const { format } = detectSubtitleFormat(source); + const { cues } = splitCues(source, format); + if (cues.length < 4) throw new Error(`need at least four cues to split ${input}`); + const total = cues[cues.length - 1].end; + const cutAt = total * ratio; + let boundary = cues.findIndex((cue) => cue.start >= cutAt); + if (boundary < 2) boundary = 2; + if (boundary > cues.length - 2) boundary = cues.length - 2; + + const render = (list) => + list + .map((cue, index) => { + const body = cue.speaker ? `${cue.speaker}：${cue.text}` : cue.text; + return `${index + 1}\n${formatTimecode(cue.start)} --> ${formatTimecode(cue.end)}\n${body}\n`; + }) + .join("\n"); + + const a = cues.slice(0, boundary); + const b = cues.slice(boundary); + aText = render(a); + bText = render(b); + // The vocabulary is "timecode" everywhere else (`cut.timecode`, the subtitle + // parser's `meta.timecodes`), so the receipt says the same thing. + by = "timecode"; + cut = { + index: boundary, + timecode: formatTimecode(cues[boundary].start), + a_end_timecode: formatTimecode(a[a.length - 1].end), + }; + aMeta = { cues: a.length, start: formatTimecode(a[0].start), end: formatTimecode(a[a.length - 1].end) }; + bMeta = { cues: b.length, start: formatTimecode(b[0].start), end: formatTimecode(b[b.length - 1].end) }; +} else { + const paragraphs = source.text.split(/\n{2,}/).map((part) => part.trim()).filter(Boolean); + if (paragraphs.length < 4) throw new Error(`need at least four paragraphs to split ${input}`); + const boundary = Math.min(Math.max(Math.round(paragraphs.length * ratio), 1), paragraphs.length - 1); + aText = `${paragraphs.slice(0, boundary).join("\n\n")}\n`; + bText = `${paragraphs.slice(boundary).join("\n\n")}\n`; + by = "paragraphs"; + cut = { index: boundary, timecode: null, a_end_timecode: null }; + aMeta = { paragraphs: boundary }; + bMeta = { paragraphs: paragraphs.length - boundary }; +} + +const aPath = join(outDir, `A${extension || ".txt"}`); +const bPath = join(outDir, `B${extension || ".txt"}`); +writeFileSync(aPath, aText, "utf8"); +writeFileSync(bPath, bText, "utf8"); + +const receipt = { + source: input, + split_by: by, + ratio, + cut, + a: { file: basename(aPath), sha256: sha256(aText), bytes: Buffer.byteLength(aText), ...aMeta }, + b: { file: basename(bPath), sha256: sha256(bText), bytes: Buffer.byteLength(bText), ...bMeta }, +}; +writeFileSync(join(outDir, "split.json"), `${JSON.stringify(receipt, null, 2)}\n`, "utf8"); + +console.log(`split (${by}, ratio ${ratio}) → ${aPath} / ${bPath}`); +console.log(` A: ${JSON.stringify(aMeta)}`); +console.log(` B: ${JSON.stringify(bMeta)}`); +console.log(" keep B away from the distiller and the judge; only the checker reads it."); diff --git a/scripts/visual-check.mjs b/scripts/visual-check.mjs new file mode 100644 index 00000000..21048c75 --- /dev/null +++ b/scripts/visual-check.mjs @@ -0,0 +1,542 @@ +#!/usr/bin/env node +/** + * Is this anchor's outcome a problem? + * + * Every appendix row must exist, be focusable and be visible; only an anchor the + * prose cites must also carry a back-link. Pure so it can be unit-tested without a + * browser. + */ +export function anchorProblem(outcome, cited) { + if (!outcome || outcome.ok !== true) return true; + if (outcome.inAppendix !== true) return true; + if (outcome.focused !== true) return true; + if (outcome.visible !== true) return true; + if (cited === true && !(outcome.backLinks >= 1)) return true; + return false; +} + +/** + * distilly visual-check — open a rendered view page in Chrome and assert the + * eight visual contracts from docs/v2/CONTRACT.md §4: + * + * 1 console is silent (no error/warning, no pageerror, no failed request) + * 2 the eight page segments exist and are non-empty + * 3 no horizontal overflow (1280 / 768 / 375 px) + * 4 dual-theme contrast spot checks (system preference + manual toggle) + * 5 every evidence anchor resolves to a focusable row in the appendix + * 6 zero network requests, CSP present, no external reference + * 7 @media print does not clip or drop content + * 8 PNG evidence is written to --out + * + * playwright is a DEVELOPMENT dependency and is never imported by the runtime: + * when it is missing this script fails loudly with install guidance. + * + * node scripts/visual-check.mjs views/.html [--out ] [--json] + */ +import { isEntryPoint } from "../src/cli/entry.mjs"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, statSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const ROOT = fileURLToPath(new URL("..", import.meta.url)); +const DEFAULT_OUT = "/tmp/dst-evidence/pr-03"; +const RESULTS = []; + +const SAMPLE_SELECTORS = [ + "#page-title", + ".claim__text", + ".claim__meta", + ".anchor-ref", + ".badge", + ".warning__text", + ".evidence__anchor", +]; + +function parseArgs(argv) { + const options = { html: null, out: DEFAULT_OUT, json: false }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--json") options.json = true; + else if (arg === "--out" || arg === "--out-dir") { + const value = argv[index + 1]; + if (!value) throw new Error(`${arg} requires a directory`); + options.out = value; + index += 1; + } else if (arg === "--help" || arg === "-h") options.help = true; + else if (arg.startsWith("--")) throw new Error(`unknown option: ${arg}`); + else if (!options.html) options.html = arg; + else throw new Error(`unexpected argument: ${arg}`); + } + return options; +} + +function usage() { + console.log(`Usage: node scripts/visual-check.mjs [--out ] [--json] + + a page produced by: distilly view render + --out PNG output directory (default ${DEFAULT_OUT}; never committed) + --json print the machine-readable result + +Exit code 0 only when all eight checks pass.`); +} + +/** playwright is a dev dependency: resolve it from the usual places, else fail loudly. */ +async function loadChromium() { + const roots = [process.env.DISTILLY_PLAYWRIGHT_ROOT, ROOT, process.cwd()].filter(Boolean); + for (const root of roots) { + try { + const require = createRequire(join(root, "index.cjs")); + const resolved = require.resolve("playwright"); + const mod = await import(pathToFileURL(resolved).href); + const chromium = mod.chromium ?? mod.default?.chromium; + if (chromium) return chromium; + } catch (error) { + /* try the next root */ + } + } + try { + const mod = await import("playwright"); + const chromium = mod.chromium ?? mod.default?.chromium; + if (chromium) return chromium; + } catch (error) { + /* fall through to the loud failure below */ + } + console.error("Error: the visual check needs playwright, which is a development dependency."); + console.error(" npm install --no-save playwright # or: pnpm add -D playwright"); + console.error(" DISTILLY_PLAYWRIGHT_ROOT= node scripts/visual-check.mjs "); + console.error(" distilly itself has zero runtime dependencies; nothing else needs playwright."); + process.exit(2); +} + +async function launch(chromium) { + try { + return await chromium.launch({ channel: "chrome" }); + } catch (error) { + return chromium.launch(); + } +} + +function record(id, name, ok, detail) { + RESULTS.push({ id, name, ok: Boolean(ok), detail }); + return Boolean(ok); +} + +/** Contrast of a node against its nearest opaque ancestor background, WCAG 2.x ratio. */ +function contrastProbe(selectors) { + const parse = (value) => { + const match = /rgba?\(([^)]+)\)/.exec(value || ""); + if (!match) return null; + const parts = match[1].split(/[\s,/]+/).filter(Boolean).map(Number); + return { r: parts[0], g: parts[1], b: parts[2], a: parts.length > 3 ? parts[3] : 1 }; + }; + const luminance = ({ r, g, b }) => { + const channel = (value) => { + const scaled = value / 255; + return scaled <= 0.03928 ? scaled / 12.92 : ((scaled + 0.055) / 1.055) ** 2.4; + }; + return 0.2126 * channel(r) + 0.7152 * channel(g) + 0.0722 * channel(b); + }; + const background = (node) => { + let current = node; + while (current && current.nodeType === 1) { + const colour = parse(getComputedStyle(current).backgroundColor); + if (colour && colour.a > 0.5) return colour; + current = current.parentElement; + } + return { r: 255, g: 255, b: 255, a: 1 }; + }; + const ratio = (a, b) => { + const first = luminance(a); + const second = luminance(b); + return (Math.max(first, second) + 0.05) / (Math.min(first, second) + 0.05); + }; + + const samples = []; + for (const selector of selectors) { + const node = document.querySelector(selector); + if (!node) { + samples.push({ selector, missing: true }); + continue; + } + const style = getComputedStyle(node); + const foreground = parse(style.color); + const behind = background(node); + samples.push({ + selector, + fontSize: Number.parseFloat(style.fontSize), + ratio: foreground ? Number(ratio(foreground, behind).toFixed(2)) : null, + foreground: style.color, + background: `rgb(${behind.r}, ${behind.g}, ${behind.b})`, + }); + } + return { theme: document.documentElement.getAttribute("data-theme-effective"), samples }; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + usage(); + return 0; + } + if (!options.html) { + usage(); + return 2; + } + const htmlPath = resolve(options.html); + if (!existsSync(htmlPath)) { + console.error(`Error: rendered page not found: ${htmlPath}`); + console.error(" fix: distilly view render (or pass the path of an existing views/.html)"); + return 2; + } + const outDir = resolve(options.out); + mkdirSync(outDir, { recursive: true }); + const url = pathToFileURL(htmlPath).href; + const ready = () => page.waitForFunction(() => document.documentElement.getAttribute("data-view-ready") === "true", null, { timeout: 15000 }); + + const chromium = await loadChromium(); + const browser = await launch(chromium); + const context = await browser.newContext({ + viewport: { width: 1280, height: 900 }, + deviceScaleFactor: 1, + colorScheme: "light", + }); + const page = await context.newPage(); + + const consoleMessages = []; + const pageErrors = []; + const failedRequests = []; + const requests = []; + page.on("console", (message) => { + if (message.type() === "error" || message.type() === "warning") { + consoleMessages.push({ type: message.type(), text: message.text() }); + } + }); + page.on("pageerror", (error) => pageErrors.push(String(error && error.message ? error.message : error))); + page.on("requestfailed", (request) => failedRequests.push({ url: request.url(), error: request.failure()?.errorText ?? null })); + page.on("request", (request) => requests.push({ url: request.url(), type: request.resourceType() })); + + const pngs = []; + const screenshot = async (name) => { + const file = join(outDir, name); + await page.screenshot({ path: file, fullPage: true }); + pngs.push({ file, bytes: statSync(file).size }); + }; + + try { + await page.goto(url, { waitUntil: "load" }); + await ready(); + + /* 1 — console silence ------------------------------------------------ */ + record( + "console", + "console has no error/warning, no page error, no failed request", + consoleMessages.length === 0 && pageErrors.length === 0 && failedRequests.length === 0, + { messages: consoleMessages, pageErrors, failedRequests }, + ); + + /* 2 — eight non-empty segments --------------------------------------- */ + const segments = await page.evaluate(() => { + const rows = [...document.querySelectorAll("[data-section]")].map((node) => ({ + id: node.getAttribute("data-section"), + chars: (node.textContent || "").trim().length, + items: node.querySelectorAll(".claim, .warning, .timeline__item, .evidence").length, + })); + const view = window.DistillyView || {}; + return { + rows, + payloadAnchors: view.view && Array.isArray(view.view.evidence) ? view.view.evidence.length : 0, + appendixAnchors: document.querySelectorAll('[data-section="evidence"] .evidence[data-anchor]').length, + shareable: Boolean(view.shareable), + quotesRendered: document.querySelectorAll(".quote[data-inlined]").length, + }; + }); + const emptySegments = segments.rows.filter((entry) => entry.chars < 8); + record( + "segments", + "the eight page segments exist and are non-empty", + segments.rows.length === 8 && emptySegments.length === 0 && segments.appendixAnchors > 0, + { + count: segments.rows.length, + empty: emptySegments.map((entry) => entry.id), + rows: segments.rows, + shareable: segments.shareable, + quotesRendered: segments.quotesRendered, + }, + ); + + /* 3 — no horizontal overflow ---------------------------------------- */ + const overflow = []; + for (const width of [1280, 768, 375]) { + await page.setViewportSize({ width, height: 900 }); + const measured = await page.evaluate(() => { + const limit = window.innerWidth + 1; + const offenders = []; + for (const node of document.querySelectorAll("body *")) { + const rect = node.getBoundingClientRect(); + if (rect.width > 0 && rect.right > limit) { + offenders.push({ + tag: node.tagName.toLowerCase(), + cls: String(node.className || "").slice(0, 60), + right: Math.round(rect.right), + }); + } + } + return { delta: document.documentElement.scrollWidth - window.innerWidth, offenders: offenders.slice(0, 5) }; + }); + overflow.push({ width, delta: measured.delta, offenders: measured.offenders }); + } + await page.setViewportSize({ width: 1280, height: 900 }); + record( + "overflow", + "no horizontal overflow at 1280/768/375 px", + overflow.every((entry) => entry.delta <= 1), + overflow, + ); + + /* 4 — dual theme contrast ------------------------------------------- */ + const themeRuns = []; + // `emulateMedia` resolves as soon as the emulation is applied; the page learns + // about it through a `matchMedia` change event and updates + // `data-theme-effective` a tick later. Probing immediately recorded the + // *previous* theme — which is how this gate went red roughly one run in two + // with byte-identical HTML (same sha256), and why the toggle then saw + // `before: "dark"` after being put back into light mode. + const settle = (scheme) => + page + .waitForFunction( + (expected) => { + const current = document.documentElement.getAttribute("data-theme-effective"); + return current === null || current === expected; + }, + scheme, + { timeout: 2000 }, + ) + .catch(() => {}); + await page.emulateMedia({ colorScheme: "light" }); + await settle("light"); + themeRuns.push(await page.evaluate(contrastProbe, SAMPLE_SELECTORS)); + await page.emulateMedia({ colorScheme: "dark" }); + await settle("dark"); + themeRuns.push(await page.evaluate(contrastProbe, SAMPLE_SELECTORS)); + await page.emulateMedia({ colorScheme: "light" }); + await settle("light"); + const toggle = await page.evaluate(() => { + const button = document.getElementById("theme-toggle"); + if (!button) return { ok: false, reason: "no #theme-toggle button" }; + const before = document.documentElement.getAttribute("data-theme-effective"); + button.click(); + return { ok: null, before, after: null, pressed: button.getAttribute("aria-pressed"), label: button.textContent }; + }); + // The click is handled by the page, so its effect is also a tick away. + toggle.after = await page + .waitForFunction( + (before) => document.documentElement.getAttribute("data-theme-effective") !== before, + toggle.before, + { timeout: 2000 }, + ) + .then(() => page.evaluate(() => document.documentElement.getAttribute("data-theme-effective"))) + .catch(() => page.evaluate(() => document.documentElement.getAttribute("data-theme-effective"))); + toggle.pressed = await page.evaluate(() => document.getElementById("theme-toggle")?.getAttribute("aria-pressed") ?? null); + toggle.ok = toggle.before === "light" && toggle.after === "dark"; + themeRuns.push(await page.evaluate(contrastProbe, SAMPLE_SELECTORS)); + const contrastFailures = []; + for (const run of themeRuns) { + for (const sample of run.samples) { + if (sample.missing) contrastFailures.push({ ...sample, theme: run.theme, reason: "sample element missing" }); + else if (sample.ratio !== null && sample.ratio < 4.5 && sample.fontSize < 24) { + contrastFailures.push({ ...sample, theme: run.theme, reason: "contrast below 4.5:1" }); + } + } + } + record( + "theme", + "dual theme (system + manual) with >= 4.5:1 contrast samples", + contrastFailures.length === 0 && toggle.ok === true && new Set(themeRuns.map((run) => run.theme)).size >= 2, + { runs: themeRuns, toggle, failures: contrastFailures }, + ); + await page.emulateMedia({ colorScheme: "light" }); + await page.evaluate(() => { + const button = document.getElementById("theme-toggle"); + if (button && document.documentElement.getAttribute("data-theme-effective") === "dark") button.click(); + }); + + /* 5 — anchors resolve into the appendix ------------------------------ */ + const anchorIds = await page.evaluate(() => + [...document.querySelectorAll('[data-section="evidence"] .evidence[data-anchor]')].map((node) => node.getAttribute("data-anchor")), + ); + const anchorProblems = []; + for (const anchor of anchorIds) { + await page.evaluate((id) => { + window.location.hash = `#anchor-${id}`; + }, anchor); + await page.waitForTimeout(40); + const outcome = await page.evaluate((id) => { + const node = document.getElementById(`anchor-${id}`); + if (!node) return { id, ok: false, reason: "no element with that id" }; + const rect = node.getBoundingClientRect(); + return { + id, + ok: true, + inAppendix: Boolean(node.closest('[data-section="evidence"]')), + focused: document.activeElement === node, + visible: rect.top < window.innerHeight && rect.bottom > 0, + backLinks: node.querySelectorAll('a[href^="#section-"]').length, + }; + }, anchor); + if (!outcome.ok || !outcome.inAppendix || !outcome.focused || !outcome.visible || outcome.backLinks === 0) { + anchorProblems.push(outcome); + } + } + await page.evaluate(() => { + try { + window.history.replaceState(null, "", window.location.pathname); + } catch (error) { + window.location.hash = ""; + } + }); + record( + "anchors", + "each evidence anchor locates a focusable row in the appendix", + anchorIds.length > 0 && anchorIds.length === segments.payloadAnchors && anchorProblems.length === 0, + { anchors: anchorIds.length, payloadAnchors: segments.payloadAnchors, problems: anchorProblems }, + ); + + /* 6 — zero network requests ----------------------------------------- */ + const staticRefs = await page.evaluate(() => { + const csp = document.querySelector('meta[http-equiv="Content-Security-Policy"]'); + return { + csp: csp ? csp.getAttribute("content") : null, + externalLinks: document.querySelectorAll('link[href]:not([href^="data:"])').length, + externalScripts: document.querySelectorAll("script[src]").length, + externalImages: document.querySelectorAll('img[src]:not([src^="data:"])').length, + embeds: document.querySelectorAll("iframe, object, embed").length, + urls: (document.documentElement.outerHTML.match(/https?:\/\/[^\s"'<>]+/g) || []).filter( + (value) => !value.includes("www.w3.org"), + ), + }; + }); + const externalRequests = requests.filter( + (entry) => !entry.url.startsWith("file:") && !entry.url.startsWith("data:") && !entry.url.startsWith("blob:"), + ); + record( + "offline", + "zero network requests, frozen CSP present, no external reference", + externalRequests.length === 0 && + Boolean(staticRefs.csp && staticRefs.csp.includes("default-src 'none'")) && + staticRefs.externalLinks === 0 && + staticRefs.externalScripts === 0 && + staticRefs.externalImages === 0 && + staticRefs.embeds === 0 && + staticRefs.urls.length === 0, + { requests: requests.length, externalRequests, staticRefs }, + ); + + /* 7 — print media does not clip -------------------------------------- */ + await page.emulateMedia({ media: "print" }); + const printReport = await page.evaluate(() => { + const nodes = [...document.querySelectorAll("[data-section]")]; + const clipped = []; + let sectionText = 0; + for (const node of nodes) { + const style = getComputedStyle(node); + sectionText += (node.textContent || "").length; + if (style.display === "none" || style.visibility === "hidden") { + clipped.push({ id: node.getAttribute("data-section"), reason: "hidden in print" }); + continue; + } + if (node.scrollWidth > node.clientWidth + 2) { + clipped.push({ id: node.getAttribute("data-section"), reason: "horizontal clip", scrollWidth: node.scrollWidth, clientWidth: node.clientWidth }); + } + if (node.scrollHeight > node.clientHeight + 2) { + clipped.push({ id: node.getAttribute("data-section"), reason: "vertical clip", scrollHeight: node.scrollHeight, clientHeight: node.clientHeight }); + } + } + return { segments: nodes.length, clipped, sectionText, overflow: document.documentElement.scrollWidth - window.innerWidth }; + }); + await screenshot("view-print.png"); + await page.emulateMedia({ media: "screen" }); + const screenReport = await page.evaluate(() => { + const nodes = [...document.querySelectorAll("[data-section]")]; + let sectionText = 0; + for (const node of nodes) sectionText += (node.textContent || "").length; + return { sectionText, segments: nodes.length }; + }); + record( + "print", + "@media print does not clip or drop content", + printReport.clipped.length === 0 && + printReport.segments === 8 && + printReport.overflow <= 1 && + printReport.sectionText === screenReport.sectionText, + { ...printReport, screenSectionText: screenReport.sectionText, screenSegments: screenReport.segments }, + ); + + /* 8 — PNG evidence --------------------------------------------------- */ + await page.goto(`${url}?theme=light`, { waitUntil: "load" }); + await ready(); + await page.setViewportSize({ width: 1280, height: 900 }); + await screenshot("view-light.png"); + await page.goto(`${url}?theme=dark`, { waitUntil: "load" }); + await ready(); + await screenshot("view-dark.png"); + // Three PNGs: print, light, dark. The 375 px capture was dropped on request — + // the narrow-viewport *check* stays (it is check 3, at 1280/768/375), so a + // layout that breaks on a phone still fails; only the picture goes away. + record( + "png", + "PNG evidence written to the output directory (print + light + dark)", + pngs.length >= 3 && pngs.every((entry) => entry.bytes > 1024), + { outDir, pngs }, + ); + + const failed = RESULTS.filter((entry) => !entry.ok); + const payload = { + command: "visual-check", + ok: failed.length === 0, + html: htmlPath, + html_sha256: createHash("sha256").update(readFileSync(htmlPath)).digest("hex"), + html_bytes: statSync(htmlPath).size, + out_dir: outDir, + shareable: segments.shareable, + appendix_anchors: segments.appendixAnchors, + checks: RESULTS, + pngs, + failed: failed.map((entry) => entry.id), + checks_passed: RESULTS.length - failed.length, + checks_total: RESULTS.length, + }; + + if (options.json) console.log(JSON.stringify(payload, null, 2)); + else { + for (const entry of RESULTS) { + console.log(`${entry.ok ? "PASS" : "FAIL"} ${entry.id.padEnd(9)} ${entry.name}`); + if (!entry.ok) console.log(` detail: ${JSON.stringify(entry.detail)}`); + } + console.log(` input: ${htmlPath} (${payload.html_bytes} bytes, sha256 ${payload.html_sha256})`); + } + console.log( + failed.length === 0 + ? `visual-check: PASS — ${RESULTS.length}/8 checks, PNGs in ${outDir} (${pngs.map((entry) => entry.file.split("/").pop()).join(", ")})` + : `visual-check: FAIL — ${failed.length}/${RESULTS.length} checks failed: ${failed.map((entry) => entry.id).join(", ")}`, + ); + return failed.length === 0 ? 0 : 1; + } finally { + await context.close(); + await browser.close(); + } +} + +// Only run when invoked directly. `tests/visual-check-rule.test.mjs` imports this +// module for its checks, and an unguarded `main()` launched a browser and printed +// the usage text during that import — which made the whole test *file* fail rather +// than any single assertion in it. +if (isEntryPoint(import.meta.url)) { + try { + process.exitCode = await main(); + } catch (error) { + console.error(`Error: ${error && error.message ? error.message : error}`); + process.exitCode = 1; + } +} diff --git a/src/cli/args.mjs b/src/cli/args.mjs new file mode 100644 index 00000000..30a4da19 --- /dev/null +++ b/src/cli/args.mjs @@ -0,0 +1,103 @@ +/** + * Minimal argument parser for the Distilly CLI (zero dependencies). + * + * Mirrors the subset of Python's argparse behaviour the ported tools relied on: + * `--flag value`, `--flag=value`, boolean switches, repeated options and + * positionals. Unknown options are a hard error — the CLI never guesses. + */ + +/** Raised for user-facing argument problems (exit code 1). */ +export class ArgError extends Error { + constructor(message) { + super(message); + this.name = "ArgError"; + } +} + +/** + * @typedef {object} OptionSpec + * @property {'boolean'|'string'} type + * @property {string} [alias] short flag without dashes, e.g. `o` + * @property {string} [help] + * @property {boolean} [multiple] collect repeats into an array + * @property {string} [value] metavar shown in help + */ + +function optionNames(longName, spec) { + const names = [`--${longName}`]; + if (spec.alias) names.push(`-${spec.alias}`); + return names; +} + +/** + * @param {string[]} argv + * @param {Record} spec + * @returns {{flags: Record, positionals: string[]}} + */ +export function parseArgs(argv, spec = {}) { + const byName = new Map(); + for (const [longName, option] of Object.entries(spec)) { + for (const name of optionNames(longName, option)) byName.set(name, longName); + } + + const flags = {}; + for (const [longName, option] of Object.entries(spec)) { + if (option.multiple) flags[longName] = []; + else if (option.type === "boolean") flags[longName] = false; + else flags[longName] = undefined; + } + + const positionals = []; + let index = 0; + while (index < argv.length) { + const arg = argv[index]; + if (arg === "--") { + positionals.push(...argv.slice(index + 1)); + break; + } + if (arg.startsWith("-") && arg !== "-") { + const equals = arg.indexOf("="); + const name = equals === -1 ? arg : arg.slice(0, equals); + const inlineValue = equals === -1 ? undefined : arg.slice(equals + 1); + const longName = byName.get(name); + if (!longName) throw new ArgError(`unrecognized argument: ${name}`); + const option = spec[longName]; + if (option.type === "boolean") { + if (inlineValue !== undefined) { + throw new ArgError(`${name} does not take a value`); + } + flags[longName] = true; + } else { + const value = inlineValue !== undefined ? inlineValue : argv[index + 1]; + if (value === undefined || (inlineValue === undefined && value.startsWith("-") && value !== "-")) { + throw new ArgError(`${name} requires a value`); + } + if (inlineValue === undefined) index += 1; + if (option.multiple) flags[longName].push(value); + else flags[longName] = value; + } + index += 1; + continue; + } + positionals.push(arg); + index += 1; + } + + return { flags, positionals }; +} + +/** True when the argument list asks for help. */ +export function wantsHelp(argv) { + return argv.includes("--help") || argv.includes("-h"); +} + +/** Render ` ` fragments for a usage line. */ +export function usageFragment(spec = {}) { + const parts = []; + for (const [longName, option] of Object.entries(spec)) { + if (option.hidden) continue; + const bare = option.alias ? `-${option.alias}, --${longName}` : `--${longName}`; + parts.push(option.type === "boolean" ? `[${bare}]` : `[${bare} <${option.value ?? "value"}>]`); + } + return parts.join(" "); +} diff --git a/src/cli/entry.mjs b/src/cli/entry.mjs new file mode 100644 index 00000000..24dff7bd --- /dev/null +++ b/src/cli/entry.mjs @@ -0,0 +1,33 @@ +/** + * "Am I the process entry point?" — the guard every executable here needs. + * + * `bin/distilly.mjs` and the two runnable scripts under `scripts/` must dispatch + * only when they were invoked directly, because tests import their internals. + * The obvious spelling is wrong in a way that fails *silently*: + * + * resolve(process.argv[1]) === fileURLToPath(import.meta.url) + * + * `import.meta.url` is always the realpath, while `argv[1]` is whatever the + * caller wrote. Under a symlink the two differ, the guard concludes "I was + * imported", and the program exits 0 having done nothing. That is not an edge + * case: `/tmp` is a symlink to `/private/tmp` on macOS, and — more importantly — + * an npm `bin` shim in `node_modules/.bin/` is a symlink, so a published + * `distilly` would have silently ignored every command. + * + * Resolving both sides is the fix. A path that cannot be resolved (an eval, a + * repl) is not an entry point, which is also the honest answer. + */ + +import { realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +/** True when `moduleUrl` names the file the process was started with. */ +export function isEntryPoint(moduleUrl) { + const invoked = process.argv[1]; + if (invoked === undefined || invoked === "") return false; + try { + return realpathSync(invoked) === realpathSync(fileURLToPath(moduleUrl)); + } catch { + return false; + } +} diff --git a/src/cli/paths.mjs b/src/cli/paths.mjs new file mode 100644 index 00000000..b51a7a0f --- /dev/null +++ b/src/cli/paths.mjs @@ -0,0 +1,41 @@ +/** + * Where a Skill family lives on disk. + * + * Two spellings of `--base-dir` grew apart: `harvest` / `parse-*` treat it as the + * directory that *contains* `skills/`, while `skill create` treats it as the + * storage root itself (`skills/colleague`). Both are reasonable, and neither is + * written down, so the same value silently means different places — a trap for + * users and for the coding agents that script this CLI. + * + * `resolveSkillsRoot` accepts both: the canonical `/skills/` wins + * whenever it exists (or `skills/` does), the storage-root reading is kept + * working for compatibility, and the caller gets a warning to surface when the + * legacy reading was used. + */ + +import { existsSync } from "node:fs"; +import { basename, join, resolve } from "node:path"; + +/** + * @param {{baseDir: string, family: string}} input + * @returns {{root: string, mode: "skills-root"|"storage-root", warning: string|null}} + */ +export function resolveSkillsRoot({ baseDir, family }) { + const base = resolve(baseDir); + const canonical = join(base, "skills", family); + if (existsSync(canonical)) return { root: canonical, mode: "skills-root", bare: false, warning: null }; + if (basename(base) === family) return { root: base, mode: "storage-root", bare: false, warning: null }; + if (existsSync(join(base, "skills"))) return { root: canonical, mode: "skills-root", bare: false, warning: null }; + // The bare spelling — pointing at the container of the family directories + // rather than at a directory that holds `skills/`. It is read as-is (nothing is + // invented underneath it) and flagged, because every caller that accepts this + // spelling silently inspects one level too high and finds nothing. + return { + root: base, + mode: "storage-root", + bare: true, + warning: + `--base-dir ${baseDir} contains no skills/ directory, so it was read as the storage root itself (${base}). ` + + `The canonical spelling is --base-dir , i.e. ${join(base, "..", "..")} for this layout.`, + }; +} diff --git a/src/cli/receipt.mjs b/src/cli/receipt.mjs new file mode 100644 index 00000000..aea2d7a6 --- /dev/null +++ b/src/cli/receipt.mjs @@ -0,0 +1,118 @@ +/** + * CLI receipts, output routing and file fingerprints. + * + * The receipt shape is frozen by `docs/v2/CONTRACT.md` §3: + * `{command, person, ok, inputs, outputs, anchors, warnings, unavailable}`. + * Every entry in `inputs`/`outputs` carries `{path, sha256, bytes}`. + * + * `--json` writes the receipt to stdout as the only stdout content (human text + * moves to stderr) so `JSON.parse(stdout)` always succeeds. + */ + +import { createHash } from "node:crypto"; +import { readFileSync, statSync } from "node:fs"; +import { relative, resolve, sep } from "node:path"; + +/** User-facing failure with an explicit remedy. */ +export class CliError extends Error { + constructor(message, { code = "error", remedy = "", exitCode = 1 } = {}) { + super(message); + this.name = "CliError"; + this.code = code; + this.remedy = remedy; + this.exitCode = exitCode; + } +} + +export function sha256Buffer(buffer) { + return createHash("sha256").update(buffer).digest("hex"); +} + +export function sha256Text(text) { + return sha256Buffer(Buffer.from(text, "utf8")); +} + +/** `{path, sha256, bytes}` for a file, or null when it is missing. */ +export function describeFile(filePath, { cwd = process.cwd() } = {}) { + let buffer; + try { + buffer = readFileSync(filePath); + } catch { + return null; + } + return { + path: displayPath(filePath, cwd), + sha256: sha256Buffer(buffer), + bytes: buffer.length, + }; +} + +/** `{path, bytes}` without hashing (used for directories and removals). */ +export function describePath(filePath, { cwd = process.cwd() } = {}) { + return { path: displayPath(filePath, cwd), bytes: directoryBytes(filePath) }; +} + +export function directoryBytes(dirPath) { + try { + return statSync(dirPath).size; + } catch { + return 0; + } +} + +/** Paths inside the working directory are reported relatively, like the old CLI. */ +export function displayPath(filePath, cwd = process.cwd()) { + const absolute = resolve(filePath); + const rel = relative(resolve(cwd), absolute); + if (rel === "") return "."; + if (!rel.startsWith("..") && !rel.startsWith(`${sep}..`)) return rel; + return absolute; +} + +/** Build the contract receipt object (field order is contractual). */ +export function createReceipt(command, options = {}) { + const receipt = { + command, + person: options.person ?? null, + ok: options.ok ?? true, + inputs: options.inputs ?? [], + outputs: options.outputs ?? [], + anchors: options.anchors ?? { total: 0, cited: 0 }, + warnings: options.warnings ?? [], + unavailable: options.unavailable ?? [], + }; + if (options.error) receipt.error = options.error; + return receipt; +} + +/** + * Route human text and machine receipts. + * In `--json` mode stdout carries exactly one JSON object; prose goes to stderr. + */ +export function createReporter(json, { stdout = process.stdout, stderr = process.stderr } = {}) { + const lines = []; + return { + json, + line(text) { + lines.push(text); + if (json) stderr.write(`${text}\n`); + else stdout.write(`${text}\n`); + }, + warn(text) { + stderr.write(`${text}\n`); + }, + /** + * A diagnostic that always goes to stderr, in both modes. Command output is + * `line()`; this is for "why the command failed", which must never pollute + * the machine channel — and must still be visible when `--json` is set. + */ + error(text) { + stderr.write(`${text}\n`); + }, + /** Write the receipt last so `JSON.parse(stdout)` sees a single object. */ + finish(receipt) { + if (!json) return; + stdout.write(`${JSON.stringify(receipt, null, 2)}\n`); + }, + }; +} diff --git a/src/collect/dingtalk.mjs b/src/collect/dingtalk.mjs new file mode 100644 index 00000000..34ecb24c --- /dev/null +++ b/src/collect/dingtalk.mjs @@ -0,0 +1,832 @@ +/** + * dingtalk.mjs — credentialed collection from DingTalk. + * + * Legacy ported: `tools/dingtalk_auto_collector.py` (790 lines). What that file + * actually proves about DingTalk's API surface matters here, so this module + * splits the channel the same way the legacy did: + * + * - **api mode** — read-only endpoints this repository has evidence for: the + * app credential exchange (`POST /v1.0/oauth2/accessToken`) and the contact + * directory card lookup (`POST /v1.0/contact/users/search`, + * `GET /v1.0/contact/users/{userId}`). Both are used by the legacy collector + * (`tools/dingtalk_auto_collector.py:106`, `:151`, `:202`). + * - **message history** — the legacy collector has **no API path** for it; it + * drives a browser instead (`tools/dingtalk_auto_collector.py:518`, + * "消息类（可选，仅用于发消息，历史消息需浏览器方案）"). Rather than invent an + * endpoint, api mode says so, loudly, and browser mode requires a consent + * token. The host performs the computer use and hands the bytes back through + * `--capture `; this module never drives a browser itself. + * + * Same discipline as the other collectors: credential from + * `~/.distilly/dingtalk_config.json` (or env), file name only in messages, raw + * bytes verbatim under `knowledge/raw/dingtalk/`, ledger upsert, loud failures + * with remediation, and a hard read-only guarantee. + */ + +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +import { consentTokenFingerprint, verify as verifyConsent } from "../consent.mjs"; + +export const CHANNEL = "dingtalk"; +export const CONFIG_FILE = "dingtalk_config.json"; +export const LEGACY_CONFIG_FILE = join(".colleague-skill", CONFIG_FILE); +export const DEFAULT_BASE_URL = "https://api.dingtalk.com"; +export const DEFAULT_LIMIT = 10; +export const DEFAULT_MAX_PAGES = 10; +export const DEFAULT_MAX_RETRIES = 4; +export const DEFAULT_MAX_BACKOFF_MS = 60_000; + +export const ENV_KEYS = { + appKey: ["DISTILLY_DINGTALK_APP_KEY", "DINGTALK_APP_KEY"], + appSecret: ["DISTILLY_DINGTALK_APP_SECRET", "DINGTALK_APP_SECRET"], +}; + +/** + * Both entries are query-shaped POSTs: neither creates, updates or deletes + * anything a user can see. Anything else non-GET is refused before it is sent. + */ +export const ALLOWED_MUTATIONS = [ + { method: "POST", path: "/v1.0/oauth2/accessToken", why: "app token exchange; no user-visible state" }, + { method: "POST", path: "/v1.0/contact/users/search", why: "read-only directory search expressed as POST" }, +]; + +const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); +const REMEDIATION_SETUP = [ + `create ~/.distilly/${CONFIG_FILE} (chmod 600) with {"app_key": "ding…", "app_secret": "…"}`, + " app: https://open-dev.dingtalk.com → 企业内部应用 → 权限 Contact.User.Read / qyapi_get_member_detail", + "or export DISTILLY_DINGTALK_APP_KEY and DISTILLY_DINGTALK_APP_SECRET for this shell only", +]; + +/** What is missing on the API side, quoted from the legacy collector's own note. */ +export const MESSAGE_API_GAP = + "DingTalk exposes no documented read API for message history; the legacy collector used a browser " + + "(tools/dingtalk_auto_collector.py:518 — “历史消息需浏览器方案”)"; + +export class CollectFailure extends Error { + constructor(reason, message, { remediation = [], exitCode = 1, kind = "failure" } = {}) { + super(message); + this.name = "CollectFailure"; + this.reason = reason; + this.remediation = remediation; + this.exitCode = exitCode; + this.kind = kind; + } +} + +export function redact(text, secrets = []) { + let output = String(text ?? ""); + for (const secret of secrets) { + if (typeof secret !== "string" || secret.length < 4) continue; + output = output.split(secret).join("[redacted]"); + } + return output; +} + +export function scrub(value, secrets = []) { + return JSON.parse( + JSON.stringify(value, (_key, item) => (typeof item === "string" ? redact(item, secrets) : item)), + ); +} + +export function distillyHome(env = process.env) { + const override = env?.DISTILLY_HOME; + return override ? resolve(String(override)) : join(homedir(), ".distilly"); +} + +export function credentialPaths(env = process.env) { + return { + primary: join(distillyHome(env), CONFIG_FILE), + legacy: join(homedir(), LEGACY_CONFIG_FILE), + }; +} + +export function loadCredential({ env = process.env, readFile = readFileSync } = {}) { + const pick = (names) => { + for (const name of names) { + const value = env?.[name]; + if (typeof value === "string" && value.trim() !== "") return value.trim(); + } + return null; + }; + const envKey = pick(ENV_KEYS.appKey); + const envSecret = pick(ENV_KEYS.appSecret); + if (envKey && envSecret) { + return { + ok: true, + source: "env", + configFile: CONFIG_FILE, + path: null, + values: { app_key: envKey, app_secret: envSecret }, + }; + } + + const { primary, legacy } = credentialPaths(env); + for (const [path, source] of [ + [primary, "config"], + [legacy, "legacy-config"], + ]) { + if (!existsSync(path)) continue; + let parsed; + try { + parsed = JSON.parse(readFile(path, "utf8")); + } catch (error) { + throw new CollectFailure( + "bad-credential-file", + `${CONFIG_FILE} is not valid JSON (${redact(error.message)}); rewrite it with {"app_key": "ding…", "app_secret": "…"}`, + { remediation: REMEDIATION_SETUP }, + ); + } + const appKey = parsed.app_key ?? parsed.appKey ?? null; + const appSecret = parsed.app_secret ?? parsed.appSecret ?? null; + if (!appKey || !appSecret) { + throw new CollectFailure("incomplete-credential", `${CONFIG_FILE} is missing app_key / app_secret`, { + remediation: REMEDIATION_SETUP, + }); + } + return { ok: true, source, configFile: CONFIG_FILE, path, values: { app_key: appKey, app_secret: appSecret } }; + } + + throw new CollectFailure("no-credential", `no credential at ~/.distilly/${CONFIG_FILE}`, { + remediation: REMEDIATION_SETUP, + }); +} + +export function assertReadOnly(url, method = "GET") { + const verb = String(method).toUpperCase(); + if (!MUTATING_METHODS.has(verb)) return true; + const path = (() => { + try { + return new URL(url).pathname; + } catch { + return String(url); + } + })(); + const allowed = ALLOWED_MUTATIONS.some((entry) => entry.method === verb && path.endsWith(entry.path)); + if (!allowed) { + throw new CollectFailure( + "write-operation-refused", + `refusing ${verb} ${path}: this collector never writes to ${CHANNEL}`, + { remediation: ["collectors are read-only; remove the mutating call instead of allowlisting it"] }, + ); + } + return true; +} + +export function parseRetryAfter(headerValue, nowMs = Date.now()) { + if (headerValue === null || headerValue === undefined || headerValue === "") return null; + const seconds = Number(headerValue); + if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000); + const date = Date.parse(String(headerValue)); + if (Number.isFinite(date)) return Math.max(0, date - nowMs); + return null; +} + +export function backoffDelay(attempt, retryAfterMs = null, options = {}) { + const { baseMs = 500, maxMs = DEFAULT_MAX_BACKOFF_MS } = options; + if (Number.isFinite(retryAfterMs) && retryAfterMs !== null) return Math.min(retryAfterMs, maxMs); + return Math.min(baseMs * 2 ** Math.max(0, attempt - 1), maxMs); +} + +export function defaultSleep(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +export async function requestJson(options) { + const { + fetchImpl, + url, + method = "GET", + headers = {}, + body, + maxRetries = DEFAULT_MAX_RETRIES, + sleep = defaultSleep, + secrets = [], + onRetry = () => {}, + authRemediation = REMEDIATION_SETUP, + } = options; + + assertReadOnly(url, method); + if (typeof fetchImpl !== "function") { + throw new CollectFailure("no-fetch", "no fetch implementation available", { + remediation: ["run on Node >= 20, or pass an injected fetch"], + }); + } + + let attempt = 0; + for (;;) { + attempt += 1; + let response; + try { + response = await fetchImpl(url, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); + } catch (error) { + const message = redact(error?.message ?? String(error), secrets); + if (attempt > maxRetries) { + throw new CollectFailure("network-error", `request failed: ${message}`, { + remediation: ["check the network/proxy and retry", ...authRemediation], + }); + } + await sleep(backoffDelay(attempt)); + onRetry({ attempt, status: null, delayMs: backoffDelay(attempt), reason: message }); + continue; + } + + const status = Number(response?.status ?? 0); + const retryAfterMs = parseRetryAfter(response?.headers?.get?.("retry-after") ?? null); + + if (status === 429 || status >= 500) { + if (attempt > maxRetries) { + throw new CollectFailure( + status === 429 ? "rate-limited" : "server-error", + status === 429 + ? `rate limited (HTTP 429) after ${maxRetries} retries` + : `server error (HTTP ${status}) after ${maxRetries} retries`, + { + remediation: [ + "retry later; already-fetched pages stay on disk and the cursor is checkpointed", + `lower --limit to stay under the ${CHANNEL} quota`, + ], + }, + ); + } + const delayMs = backoffDelay(attempt, retryAfterMs); + onRetry({ attempt, status, delayMs, reason: `HTTP ${status}` }); + await sleep(delayMs); + continue; + } + + const text = await response.text(); + if (status === 401 || status === 403) { + throw new CollectFailure("unauthorized", `HTTP ${status} from ${CHANNEL}; app credential rejected`, { + remediation: [ + `the credential in ~/.distilly/${CONFIG_FILE} was rejected — regenerate AppKey/AppSecret`, + "check the app scopes at https://open-dev.dingtalk.com", + ], + }); + } + if (status >= 400) { + throw new CollectFailure("http-error", `HTTP ${status}: ${redact(text.slice(0, 200), secrets)}`, { + remediation: ["check the request parameters", ...authRemediation], + }); + } + + let json = null; + try { + json = JSON.parse(text); + } catch { + throw new CollectFailure("invalid-json", `${CHANNEL} returned a non-JSON body`, { + remediation: ["retry later; if it persists the endpoint may have changed"], + }); + } + return { status, text, json, attempts: attempt }; + } +} + +export function sha256Hex(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function slug(text, fallback = "target") { + const slugged = String(text ?? "") + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^[-._]+|[-._]+$/g, "") + .slice(0, 64); + return slugged || fallback; +} + +export function knowledgeRoot({ root = process.cwd(), person, family = "colleague" } = {}) { + return person ? join(resolve(root), "skills", slug(family), slug(person), "knowledge") : join(resolve(root), "knowledge"); +} + +export function writeRaw(knowledgeDir, name, bytes) { + const dir = join(knowledgeDir, "raw", CHANNEL); + mkdirSync(dir, { recursive: true }); + const path = join(dir, `${slug(name, "page")}.json`); + const staging = `${path}.${process.pid}.tmp`; + const buffer = Buffer.from(bytes); + try { + writeFileSync(staging, buffer); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { + path, + relativePath: `raw/${CHANNEL}/${slug(name, "page")}.json`, + bytes: buffer.length, + sha256: sha256Hex(buffer), + }; +} + +export function appendLedger(knowledgeDir, entries) { + if (entries.length === 0) return { path: join(knowledgeDir, "index.json"), added: 0, total: 0, existed: false }; + const path = join(knowledgeDir, "index.json"); + let existing = []; + if (existsSync(path)) { + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + existing = Array.isArray(parsed) ? parsed : []; + } catch (error) { + throw new CollectFailure("bad-ledger", `knowledge/index.json is not valid JSON: ${redact(error.message)}`, { + remediation: ["repair or remove knowledge/index.json, then rerun the collect"], + }); + } + } + const byId = new Map(existing.filter((e) => e && typeof e === "object").map((e) => [e.id, e])); + let added = 0; + for (const entry of entries) { + if (!byId.has(entry.id)) added += 1; + byId.set(entry.id, entry); + } + const merged = [...byId.values()].sort((a, b) => String(a.id).localeCompare(String(b.id))); + const staging = `${path}.${process.pid}.tmp`; + try { + writeFileSync(staging, `${JSON.stringify(merged, null, 2)}\n`); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { path, added, total: merged.length, existed: existing.length > 0 }; +} + +/** Issues an app access token. Returns `{token, expiresIn}` — never logged. */ +export async function fetchAppToken({ fetchImpl, env = process.env, credential, baseUrl, maxRetries, sleep, secrets }) { + const response = await requestJson({ + fetchImpl, + url: `${baseUrl}/v1.0/oauth2/accessToken`, + method: "POST", + headers: { "content-type": "application/json" }, + body: { appKey: credential.values.app_key, appSecret: credential.values.app_secret }, + maxRetries, + sleep, + secrets, + }); + const token = response.json?.accessToken; + if (!token) { + throw new CollectFailure("auth-failed", "DingTalk rejected the app credential (no accessToken in the response)", { + remediation: REMEDIATION_SETUP, + }); + } + return { token, expiresIn: Number(response.json?.expireIn ?? 7200) }; +} + +/** + * api mode: one directory-card lookup for `name`, plus the profile detail of the + * single match. No pagination is claimed — the legacy collector only ever called + * `/v1.0/contact/users/search` with `offset: 0` and never looped, so the + * continuation parameter is left untested and documented as a known gap. + */ +export async function collectDirectory(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + name, + limit = DEFAULT_LIMIT, + maxRetries = DEFAULT_MAX_RETRIES, + sleep = defaultSleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_DINGTALK_BASE_URL || DEFAULT_BASE_URL, + onProgress = () => {}, + } = options; + + const knowledgeDir = knowledgeRoot({ root, person, family }); + const outputs = []; + const warnings = []; + const ledgerEntries = []; + let secrets = []; + let credential = null; + let requests = 0; + let users = 0; + + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + resource: "directory-card", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + }; + const fail = (failure) => ({ + ok: false, + exitCode: failure.exitCode ?? 1, + receipt: scrub( + { + ...base, + ok: false, + person: person ?? null, + name: name ?? null, + requests, + errors: [redact(failure.message, secrets)], + unavailable: [ + { + channel: CHANNEL, + reason: redact(`${failure.reason}: ${failure.message}`, secrets), + remediation: failure.remediation ?? [], + }, + ], + }, + secrets, + ), + }); + + try { + if (!name) { + throw new CollectFailure("missing-target", "collect dingtalk needs --name ", { + remediation: [ + "api mode reads the contact directory card; message history has no API — see --mode browser", + MESSAGE_API_GAP, + ], + }); + } + credential = loadCredential({ env }); + secrets = [credential.values.app_secret]; + base.credential_source = credential.source; + + const size = Math.min(Math.max(1, Number(limit) || DEFAULT_LIMIT), 50); + const { token } = await fetchAppToken({ fetchImpl, env, credential, baseUrl, maxRetries, sleep, secrets }); + secrets = [...secrets, token]; + requests += 1; + + const search = await requestJson({ + fetchImpl, + url: `${baseUrl}/v1.0/contact/users/search`, + method: "POST", + headers: { "content-type": "application/json", "x-acs-dingtalk-access-token": token }, + body: { searchText: name, offset: 0, size }, + maxRetries, + sleep, + secrets, + }); + requests += 1; + + const list = Array.isArray(search.json?.list) ? search.json.list : []; + users = list.length; + const stored = writeRaw(knowledgeDir, `${name}-search`, search.text); + outputs.push({ path: stored.path, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }); + ledgerEntries.push({ + id: `${CHANNEL}:${slug(name)}:search`, + kind: "raw", + origin: stored.relativePath, + source: CHANNEL, + fetched_at: now, + bytes: stored.bytes, + sha256: stored.sha256, + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + method: "api-app-token", + items: list.length, + warnings: [], + }); + if (list.length >= size) { + warnings.push( + `search returned a full page (${size}); the continuation parameter is not verified against a live tenant (known gap)`, + ); + } + + if (list.length === 1) { + const userId = list[0]?.userId ?? list[0]?.unionId; + if (userId) { + const detail = await requestJson({ + fetchImpl, + url: `${baseUrl}/v1.0/contact/users/${encodeURIComponent(userId)}`, + headers: { "x-acs-dingtalk-access-token": token }, + maxRetries, + sleep, + secrets, + }); + requests += 1; + const detailStored = writeRaw(knowledgeDir, `${name}-profile`, detail.text); + outputs.push({ path: detailStored.path, sha256: detailStored.sha256, bytes: detailStored.bytes, kind: "raw" }); + ledgerEntries.push({ + id: `${CHANNEL}:${slug(name)}:profile`, + kind: "raw", + origin: detailStored.relativePath, + source: CHANNEL, + fetched_at: now, + bytes: detailStored.bytes, + sha256: detailStored.sha256, + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + method: "api-app-token", + items: 1, + warnings: [], + }); + } + } + + const ledger = appendLedger(knowledgeDir, ledgerEntries); + onProgress(`directory: ${list.length} match(es)`); + warnings.push(MESSAGE_API_GAP); + warnings.push( + "message history: run `distilly collect dingtalk --mode browser --consent ` and let the host capture it", + ); + + return { + ok: true, + exitCode: 0, + receipt: scrub( + { + ...base, + ok: true, + person: person ?? null, + name, + requests, + items: users, + outputs, + ledger: { path: ledger.path, added: ledger.added, total: ledger.total }, + unavailable: [], + }, + secrets, + ), + }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + const result = fail(error); + if (ledgerEntries.length > 0) { + try { + const ledger = appendLedger(knowledgeDir, ledgerEntries); + result.receipt.ledger = { path: ledger.path, added: ledger.added, total: ledger.total }; + result.receipt.partial = true; + } catch { + result.receipt.warnings.push("could not register the partial pages in knowledge/index.json"); + } + } + return result; + } +} + +/** + * browser mode: the consent gate plus capture registration. This module never + * launches a browser — the host does the computer use and writes the captured + * bytes with `--capture `; we verify the grant, store the bytes verbatim + * and register them in the ledger with their provenance. + */ +export function registerCapture(options = {}) { + const { + env = process.env, + root = process.cwd(), + person, + family = "colleague", + scope = `collect:${CHANNEL}:browser`, + consentToken, + capturePath, + label, + producer = "host:computer-use", + now = new Date().toISOString(), + readFile = readFileSync, + } = options; + + const knowledgeDir = knowledgeRoot({ root, person, family }); + const base = { + command: "collect", + channel: CHANNEL, + mode: "browser", + resource: "messages", + ok: false, + inputs: [], + outputs: [], + warnings: [], + unavailable: [], + credential_file: null, + }; + + const verification = verifyConsent(consentToken, { env, scope }); + if (!verification.ok) { + return { + ok: false, + exitCode: 2, + receipt: { + ...base, + ok: false, + status: "waiting-for-user-consent", + person: person ?? null, + errors: [`waiting for user consent (${verification.reason})`], + unavailable: [ + { + channel: CHANNEL, + reason: `waiting for user consent: ${verification.reason}`, + scope, + remediation: verification.remediation, + }, + ], + }, + }; + } + + if (!capturePath) { + return { + ok: true, + exitCode: 0, + receipt: { + ...base, + ok: true, + status: "awaiting-host-capture", + person: person ?? null, + consent: { + scope: verification.record.scope, + granted_at: verification.record.granted_at, + expires_at: verification.record.expires_at, + token_sha256_12: consentTokenFingerprint(consentToken), + }, + host_steps: [ + "Host (computer use): open the DingTalk client, open the target conversation, load the requested range.", + "Host: write the captured messages to a file, verbatim, and rerun with --capture .", + "This tool does not click, scroll, send or react — consent only authorises the host's capture.", + MESSAGE_API_GAP, + ], + unavailable: [], + }, + }; + } + + let bytes; + try { + bytes = readFile(capturePath); + } catch (error) { + return { + ok: false, + exitCode: 1, + receipt: { + ...base, + ok: false, + person: person ?? null, + errors: [`cannot read --capture ${capturePath}: ${redact(error.message)}`], + unavailable: [ + { + channel: CHANNEL, + reason: `capture-unreadable: ${redact(error.message)}`, + remediation: ["point --capture at a readable file produced by the host"], + }, + ], + }, + }; + } + + const stored = writeRaw(knowledgeDir, label ?? `browser-${new Date(now).toISOString().slice(0, 10)}`, bytes); + const entry = { + id: `${CHANNEL}:${slug(label ?? "browser")}:capture`, + kind: "raw", + origin: stored.relativePath, + source: CHANNEL, + fetched_at: now, + bytes: stored.bytes, + sha256: stored.sha256, + credentialed: false, + method: "browser-host", + provenance: { method: "browser-host", producer, confidence: "host-reported" }, + consent: { + scope: verification.record.scope, + granted_at: verification.record.granted_at, + expires_at: verification.record.expires_at, + token_sha256_12: consentTokenFingerprint(consentToken), + }, + warnings: [], + }; + const ledger = appendLedger(knowledgeDir, [entry]); + + return { + ok: true, + exitCode: 0, + receipt: scrub( + { + ...base, + ok: true, + status: "captured", + person: person ?? null, + outputs: [{ path: stored.path, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }], + ledger: { path: ledger.path, added: ledger.added, total: ledger.total }, + provenance: entry.provenance, + unavailable: [], + }, + [], + ), + }; +} + +export async function collect(options = {}) { + return options.mode === "browser" ? registerCapture(options) : collectDirectory(options); +} + +export const HELP = `distilly collect dingtalk — 钉钉采集 / DingTalk collection + +用法 (zh): + distilly collect dingtalk --name <姓名> [--person ] [--limit 10] [--json] + 只读的通讯录名片采集（企业应用凭据）。 + distilly collect dingtalk --mode browser --consent [--capture ] [--json] + 消息历史没有公开读接口（见 tools/dingtalk_auto_collector.py:518）；浏览器采集由宿主完成， + 本工具只做同意门 + 原样落盘 + 账本登记。无 token / 已过期 → exit 2（等待用户同意）。 + +凭据：~/.distilly/${CONFIG_FILE}（app_key / app_secret，0600）或 + DISTILLY_DINGTALK_APP_KEY / DISTILLY_DINGTALK_APP_SECRET。错误信息只出现配置文件名。 +只读：仅允许 POST /v1.0/oauth2/accessToken 与 POST /v1.0/contact/users/search（都是查询语义）， + 其余非 GET 请求在 assertReadOnly() 里被拒绝。 + +--- +## English + distilly collect dingtalk --name [--json] + read-only directory card via the app credential. + distilly collect dingtalk --mode browser --consent [--capture ] [--json] + message history has no public read API; the host captures it, this tool only gates + consent, stores the bytes verbatim and registers them in the ledger. No/expired token → exit 2. +`; + +export function parseFlags(argv) { + const flags = { _: [] }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (!arg.startsWith("--")) { + flags._.push(arg); + continue; + } + const name = arg.slice(2); + if (name === "json" || name === "help") { + flags[name] = true; + continue; + } + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + flags[name] = value; + index += 1; + } + return flags; +} + +export async function runCollectCli(argv, io = {}) { + const out = io.stdout ?? ((line) => process.stdout.write(`${line}\n`)); + const err = io.stderr ?? ((line) => process.stderr.write(`${line}\n`)); + + let flags; + try { + flags = parseFlags(argv); + } catch (error) { + err(`Error: ${error.message}`); + return 1; + } + if (flags.help) { + out(HELP); + return 0; + } + + const result = await collect({ + fetch: io.fetch ?? globalThis.fetch, + env: io.env ?? process.env, + root: flags.root ?? process.cwd(), + person: flags.person, + family: flags.family, + mode: flags.mode ?? "api", + name: flags.name, + limit: flags.limit ? Number(flags.limit) : undefined, + maxRetries: flags["max-retries"] ? Number(flags["max-retries"]) : undefined, + consentToken: flags.consent, + capturePath: flags.capture, + label: flags.label, + sleep: io.sleep, + now: io.now, + }); + + if (flags.json) out(JSON.stringify(result.receipt, null, 2)); + if (result.ok) { + out(`ok (${result.receipt.status ?? "collected"}) → ${result.receipt.outputs.length} raw file(s)`); + for (const warning of result.receipt.warnings ?? []) err(`warning: ${warning}`); + for (const step of result.receipt.host_steps ?? []) err(`host: ${step}`); + } else { + err(`Error: ${result.receipt.errors?.[0] ?? "collect failed"}`); + for (const entry of result.receipt.unavailable ?? []) { + err(`unavailable: ${entry.channel} — ${entry.reason}`); + for (const step of entry.remediation ?? []) err(` fix: ${step}`); + } + for (const warning of result.receipt.warnings ?? []) err(`warning: ${warning}`); + } + return result.exitCode; +} diff --git a/src/collect/discord.mjs b/src/collect/discord.mjs new file mode 100644 index 00000000..66aa80b0 --- /dev/null +++ b/src/collect/discord.mjs @@ -0,0 +1,308 @@ +/** + * discord.mjs — the Discord channel, read through the REST API with a bot token. + * + * Legacies: `tools/discord_collector.py` fetched messages for a channel and wrote + * them to a text file. This keeps the read path and changes the shape of the + * output: raw pages verbatim in `knowledge/raw/discord/`, the same pages turned + * into anchored text by the existing Discord export parser (`parse/chat.mjs`), and + * one ledger entry per page — so a collected channel is immediately derivable. + * + * Read-only by construction: the allowlist is GET-only, and a Discord bot token + * cannot delete or post through any code path here. + */ + +import { join, resolve } from "node:path"; + +import { KnowledgeStore } from "../knowledge/store.mjs"; +import { loadLedger, recordDocument, saveLedger } from "../knowledge/ledger.mjs"; +import { SourceFile } from "../parse/common.mjs"; +import { parseChat } from "../parse/chat.mjs"; +import { + CollectFailure, + DEFAULT_MAX_PAGES, + DEFAULT_MAX_RETRIES, + assertReadOnly, + clearCheckpoint, + loadCredential, + readCheckpoint, + redact, + requestJson, + scrub, + writeCheckpoint, + writeRaw, + knowledgeRoot, +} from "./kit.mjs"; + +export const CHANNEL = "discord"; +export const CONFIG_FILE = "discord_config.json"; +export const ENV_KEYS = ["DISTILLY_DISCORD_BOT_TOKEN", "DISCORD_BOT_TOKEN"]; +export const DEFAULT_BASE_URL = "https://discord.com/api/v10"; +export const DEFAULT_PAGE_SIZE = 100; + +/** + * Every call this module may make. Discord's read API is GET-only, so the list is + * GET-only too — a mutating verb is refused before a socket is opened. + */ +export const ALLOWED_CALLS = [ + { method: "GET", path: "/api/v10/users/@me" }, + { method: "GET", path: "/api/v10/guilds/" }, + { method: "GET", path: "/api/v10/channels/" }, +]; + +const REMEDIATION = [ + "create a bot at https://discord.com/developers/applications, give it the “Read Message History” permission", + `write ~/.distilly/${CONFIG_FILE} as {"bot_token": "…"}`, +]; + +/** Discord's rate limits answer 429 with a JSON `retry_after` in **seconds**. */ +export const parseDiscordRetryAfter = (json) => { + const value = json?.retry_after; + return typeof value === "number" ? value * 1000 : null; +}; + +/** + * @param {{fetch?: Function, env?: object, root?: string, person: string, family?: string, + * channelId: string, limit?: number, maxPages?: number, maxRetries?: number, + * since?: string, resume?: boolean, sleep?: Function, now?: string, baseUrl?: string}} options + */ +export async function collect(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + channelId, + limit = DEFAULT_PAGE_SIZE, + maxPages = DEFAULT_MAX_PAGES, + maxRetries = DEFAULT_MAX_RETRIES, + since, + resume = true, + sleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_DISCORD_BASE_URL || DEFAULT_BASE_URL, + } = options; + + const outputs = []; + const warnings = []; + const retries = []; + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + retries, + }; + let secrets = []; + const fail = (failure) => ({ + ok: false, + exitCode: failure.exitCode ?? 1, + receipt: scrub( + { + ...base, + ok: false, + person: person ?? null, + channel_id: channelId ?? null, + errors: [redact(failure.message, secrets)], + unavailable: [{ channel: CHANNEL, reason: redact(`${failure.reason}: ${failure.message}`, secrets), remediation: failure.remediation ?? [] }], + }, + secrets, + ), + }); + + try { + if (!person) throw new CollectFailure("missing-person", "collect discord needs --person ", { remediation: ["pass --person"] }); + if (!channelId) { + throw new CollectFailure("missing-target", "collect discord needs --channel-id ", { + remediation: ["enable Developer Mode in Discord and copy the channel id"], + }); + } + + const credential = loadCredential({ env, configFile: CONFIG_FILE, envKeys: ENV_KEYS, fields: ["bot_token"] }); + secrets = [credential.values.bot_token]; + base.credential_source = credential.source; + + const knowledgeDir = knowledgeRoot({ root, person, family }); + const store = new KnowledgeStore(join(resolve(root), "skills", family, person)); + const ledger = loadLedger(store); + const checkpoint = resume ? readCheckpoint({ env, root, channel: CHANNEL, target: channelId }) : null; + let cursor = since ?? checkpoint?.cursor ?? null; + if (checkpoint?.cursor && since === undefined) warnings.push(`resuming from checkpoint (${checkpoint.pages ?? 0} page(s) done)`); + + const pageSize = Math.min(Math.max(1, Number(limit) || DEFAULT_PAGE_SIZE), 100); + const headers = { authorization: `Bot ${credential.values.bot_token}`, accept: "application/json" }; + let pages = 0; + let items = 0; + let requests = 0; + const textEntries = []; + let anchors = 0; + + for (;;) { + if (pages >= maxPages) { + warnings.push(`stopped after --max-pages ${maxPages}; rerun to continue from the cursor`); + break; + } + const params = new URLSearchParams({ limit: String(pageSize) }); + if (cursor) params.set("before", cursor); + const url = `${baseUrl}/channels/${encodeURIComponent(channelId)}/messages?${params.toString()}`; + const response = await requestJson({ + fetchImpl, + url, + method: "GET", + headers, + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + onRetry: (info) => { + retries.push(info); + warnings.push(`retry ${info.attempt} after ${info.reason} (waited ${info.delayMs}ms)`); + }, + }); + requests += 1; + const page = Array.isArray(response.json) ? response.json : []; + pages += 1; + items += page.length; + + const name = `${channelId}-p${String(pages).padStart(3, "0")}`; + const stored = writeRaw(knowledgeDir, CHANNEL, name, response.text); + outputs.push({ path: stored.relativePath, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }); + + // The Discord export parser already knows this message shape, so the same + // page that was stored raw is what the derivation reads as anchored text. + try { + const file = new SourceFile({ path: `${name}.json`, name: `${name}.json`, raw: new Uint8Array(Buffer.from(response.text, "utf8")) }); + const document = parseChat(file, { + source: CHANNEL, + method: "api-bot-token", + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + format: "discord-messages", + }); + for (const warning of document.warnings ?? []) warnings.push(`${name}: ${warning.message ?? warning}`); + const recorded = recordDocument(store, ledger, { ...document, fetched_at: now }, { fetched_at: now }); + if (recorded.written.text) { + outputs.push({ + path: recorded.written.text.relativePath, + sha256: recorded.written.text.sha256, + bytes: recorded.written.text.bytes, + kind: "text", + }); + textEntries.push(recorded.entry?.id ?? null); + anchors += recorded.entry?.anchor_count ?? 0; + } + } catch (error) { + warnings.push(`${name}: ${redact(error.message, secrets)}; the raw page is stored, the text is not`); + } + + if (page.length === 0) break; + cursor = page[page.length - 1].id ?? null; + if (!cursor) break; + if (page.length < pageSize) break; + if (resume) writeCheckpoint({ env, root, channel: CHANNEL, target: channelId }, { channel: CHANNEL, target: channelId, cursor, pages, items, updated_at: now }); + } + + saveLedger(store, ledger); + if (resume) clearCheckpoint({ env, root, channel: CHANNEL, target: channelId }); + + const receipt = { + ...base, + ok: true, + person, + channel_id: channelId, + pages, + items, + requests, + cursor, + text_entries: textEntries.filter(Boolean), + anchors: { total: anchors, cited: 0 }, + ledger: { path: store.ledgerPath, total: ledger.length }, + unavailable: [], + }; + return { ok: true, exitCode: 0, receipt: scrub(receipt, secrets) }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + return fail(error); + } +} + +/* ------------------------------------------------------------------ CLI */ + +export function parseDiscordArgs(argv) { + const options = { person: null, baseDir: process.cwd(), channelId: null, limit: DEFAULT_PAGE_SIZE, maxPages: DEFAULT_MAX_PAGES, json: false, since: null }; + const takesValue = { "--person": "person", "--base-dir": "baseDir", "--channel-id": "channelId", "--limit": "limit", "--max-pages": "maxPages", "--since": "since" }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--json") options.json = true; + else if (takesValue[arg]) { + const value = argv[index + 1]; + if (value === undefined) return { error: `${arg} requires a value` }; + options[takesValue[arg]] = /limit|maxPages/.test(takesValue[arg]) ? Number(value) : value; + index += 1; + } else if (arg.startsWith("--")) return { error: `unknown option: ${arg}` }; + else if (!options.channelId) options.channelId = arg; + else return { error: `unexpected argument: ${arg}` }; + } + if (!options.person) return { error: "--person is required" }; + if (!options.channelId) return { error: "collect discord needs --channel-id " }; + return { options }; +} + +export async function runCollectCli(argv, io = {}) { + const out = typeof io.stdout === "function" ? io.stdout : (line) => process.stdout.write(`${line}\n`); + const err = typeof io.stderr === "function" ? io.stderr : (line) => process.stderr.write(`${line}\n`); + const parsed = parseDiscordArgs(argv); + if (parsed.error) { + return { + ok: false, + exitCode: 2, + receipt: { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + person: null, + warnings: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + unavailable: [{ channel: CHANNEL, reason: parsed.error, remediation: ["distilly collect discord --help"] }], + error: { code: "collect/usage", message: parsed.error, remedy: "distilly collect discord --help" }, + }, + }; + } + const { options } = parsed; + const result = await collect({ + root: options.baseDir, + person: options.person, + channelId: options.channelId, + limit: options.limit, + maxPages: options.maxPages, + since: options.since, + ...(io.fetch ? { fetch: io.fetch } : {}), + }); + if (!options.json) { + if (result.ok) { + out(`collect discord: ${result.receipt.items} message(s) in ${result.receipt.pages} page(s), ${result.receipt.anchors.total} anchor(s)`); + for (const output of result.receipt.outputs) out(` ${output.kind}: ${output.path}`); + } + for (const warning of result.receipt.warnings ?? []) err(` warning: ${warning}`); + for (const failure of result.receipt.unavailable ?? []) { + err(` unavailable: ${failure.reason}`); + for (const step of failure.remediation ?? []) err(` ${step}`); + } + } + return result; +} + +export { REMEDIATION as DISCORD_REMEDIATION }; diff --git a/src/collect/feishu-browser.mjs b/src/collect/feishu-browser.mjs new file mode 100644 index 00000000..2807dfdd --- /dev/null +++ b/src/collect/feishu-browser.mjs @@ -0,0 +1,316 @@ +/** + * feishu-browser.mjs — Feishu pages a **host** captured with computer use. + * + * `tools/feishu_browser.py` drove Playwright against the user's Chrome profile. + * v2 deliberately does not do that: driving a browser and injecting input is + * computer use, which belongs to the host under an explicit consent token (see + * `src/collect/x.mjs` and `docs/v2/CONTRACT.md` §4). So the port keeps what the + * pipeline actually owns — + * + * 1. the consent gate (`collect:feishu:browser`, exit 2 while waiting), + * 2. the plan that tells the host what to capture and how to hand it over, + * 3. the verbatim sink for the capture, and + * 4. the normalisation that turns it into `knowledge/text/*.md` with anchors, + * + * — and leaves the browsing itself to the host. Nothing here imports playwright, + * and no code path can open a page or send an event. + */ + +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +import { KnowledgeStore } from "../knowledge/store.mjs"; +import { loadLedger, recordDocument, saveLedger } from "../knowledge/ledger.mjs"; +import { SourceFile, buildDocument, recordsFromCharSpans, stripHtml } from "../parse/common.mjs"; +import { consentTokenFingerprint, verify as verifyConsent } from "../consent.mjs"; +import { redact } from "./feishu.mjs"; + +export const CHANNEL = "feishu"; +export const MODE = "browser"; +export const BROWSER_SCOPE = "collect:feishu:browser"; +/** What the host must be able to do; printed in the plan, asserted by the tests. */ +export const PRODUCER = "host:computer-use"; + +/** The URL→page-type table the Python tool used, plus the chat case. */ +export function detectPageType(url) { + const text = String(url ?? ""); + if (text === "") return null; + if (text.includes("/wiki/")) return "wiki"; + if (text.includes("/docx/") || text.includes("/docs/")) return "doc"; + if (text.includes("/sheets/") || text.includes("/spreadsheets/")) return "sheet"; + if (text.includes("/base/")) return "base"; + if (text.includes("/messages/") || text.includes("/chat/")) return "chat"; + return null; +} + +/** What the host has to do, spelled out so a coding agent can follow it. */ +export function browserPlan({ target = null, url = null, pageType = null, scope = BROWSER_SCOPE, now = new Date().toISOString() } = {}) { + return { + scope, + target, + url, + page_type: pageType, + prepared_at: now, + steps: [ + "Host (computer use): open the Feishu page in the browser the user is signed in to — this module never opens one.", + pageType === "chat" + ? "Host: scroll the chat back far enough to cover the requested window, then copy the visible transcript." + : "Host: open the document and copy its text (a sheet: copy the used range as CSV/TSV).", + "Host: write the copied text (or the saved HTML) to a file.", + `Host: register the capture with: distilly collect feishu --mode browser --consent --capture --url `, + "This module then stores the capture verbatim, normalises it into knowledge/text with anchors, and writes the ledger entry.", + ], + forbidden: [ + "this module never drives a browser, never injects into one and never sends input events", + "the capture is host-reported: its provenance says so rather than claiming a verified session", + ], + }; +} + +const looksLikeHtml = (text) => /<\/?(?:html|body|div|p|span|table|td)\b/i.test(text); + +/** + * Store and record one host capture. + * + * @param {{env?: object, root?: string, person: string, family?: string, consentToken?: string, + * capturePath?: string, url?: string, pageType?: string, label?: string, producer?: string, + * now?: string, readFile?: Function, scope?: string}} input + */ +export function collectBrowser(input = {}) { + const { + env = process.env, + root = process.cwd(), + person, + family = "colleague", + scope = BROWSER_SCOPE, + consentToken, + capturePath, + url = null, + label, + producer = PRODUCER, + now = new Date().toISOString(), + readFile = readFileSync, + } = input; + + const pageType = input.pageType ?? detectPageType(url) ?? (url ? "page" : null); + const base = { + command: "collect", + channel: CHANNEL, + mode: MODE, + ok: false, + person: person ?? null, + target: url, + page_type: pageType, + credential_file: null, + inputs: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + warnings: [], + unavailable: [], + }; + + const verification = verifyConsent(consentToken, { env, scope }); + if (!verification.ok) { + return { + ok: false, + exitCode: 2, + receipt: { + ...base, + status: "waiting-for-user-consent", + consent_scope: scope, + errors: [`waiting for user consent (${verification.reason})`], + unavailable: [ + { + channel: CHANNEL, + reason: `waiting for user consent: ${verification.reason}`, + scope, + remediation: verification.remediation, + }, + ], + }, + }; + } + + const consent = { + scope: verification.record.scope, + granted_at: verification.record.granted_at, + expires_at: verification.record.expires_at, + token_sha256_12: consentTokenFingerprint(consentToken), + }; + const provenance = { method: "browser-host", producer, confidence: "host-reported" }; + + if (!capturePath) { + return { + ok: true, + exitCode: 0, + receipt: { ...base, ok: true, status: "awaiting-host-capture", consent, provenance, plan: browserPlan({ target: url, url, pageType, scope, now }) }, + }; + } + + let bytes; + try { + bytes = readFile(capturePath); + } catch (error) { + return { + ok: false, + exitCode: 1, + receipt: { + ...base, + errors: [`cannot read --capture ${capturePath}: ${redact(error.message)}`], + consent, + unavailable: [ + { channel: CHANNEL, reason: `capture-unreadable: ${redact(error.message)}`, remediation: ["point --capture at a file produced by the host"] }, + ], + }, + }; + } + + const raw = Buffer.isBuffer(bytes) ? bytes : Buffer.from(String(bytes), "utf8"); + const decoded = raw.toString("utf8"); + const html = looksLikeHtml(decoded); + const text = html ? stripHtml(decoded).text : decoded; + const name = label ?? `browser-${pageType ?? "page"}-${String(now).slice(0, 10)}`; + // The capture is stored **verbatim**: the raw payload is what the host handed + // over, not the text we derived from it. Building the `SourceFile` from the + // stripped text (as this did) meant the HTML was gone after a run — the one + // thing the raw vault exists to prevent — and the anchor offsets pointed into a + // file that was never on disk in that form. + const file = new SourceFile({ + path: `${name}${html ? ".html" : ".txt"}`, + name: `${name}${html ? ".html" : ".txt"}`, + raw: new Uint8Array(raw), + }); + + const spans = []; + const pattern = /[^\n]+/g; + let match; + while ((match = pattern.exec(text)) !== null) { + // Paragraph text only: `assembleContent` locates each one inside the raw + // payload, so an anchor points at the bytes the host captured. A paragraph the + // raw HTML does not contain verbatim keeps a `null` range rather than a guess. + spans.push({ text: match[0], kind: "paragraph" }); + } + const document = buildDocument({ + parser: "feishu-browser", + format: pageType, + kind: pageType === "chat" ? "message" : "doc", + method: "browser-host", + credentialed: false, + source: CHANNEL, + origin: url ?? `${name}.txt`, + files: [file], + records: spans, + meta: { page_type: pageType, url, producer }, + }); + + const store = new KnowledgeStore(join(resolve(root), "skills", family, person)); + const ledger = loadLedger(store); + const recorded = recordDocument(store, ledger, { ...document, fetched_at: now }, { fetched_at: now }); + // `buildEntry` keeps the pipeline's own fields; the capture's provenance is + // added here so a reader can see this text came from a host, not from an API. + const entry = ledger.find((candidate) => candidate.id === recorded.entry.id); + if (entry) { + entry.url = url; + entry.urls = url ? [url] : []; + entry.provenance = provenance; + entry.consent = consent; + entry.host_capture = { file: capturePath, bytes: raw.length, sha256: recorded.written.files[0]?.sha256 ?? null, html: looksLikeHtml(decoded) }; + } + saveLedger(store, ledger); + + const rawFile = recorded.written.files[0]; + const textFile = recorded.written.text; + return { + ok: true, + exitCode: 0, + receipt: { + ...base, + ok: true, + status: "captured", + consent, + provenance, + page_type: pageType, + outputs: [ + ...(rawFile ? [{ path: rawFile.relativePath, sha256: rawFile.sha256, bytes: rawFile.bytes, kind: "raw" }] : []), + ...(textFile ? [{ path: textFile.relativePath, sha256: textFile.sha256, bytes: textFile.bytes, kind: "text" }] : []), + ], + anchors: { total: recorded.entry?.anchor_count ?? 0, cited: 0 }, + entry: recorded.entry?.id ?? null, + warnings: [...(document.warnings ?? []).map((warning) => warning.message ?? warning)], + unavailable: [], + }, + }; +} + +/* ------------------------------------------------------------------ CLI */ + +export function parseBrowserArgs(argv) { + const options = { person: null, baseDir: process.cwd(), consent: null, capture: null, url: null, label: null, scope: BROWSER_SCOPE }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--json") continue; + else if (arg === "--mode") { + const value = argv[index + 1]; + if (value !== "browser") return { error: `feishu-browser handles --mode browser only (got ${value ?? "nothing"})` }; + index += 1; + } else if (["--person", "--base-dir", "--consent", "--capture", "--url", "--label"].includes(arg)) { + const value = argv[index + 1]; + if (value === undefined) return { error: `${arg} requires a value` }; + options[{ "--person": "person", "--base-dir": "baseDir", "--consent": "consent", "--capture": "capture", "--url": "url", "--label": "label" }[arg]] = value; + index += 1; + } else if (arg.startsWith("--")) return { error: `unknown option: ${arg}` }; + else if (!options.url) options.url = arg; + else return { error: `unexpected argument: ${arg}` }; + } + if (!options.person) return { error: "--person is required" }; + if (!options.url) return { error: "collect feishu --mode browser needs --url " }; + return { options }; +} + +export function runCollectCli(argv, io = {}) { + const out = typeof io.stdout === "function" ? io.stdout : (line) => process.stdout.write(`${line}\n`); + const err = typeof io.stderr === "function" ? io.stderr : (line) => process.stderr.write(`${line}\n`); + const parsed = parseBrowserArgs(argv); + if (parsed.error) { + return { + ok: false, + exitCode: 2, + receipt: { + command: "collect", + channel: CHANNEL, + mode: MODE, + ok: false, + warnings: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + errors: [parsed.error], + unavailable: [{ channel: CHANNEL, reason: parsed.error, remediation: ["distilly collect feishu --help"] }], + }, + }; + } + const { options } = parsed; + const result = collectBrowser({ + root: options.baseDir, + person: options.person, + consentToken: options.consent, + capturePath: options.capture, + url: options.url, + label: options.label, + }); + + if (!options.json) { + if (result.receipt.status === "waiting-for-user-consent") { + err(`collect feishu (browser): ${result.receipt.errors[0]}`); + for (const item of result.receipt.unavailable) for (const step of item.remediation ?? []) err(` ${step}`); + } else if (result.receipt.status === "awaiting-host-capture") { + out("collect feishu (browser): waiting for the host capture"); + for (const step of result.receipt.plan.steps) out(` ${step}`); + } else if (result.ok) { + out(`collect feishu (browser): ${result.receipt.anchors.total} anchor(s) from ${options.capture}`); + for (const output of result.receipt.outputs) out(` ${output.kind}: ${output.path}`); + } + for (const warning of result.receipt.warnings ?? []) err(` warning: ${warning}`); + for (const failure of result.receipt.errors ?? []) err(` error: ${failure}`); + } + return result; +} diff --git a/src/collect/feishu-mcp.mjs b/src/collect/feishu-mcp.mjs new file mode 100644 index 00000000..5e540859 --- /dev/null +++ b/src/collect/feishu-mcp.mjs @@ -0,0 +1,442 @@ +/** + * feishu-mcp.mjs — the MCP route to Feishu (ported from `tools/feishu_mcp_client.py`). + * + * The Python client shelled out to `npx -y feishu-mcp --stdio` once per call, + * passing the app credential through the environment, and printed the tool's text + * result. The port keeps that contract and changes three things the pipeline + * needs: + * + * - the transport is **injected** (`transport.call(tool, args)`), so the whole + * route is testable without npx, a tenant or a network; + * - the credential only ever travels in the child process environment, never in + * argv, and a receipt can only ever name the config file (`feishu_config.json`); + * - the tool result is turned into a **document** (raw bytes + text + anchors), + * so an MCP-collected chat or doc lands in the evidence spine exactly like a + * file harvest does. + * + * Read-only by construction: the tool allowlist is closed, and an unknown tool is + * refused before anything is spawned. + */ + +import { spawn } from "node:child_process"; +import { join, resolve } from "node:path"; + +import { KnowledgeStore, sha256Hex } from "../knowledge/store.mjs"; +import { loadLedger, recordDocument, saveLedger } from "../knowledge/ledger.mjs"; +import { SourceFile, buildDocument, recordsFromCharSpans } from "../parse/common.mjs"; +import { parseFeishu } from "../parse/feishu.mjs"; +import { CONFIG_FILE, loadCredential, redact } from "./feishu.mjs"; + +export const CHANNEL = "feishu"; +export const MODE = "mcp"; +export const MCP_COMMAND = ["npx", "-y", "feishu-mcp", "--stdio"]; + +/** The only tools this client may call. Adding one is a deliberate act. */ +export const ALLOWED_TOOLS = Object.freeze([ + "get_wiki_node", + "get_doc_content", + "get_spreadsheet_content", + "get_chat_messages", + "list_wiki_nodes", +]); + +/** URL → (token, kind), the same patterns the Python client used. */ +const URL_PATTERNS = [ + [/\/wiki\/([A-Za-z0-9]+)/, "wiki"], + [/\/docx\/([A-Za-z0-9]+)/, "docx"], + [/\/docs\/([A-Za-z0-9]+)/, "doc"], + [/\/sheets\/([A-Za-z0-9]+)/, "sheet"], + [/\/base\/([A-Za-z0-9]+)/, "base"], +]; + +export function extractDocToken(url) { + for (const [pattern, kind] of URL_PATTERNS) { + const match = pattern.exec(String(url)); + if (match) return { token: match[1], kind }; + } + throw new Error(`cannot read a document token out of the URL: ${url}`); +} + +/** Which tool a URL maps to, and with which arguments. */ +export function toolForUrl(url) { + const { token, kind } = extractDocToken(url); + if (kind === "wiki") return { tool: "get_wiki_node", arguments: { token }, kind }; + if (kind === "docx" || kind === "doc") return { tool: "get_doc_content", arguments: { doc_token: token }, kind }; + if (kind === "sheet") return { tool: "get_spreadsheet_content", arguments: { spreadsheet_token: token }, kind }; + throw new Error(`no MCP tool reads a ${kind} document`); +} + +/** + * The default transport: one `npx -y feishu-mcp --stdio` process per call, JSON-RPC + * over stdin, credentials in the environment only. + */ +export function spawnMcpTransport({ command = MCP_COMMAND, timeoutMs = 30_000, spawnImpl = spawn } = {}) { + return { + async call(tool, args, { config, env = process.env } = {}) { + if (!ALLOWED_TOOLS.includes(tool)) throw new Error(`tool ${tool} is not on the allowlist`); + const payload = JSON.stringify({ jsonrpc: "2.0", method: "tools/call", params: { name: tool, arguments: args }, id: 1 }); + const childEnv = { + ...env, + FEISHU_APP_ID: config.app_id ?? "", + FEISHU_APP_SECRET: config.app_secret ?? "", + ...(config.mode === "user" && config.user_token ? { FEISHU_USER_ACCESS_TOKEN: config.user_token } : {}), + }; + return await new Promise((settle, fail) => { + const child = spawnImpl(command[0], command.slice(1), { env: childEnv, stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill?.(); + fail(new Error(`feishu-mcp did not answer within ${timeoutMs}ms`)); + }, timeoutMs); + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", (error) => { + clearTimeout(timer); + fail(new Error(error.code === "ENOENT" ? "npx was not found; install Node.js, or npm install -g feishu-mcp" : error.message)); + }); + child.on("close", (code) => { + clearTimeout(timer); + if (code !== 0) { + fail(new Error(`feishu-mcp exited ${code}: ${stderr.slice(0, 200) || "no stderr"}`)); + return; + } + try { + settle(JSON.parse(stdout.slice(stdout.indexOf("{")))); + } catch (error) { + fail(new Error(`feishu-mcp returned a non-JSON answer: ${error.message}`)); + } + }); + child.stdin.end(payload); + }); + }, + }; +} + +/** + * MCP tool results wrap their payload: `{result: [{type:"text", text:"…"}]}` or a + * plain string, and an error comes back as `{error: …}`. Both shapes are read the + * same way the Python client read them. + */ +export function readToolResult(result) { + if (result && typeof result === "object" && "error" in result && result.error) { + return { error: typeof result.error === "string" ? result.error : JSON.stringify(result.error) }; + } + const payload = result && typeof result === "object" && "result" in result ? result.result : result; + if (typeof payload === "string") return { text: payload }; + if (Array.isArray(payload)) { + const parts = payload + .map((item) => (item && typeof item === "object" && item.type === "text" ? item.text : null)) + .filter((text) => typeof text === "string"); + if (parts.length > 0) return { text: parts.join("\n") }; + return { value: payload }; + } + if (payload && typeof payload === "object") return { value: payload }; + return { text: String(payload ?? "") }; +} + +/** Message arrays arrive as text (a JSON string) or already parsed. */ +export function asMessages({ text, value }) { + if (Array.isArray(value)) return value; + if (typeof text === "string") { + try { + const parsed = JSON.parse(text); + if (Array.isArray(parsed)) return parsed; + if (parsed && Array.isArray(parsed.items)) return parsed.items; + } catch { + return null; + } + } + return null; +} + +/** A document whose paragraphs are the lines of a fetched text (docs, sheets, wiki). */ +function documentFromText({ text, name, source, method, kind, credentialed, credentialSource, origin }) { + const file = new SourceFile({ path: name, name, raw: new Uint8Array(Buffer.from(text, "utf8")) }); + const spans = []; + const pattern = /[^\n]+/g; + let match; + while ((match = pattern.exec(text)) !== null) { + spans.push({ text: match[0], charStart: match.index, charEnd: match.index + match[0].length, kind: "paragraph" }); + } + return buildDocument({ + parser: "feishu-mcp", + format: kind, + kind: "doc", + method, + credentialed, + credential_source: credentialSource, + credential_file: CONFIG_FILE, + source, + origin: origin ?? name, + files: [file], + records: recordsFromCharSpans(file, spans), + }); +} + +/** + * Run one MCP read and record it. + * + * @param {{transport: object, config: object, tool: string, arguments: object, target: string, + * root: string, person: string, family?: string, source?: string, now?: string, + * env?: object, label?: string}} input + */ +/** Parse `collect feishu --mode mcp` arguments; returns `{options}` or `{error}`. */ +export function parseMcpArgs(argv) { + const options = { person: null, baseDir: process.cwd(), url: null, chatId: null, target: null, limit: 500, json: false, label: null }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--json") options.json = true; + else if (arg === "--mode") { + const value = argv[index + 1]; + if (value !== "mcp") return { error: `feishu-mcp handles --mode mcp only (got ${value ?? "nothing"})` }; + index += 1; + } else if (["--person", "--base-dir", "--url", "--chat-id", "--target", "--limit", "--label"].includes(arg)) { + const value = argv[index + 1]; + if (value === undefined) return { error: `${arg} requires a value` }; + const key = { "--person": "person", "--base-dir": "baseDir", "--url": "url", "--chat-id": "chatId", "--target": "target", "--limit": "limit", "--label": "label" }[arg]; + options[key] = key === "limit" ? Number(value) : value; + index += 1; + } else if (arg.startsWith("--")) return { error: `unknown option: ${arg}` }; + else if (!options.chatId) options.chatId = arg; + else return { error: `unexpected argument: ${arg}` }; + } + if (!options.url && !options.chatId) return { error: "collect feishu --mode mcp needs --url or --chat-id " }; + if (!options.person) return { error: "--person is required" }; + return { options }; +} + +export async function collectViaMcp(input) { + const { + transport, + config, + tool, + arguments: toolArgs, + target, + root, + person, + family = "colleague", + source = CHANNEL, + now = new Date().toISOString(), + env = process.env, + } = input; + + if (!ALLOWED_TOOLS.includes(tool)) { + return { ok: false, exitCode: 2, receipt: { command: "collect", channel: CHANNEL, mode: MODE, ok: false, warnings: [], errors: [`tool ${tool} is not on the allowlist`], unavailable: [{ channel: CHANNEL, reason: `unknown MCP tool ${tool}`, remediation: [`allowed: ${ALLOWED_TOOLS.join(", ")}`] }] } }; + } + + const warnings = []; + const secrets = [config.app_secret, config.user_token].filter(Boolean); + let result; + try { + result = await transport.call(tool, toolArgs, { config, env }); + } catch (error) { + return { + ok: false, + exitCode: 1, + receipt: { + command: "collect", + channel: CHANNEL, + mode: MODE, + ok: false, + person: person ?? null, + target: target ?? null, + credential_file: CONFIG_FILE, + warnings, + errors: [redact(error.message, secrets)], + unavailable: [ + { + channel: CHANNEL, + reason: redact(`MCP call failed: ${error.message}`, secrets), + remediation: ["npm install -g feishu-mcp", "check the app scopes: docs:doc:readonly, wiki:wiki:readonly, im:message:readonly"], + }, + ], + }, + }; + } + + const read = readToolResult(result); + if (read.error) { + return { + ok: false, + exitCode: 1, + receipt: { + command: "collect", + channel: CHANNEL, + mode: MODE, + ok: false, + person: person ?? null, + target: target ?? null, + credential_file: CONFIG_FILE, + warnings, + errors: [redact(read.error, secrets)], + unavailable: [{ channel: CHANNEL, reason: redact(`MCP returned an error: ${read.error}`, secrets), remediation: ["check the token scopes and the target id"] }], + }, + }; + } + + const messages = asMessages(read); + const label = input.label ?? `${tool}-${String(target ?? "result").replace(/[^A-Za-z0-9_-]+/g, "-").slice(0, 40)}`; + const rawText = typeof read.text === "string" ? read.text : JSON.stringify(read.value ?? messages ?? "", null, 2); + + const store = new KnowledgeStore(join(resolve(root), "skills", family, person)); + const ledger = loadLedger(store); + + let document; + if (messages) { + const file = new SourceFile({ path: `${label}.json`, name: `${label}.json`, raw: new Uint8Array(Buffer.from(JSON.stringify(messages), "utf8")) }); + document = parseFeishu(file, { + source, + method: `mcp-${tool}`, + credentialed: true, + credential_source: input.credentialSource ?? "feishu_config.json", + credential_file: CONFIG_FILE, + }); + warnings.push(...(document.warnings ?? []).map((warning) => warning.message ?? warning)); + } else { + document = documentFromText({ + text: rawText, + name: `${label}.txt`, + source, + method: `mcp-${tool}`, + kind: tool === "get_wiki_node" ? "wiki" : tool === "get_spreadsheet_content" ? "sheet" : "docx", + credentialed: true, + credentialSource: input.credentialSource ?? "feishu_config.json", + origin: target ?? label, + }); + } + + const recorded = recordDocument(store, ledger, { ...document, fetched_at: now }, { fetched_at: now }); + saveLedger(store, ledger); + const raw = recorded.written.files[0]; + const text = recorded.written.text; + const outputs = [ + ...(raw ? [{ path: raw.relativePath, sha256: raw.sha256, bytes: raw.bytes, kind: "raw" }] : []), + ...(text ? [{ path: text.relativePath, sha256: text.sha256, bytes: text.bytes, kind: "text" }] : []), + ]; + + const receipt = { + command: "collect", + channel: CHANNEL, + mode: MODE, + ok: true, + person, + target: target ?? null, + tool, + credential_file: CONFIG_FILE, + messages: messages ? messages.length : 0, + anchors: { total: recorded.entry?.anchor_count ?? 0, cited: 0 }, + outputs, + warnings, + unavailable: [], + }; + return { ok: true, exitCode: 0, receipt: secrets.reduce((value, secret) => redact(JSON.stringify(value), [secret]) && value, receipt) }; +} + +/** Messages from `get_chat_messages` when the tool returns raw API items. */ +export function normaliseMcpMessages(items) { + return (items ?? []).map((item) => ({ + message_id: item.message_id ?? item.id ?? null, + msg_type: item.msg_type ?? (item.content ? "text" : "unknown"), + create_time: item.create_time ?? item.timestamp ?? null, + sender: item.sender ?? { id: item.sender_id ?? null }, + body: item.body ?? { content: item.content ?? "" }, + })); +} + +/** + * `distilly collect feishu --mode mcp …` + * + * Mirrors the other channels' CLI entry: parse, load the credential (naming the + * config file, never its contents), route one allowed tool call through the MCP + * client, and print either the receipt or a human line. `io.transport` lets the + * tests drive the client without spawning `npx`. + */ +export async function runCollectCli(argv, io = {}) { + const out = typeof io.stdout === "function" ? io.stdout : (line) => process.stdout.write(`${line}\n`); + const err = typeof io.stderr === "function" ? io.stderr : (line) => process.stderr.write(`${line}\n`); + const parsed = parseMcpArgs(argv); + if (parsed.error) { + const receipt = { + command: "collect", + channel: CHANNEL, + mode: MODE, + ok: false, + credential_file: CONFIG_FILE, + warnings: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + errors: [parsed.error], + unavailable: [{ channel: CHANNEL, reason: parsed.error, remediation: ["distilly collect feishu --help"] }], + }; + if (!parsed.options?.json) err(`collect feishu (mcp): ${parsed.error}`); + else out(JSON.stringify(receipt, null, 2)); + return { ok: false, exitCode: 2, receipt }; + } + + const { options } = parsed; + // `loadCredential` *throws* a `CollectFailure` when there is no credential at all + // (rather than returning empty values), so the CLI has to catch it to produce the + // contract-shaped failure with `credential_file` named and no secret in it. + let credential = null; + let credentialError = null; + try { + credential = loadCredential({ configFile: CONFIG_FILE, envKeys: ["FEISHU_APP_ID", "FEISHU_APP_SECRET"], fields: ["app_id", "app_secret"], env: io.env ?? process.env }); + } catch (error) { + credentialError = error; + } + if (credentialError || !credential.values?.app_id || !credential.values?.app_secret) { + const reason = credentialError ? credentialError.message : `no usable credential: ${CONFIG_FILE} is missing or incomplete`; + const receipt = { + command: "collect", + channel: CHANNEL, + mode: MODE, + ok: false, + credential_file: CONFIG_FILE, + warnings: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + errors: [reason], + unavailable: [ + { + channel: CHANNEL, + reason, + remediation: credentialError?.remediation ?? [`create ~/.distilly/${CONFIG_FILE} with app_id and app_secret`, "or export FEISHU_APP_ID / FEISHU_APP_SECRET"], + }, + ], + }; + if (options.json) out(JSON.stringify(receipt, null, 2)); + else { + err(`collect feishu (mcp): ${reason}`); + for (const step of receipt.unavailable[0].remediation) err(` fix: ${step}`); + } + return { ok: false, exitCode: 1, receipt }; + } + + // A chat id and a document URL route to different tools: `toolForUrl` parses a + // *document* URL and has nothing to say about `oc_…`, so asking it about a chat + // id threw before any call was made. + const routed = options.chatId + ? { tool: "get_chat_messages", arguments: { chat_id: options.chatId, page_size: 50 }, kind: "chat" } + : toolForUrl(options.url); + const tool = routed.tool; + const target = options.target ?? options.chatId ?? extractDocToken(options.url).token; + const result = await collectViaMcp({ + transport: io.transport, + config: credential.values, + tool, + arguments: routed.arguments, + target, + root: options.baseDir, + person: options.person, + env: io.env ?? process.env, + }); + + if (options.json) out(JSON.stringify(result.receipt, null, 2)); + else if (result.ok) out(`collect feishu (mcp): ${result.receipt.messages} message(s) via ${result.receipt.tool}`); + else { + for (const failure of result.receipt.errors ?? []) err(`collect feishu (mcp): ${failure}`); + for (const item of result.receipt.unavailable ?? []) for (const step of item.remediation ?? []) err(` fix: ${step}`); + } + return result; +} diff --git a/src/collect/feishu.mjs b/src/collect/feishu.mjs new file mode 100644 index 00000000..01db43b8 --- /dev/null +++ b/src/collect/feishu.mjs @@ -0,0 +1,801 @@ +/** + * feishu.mjs — credentialed collection from Feishu / Lark open APIs. + * + * Legacies ported: `tools/feishu_auto_collector.py` (960 lines, SDK `requests`). + * This module keeps the same credential location and the same two read paths + * (tenant token for group chats, user token for p2p), but: + * + * - the network goes through an **injected `fetch`** (default `globalThis.fetch`), + * so the whole channel is testable with a mock and never needs a live tenant; + * - raw response bytes are stored **verbatim** in `knowledge/raw/feishu/…` and + * registered in `knowledge/index.json` (contract §2); + * - pagination is cursor-driven, rate limits back off with `Retry-After`, and an + * interrupted run leaves a checkpoint so the next run resumes at the cursor + * instead of re-fetching finished pages; + * - credential values never reach stdout, stderr or a receipt — only the file + * name (`feishu_config.json`) is ever printed. See `redact()` / `scrub()`. + * + * Read-only by construction: every request goes through `assertReadOnly()`, which + * rejects any method+path pair outside `ALLOWED_MUTATIONS` (the tenant token + * exchange — a POST that changes no user-visible state). There is no code path + * that can like, follow, post or send anything; that is a capability-level + * guarantee, not a prompt instruction. + */ + +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +export const CHANNEL = "feishu"; +/** Only the *name* may ever be printed. Never its contents. */ +export const CONFIG_FILE = "feishu_config.json"; +export const LEGACY_CONFIG_FILE = join(".colleague-skill", CONFIG_FILE); +export const DEFAULT_BASE_URL = "https://open.feishu.cn/open-apis"; +export const DEFAULT_PAGE_SIZE = 50; +export const DEFAULT_MAX_PAGES = 10; +export const DEFAULT_MAX_RETRIES = 4; +export const DEFAULT_MAX_BACKOFF_MS = 60_000; + +/** Env fallbacks, checked before the config file. Names are not secret. */ +export const ENV_KEYS = { + appId: ["DISTILLY_FEISHU_APP_ID", "FEISHU_APP_ID"], + appSecret: ["DISTILLY_FEISHU_APP_SECRET", "FEISHU_APP_SECRET"], + userToken: ["DISTILLY_FEISHU_USER_ACCESS_TOKEN", "FEISHU_USER_ACCESS_TOKEN"], +}; + +/** + * The only non-GET calls this module may make. Anything else fails loudly + * before the request leaves the process. + */ +export const ALLOWED_MUTATIONS = [ + { + method: "POST", + path: "/auth/v3/tenant_access_token/internal", + why: "tenant token exchange; creates no user-visible state", + }, +]; + +const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); +const REMEDIATION_SETUP = [ + `create ~/.distilly/${CONFIG_FILE} (chmod 600) with {"app_id": "cli_…", "app_secret": "…"}`, + " app: https://open.feishu.cn → 开发者后台 → 创建企业自建应用 → 权限 im:message:readonly, im:chat:readonly", + "or export DISTILLY_FEISHU_APP_ID and DISTILLY_FEISHU_APP_SECRET for this shell only", + "private (p2p) chats additionally need a user token: DISTILLY_FEISHU_USER_ACCESS_TOKEN", +]; + +// ─── failures ──────────────────────────────────────────────────────────────── + +/** An expected, loud failure. `reason` is a stable machine-readable token. */ +export class CollectFailure extends Error { + constructor(reason, message, { remediation = [], exitCode = 1, kind = "failure" } = {}) { + super(message); + this.name = "CollectFailure"; + this.reason = reason; + this.remediation = remediation; + this.exitCode = exitCode; + this.kind = kind; + } +} + +// ─── secret discipline ─────────────────────────────────────────────────────── + +/** Replace every credential value with `[redacted]`; keep file names visible. */ +export function redact(text, secrets = []) { + let output = String(text ?? ""); + for (const secret of secrets) { + if (typeof secret !== "string" || secret.length < 4) continue; + output = output.split(secret).join("[redacted]"); + } + return output; +} + +/** Mechanical net: no secret can survive a round-trip through a receipt. */ +export function scrub(value, secrets = []) { + return JSON.parse( + JSON.stringify(value, (_key, item) => (typeof item === "string" ? redact(item, secrets) : item)), + ); +} + +export function distillyHome(env = process.env) { + const override = env?.DISTILLY_HOME; + return override ? resolve(String(override)) : join(homedir(), ".distilly"); +} + +export function credentialPaths(env = process.env) { + return { + primary: join(distillyHome(env), CONFIG_FILE), + legacy: join(homedir(), LEGACY_CONFIG_FILE), + }; +} + +/** + * Load `{app_id, app_secret, user_access_token?}` from env or the config file. + * Returns values for the caller only; every message names the *file*, never a + * value, and the config file is never printed or copied anywhere else. + */ +export function loadCredential({ env = process.env, readFile = readFileSync } = {}) { + const pick = (names) => { + for (const name of names) { + const value = env?.[name]; + if (typeof value === "string" && value.trim() !== "") return value.trim(); + } + return null; + }; + + const envAppId = pick(ENV_KEYS.appId); + const envAppSecret = pick(ENV_KEYS.appSecret); + const envUserToken = pick(ENV_KEYS.userToken); + if (envAppId && envAppSecret) { + return { + ok: true, + source: "env", + configFile: CONFIG_FILE, + path: null, + values: { app_id: envAppId, app_secret: envAppSecret, user_access_token: envUserToken }, + }; + } + + const { primary, legacy } = credentialPaths(env); + for (const [path, source] of [ + [primary, "config"], + [legacy, "legacy-config"], + ]) { + if (!existsSync(path)) continue; + let parsed; + try { + parsed = JSON.parse(readFile(path, "utf8")); + } catch (error) { + throw new CollectFailure( + "bad-credential-file", + `${CONFIG_FILE} is not valid JSON (${redact(error.message)}); rewrite it with {"app_id": "cli_…", "app_secret": "…"}`, + { remediation: REMEDIATION_SETUP }, + ); + } + const appId = parsed.app_id ?? parsed.appId ?? null; + const appSecret = parsed.app_secret ?? parsed.appSecret ?? null; + const userToken = parsed.user_access_token ?? parsed.userToken ?? null; + if (!appId || !appSecret) { + throw new CollectFailure( + "incomplete-credential", + `${CONFIG_FILE} is missing app_id / app_secret`, + { remediation: REMEDIATION_SETUP }, + ); + } + return { + ok: true, + source, + configFile: CONFIG_FILE, + path, + values: { app_id: appId, app_secret: appSecret, user_access_token: userToken }, + }; + } + + throw new CollectFailure( + "no-credential", + `no credential at ~/.distilly/${CONFIG_FILE}`, + { remediation: REMEDIATION_SETUP }, + ); +} + +// ─── read-only guard + HTTP ────────────────────────────────────────────────── + +export function assertReadOnly(url, method = "GET") { + const verb = String(method).toUpperCase(); + if (!MUTATING_METHODS.has(verb)) return true; + const path = (() => { + try { + return new URL(url).pathname; + } catch { + return String(url); + } + })(); + const allowed = ALLOWED_MUTATIONS.some((entry) => entry.method === verb && path.endsWith(entry.path)); + if (!allowed) { + throw new CollectFailure( + "write-operation-refused", + `refusing ${verb} ${path}: this collector never writes to ${CHANNEL}`, + { remediation: ["collectors are read-only; remove the mutating call instead of allowlisting it"] }, + ); + } + return true; +} + +export function parseRetryAfter(headerValue, nowMs = Date.now()) { + if (headerValue === null || headerValue === undefined || headerValue === "") return null; + const seconds = Number(headerValue); + if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000); + const date = Date.parse(String(headerValue)); + if (Number.isFinite(date)) return Math.max(0, date - nowMs); + return null; +} + +/** `Retry-After` wins; otherwise exponential backoff, capped. */ +export function backoffDelay(attempt, retryAfterMs = null, options = {}) { + const { baseMs = 500, maxMs = DEFAULT_MAX_BACKOFF_MS } = options; + if (Number.isFinite(retryAfterMs) && retryAfterMs !== null) return Math.min(retryAfterMs, maxMs); + return Math.min(baseMs * 2 ** Math.max(0, attempt - 1), maxMs); +} + +export function defaultSleep(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +/** + * One JSON request with retry/backoff. Never returns a body on 4xx/5xx. + * @returns {Promise<{status: number, text: string, json: any, attempts: number}>} + */ +export async function requestJson(options) { + const { + fetchImpl, + url, + method = "GET", + headers = {}, + body, + maxRetries = DEFAULT_MAX_RETRIES, + sleep = defaultSleep, + secrets = [], + onRetry = () => {}, + authRemediation = REMEDIATION_SETUP, + } = options; + + assertReadOnly(url, method); + if (typeof fetchImpl !== "function") { + throw new CollectFailure("no-fetch", "no fetch implementation available", { + remediation: ["run on Node >= 20, or pass an injected fetch"], + }); + } + + let attempt = 0; + for (;;) { + attempt += 1; + let response; + try { + response = await fetchImpl(url, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); + } catch (error) { + const message = redact(error?.message ?? String(error), secrets); + if (attempt > maxRetries) { + throw new CollectFailure("network-error", `request failed: ${message}`, { + remediation: ["check the network/proxy and retry", ...authRemediation], + }); + } + await sleep(backoffDelay(attempt)); + onRetry({ attempt, status: null, delayMs: backoffDelay(attempt), reason: message }); + continue; + } + + const status = Number(response?.status ?? 0); + const retryAfterMs = parseRetryAfter(response?.headers?.get?.("retry-after") ?? null); + + if (status === 429 || status >= 500) { + if (attempt > maxRetries) { + throw new CollectFailure( + status === 429 ? "rate-limited" : "server-error", + status === 429 + ? `rate limited (HTTP 429) after ${maxRetries} retries` + : `server error (HTTP ${status}) after ${maxRetries} retries`, + { + remediation: [ + "retry later; already-fetched pages stay on disk and the cursor is checkpointed", + `lower --limit / --max-pages to stay under the ${CHANNEL} quota`, + ], + }, + ); + } + const delayMs = backoffDelay(attempt, retryAfterMs); + onRetry({ attempt, status, delayMs, reason: `HTTP ${status}` }); + await sleep(delayMs); + continue; + } + + const text = await response.text(); + if (status === 401 || status === 403) { + throw new CollectFailure("unauthorized", `HTTP ${status} from ${CHANNEL}; credential rejected or missing scope`, { + remediation: [ + "the credential in ~/.distilly/" + CONFIG_FILE + " was rejected — regenerate it", + "check the app scopes: im:message:readonly, im:chat:readonly (add im:message for p2p)", + ], + }); + } + if (status >= 400) { + throw new CollectFailure("http-error", `HTTP ${status}: ${redact(text.slice(0, 200), secrets)}`, { + remediation: ["check the request parameters", ...authRemediation], + }); + } + + let json = null; + try { + json = JSON.parse(text); + } catch { + throw new CollectFailure("invalid-json", `${CHANNEL} returned a non-JSON body`, { + remediation: ["retry later; if it persists the endpoint may have changed"], + }); + } + return { status, text, json, attempts: attempt }; + } +} + +// ─── sink: raw bytes + ledger ──────────────────────────────────────────────── + +export function sha256Hex(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function slug(text, fallback = "target") { + const slugged = String(text ?? "") + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^[-._]+|[-._]+$/g, "") + .slice(0, 64); + return slugged || fallback; +} + +/** `--person lin-gong` → `skills/colleague/lin-gong/knowledge`, else `/knowledge`. */ +export function knowledgeRoot({ root = process.cwd(), person, family = "colleague" } = {}) { + return person ? join(resolve(root), "skills", slug(family), slug(person), "knowledge") : join(resolve(root), "knowledge"); +} + +/** + * Write raw bytes verbatim, creating `knowledge/raw//` only now — a run + * that fails before its first successful page leaves no `knowledge/` behind. + */ +export function writeRaw(knowledgeDir, name, bytes) { + const dir = join(knowledgeDir, "raw", CHANNEL); + mkdirSync(dir, { recursive: true }); + const path = join(dir, `${slug(name, "page")}.json`); + const staging = `${path}.${process.pid}.tmp`; + const buffer = Buffer.from(bytes); + try { + writeFileSync(staging, buffer); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { + path, + relativePath: `raw/${CHANNEL}/${slug(name, "page")}.json`, + bytes: buffer.length, + sha256: sha256Hex(buffer), + }; +} + +/** + * Upsert entries into `knowledge/index.json` (an array, contract §2). Entries + * with an identical `id` are replaced, which makes re-running a collect + * idempotent in the ledger. + */ +export function appendLedger(knowledgeDir, entries) { + if (entries.length === 0) return { path: join(knowledgeDir, "index.json"), added: 0, total: 0, existed: false }; + const path = join(knowledgeDir, "index.json"); + let existing = []; + if (existsSync(path)) { + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + existing = Array.isArray(parsed) ? parsed : []; + } catch (error) { + throw new CollectFailure("bad-ledger", `knowledge/index.json is not valid JSON: ${redact(error.message)}`, { + remediation: ["repair or remove knowledge/index.json, then rerun the collect"], + }); + } + } + const byId = new Map(existing.filter((e) => e && typeof e === "object").map((e) => [e.id, e])); + let added = 0; + for (const entry of entries) { + if (!byId.has(entry.id)) added += 1; + byId.set(entry.id, entry); + } + const merged = [...byId.values()].sort((a, b) => String(a.id).localeCompare(String(b.id))); + const staging = `${path}.${process.pid}.tmp`; + try { + writeFileSync(staging, `${JSON.stringify(merged, null, 2)}\n`); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { path, added, total: merged.length, existed: existing.length > 0 }; +} + +// ─── resume checkpoints (outside the repo, next to the credential) ─────────── + +export function statePath({ env = process.env, root = process.cwd(), target } = {}) { + const key = sha256Hex(Buffer.from(`${resolve(root)}\n${target ?? ""}`, "utf8")).slice(0, 12); + return join(distillyHome(env), "state", `${CHANNEL}-${key}.json`); +} + +export function readCheckpoint(options) { + const path = statePath(options); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +export function writeCheckpoint(options, value) { + const path = statePath(options); + mkdirSync(dirname(path), { recursive: true }); + const staging = `${path}.${process.pid}.tmp`; + writeFileSync(staging, `${JSON.stringify(value, null, 2)}\n`); + renameSync(staging, path); + return path; +} + +export function clearCheckpoint(options) { + const path = statePath(options); + if (existsSync(path)) rmSync(path, { force: true }); + return path; +} + +// ─── collect ───────────────────────────────────────────────────────────────── + +export const BROWSER_STEPS = [ + "This module does not drive a browser: computer use is a host capability.", + `Host: open the Feishu client, open the target chat, scroll to the requested range.`, + "Host: hand the captured payload back to the user (copy the text into a file).", +]; + +/** + * Collect messages from one Feishu chat. + * + * @param {object} options + * @param {Function} [options.fetch] injected fetch (default `globalThis.fetch`) + * @param {object} [options.env] environment (default `process.env`) + * @param {string} [options.root] workspace root that contains `knowledge/` + * @param {string} [options.person] Skill slug; then the root is `skills///knowledge` + * @param {string} options.chatId `oc_…` chat container id (required) + * @param {number} [options.limit] page size + * @param {number} [options.maxPages] + * @param {number} [options.maxRetries] + * @param {string} [options.since] explicit cursor to resume from + * @param {boolean} [options.resume] read/write the checkpoint (default true) + * @param {Function} [options.sleep] injected sleeper so tests never really wait + * @param {string} [options.now] ISO timestamp used for `fetched_at` + */ +export async function collect(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + chatId, + limit = DEFAULT_PAGE_SIZE, + maxPages = DEFAULT_MAX_PAGES, + maxRetries = DEFAULT_MAX_RETRIES, + since, + resume = true, + sleep = defaultSleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_FEISHU_BASE_URL || DEFAULT_BASE_URL, + useUserToken = Boolean(env?.DISTILLY_FEISHU_USE_USER_TOKEN), + onProgress = () => {}, + } = options; + + const knowledgeDir = knowledgeRoot({ root, person, family }); + const outputs = []; + const warnings = []; + const ledgerEntries = []; + const retries = []; + const onRetry = (info) => { + retries.push(info); + warnings.push(`retry ${info.attempt} after ${info.reason} (waited ${info.delayMs}ms)`); + onProgress(`retry ${info.attempt}: ${info.reason}`); + }; + + let secrets = []; + let credential = null; + let pages = 0; + let items = 0; + let requests = 0; + let cursor = since ?? null; + let checkpoint = null; + + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + retries, + }; + const fail = (failure) => { + const receipt = { + ...base, + ok: false, + person: person ?? null, + chat_id: chatId ?? null, + pages, + items, + requests, + cursor, + resumed_from: checkpoint?.cursor ?? null, + errors: [redact(failure.message, secrets)], + unavailable: [ + { + channel: CHANNEL, + reason: redact(`${failure.reason}: ${failure.message}`, secrets), + remediation: failure.remediation ?? [], + }, + ], + }; + return { ok: false, exitCode: failure.exitCode ?? 1, receipt: scrub(receipt, secrets) }; + }; + + try { + if (!chatId) { + throw new CollectFailure("missing-target", "collect feishu needs --chat-id ", { + remediation: [ + "find the chat id in the Feishu URL, or via GET /im/v1/chats with the app credential", + ], + }); + } + + credential = loadCredential({ env }); + secrets = [credential.values.app_secret, credential.values.user_access_token].filter(Boolean); + base.credential_source = credential.source; + + if (resume) checkpoint = readCheckpoint({ env, root, target: chatId }); + if (since === undefined && checkpoint?.cursor) { + cursor = checkpoint.cursor; + pages = Number(checkpoint.pages ?? 0); + warnings.push(`resuming from checkpoint cursor (page ${pages} done)`); + } + + const userToken = useUserToken ? credential.values.user_access_token : null; + if (useUserToken && !userToken) { + warnings.push("DISTILLY_FEISHU_USE_USER_TOKEN is set but no user token is configured; using the tenant token"); + } + let bearer = userToken; + if (!bearer) { + const auth = await requestJson({ + fetchImpl, + url: `${baseUrl}/auth/v3/tenant_access_token/internal`, + method: "POST", + headers: { "content-type": "application/json" }, + body: { app_id: credential.values.app_id, app_secret: credential.values.app_secret }, + maxRetries, + sleep, + secrets, + onRetry, + }); + requests += 1; + if (auth.json?.code !== 0 || !auth.json?.tenant_access_token) { + throw new CollectFailure( + "auth-failed", + `Feishu rejected the app credential (code=${auth.json?.code ?? "?"})`, + { remediation: REMEDIATION_SETUP }, + ); + } + bearer = auth.json.tenant_access_token; + secrets = [...secrets, bearer]; + } + + let hasMore = true; + const pageSize = Math.min(Math.max(1, Number(limit) || DEFAULT_PAGE_SIZE), 50); + while (hasMore) { + if (pages >= maxPages) { + warnings.push(`stopped after --max-pages ${maxPages}; rerun to continue from the cursor`); + break; + } + const params = new URLSearchParams({ + container_id_type: "chat", + container_id: chatId, + page_size: String(pageSize), + sort_type: "ByCreateTimeDesc", + }); + if (cursor) params.set("page_token", cursor); + + const response = await requestJson({ + fetchImpl, + url: `${baseUrl}/im/v1/messages?${params.toString()}`, + headers: { authorization: `Bearer ${bearer}` }, + maxRetries, + sleep, + secrets, + onRetry, + }); + requests += 1; + + if (response.json?.code !== 0) { + throw new CollectFailure( + "api-error", + `Feishu code=${response.json?.code ?? "?"}: ${redact(response.json?.msg ?? "", secrets)}`, + { + remediation: [ + "check the app scopes (im:message:readonly) and that the bot is in the chat", + ...REMEDIATION_SETUP, + ], + }, + ); + } + + pages += 1; + const data = response.json?.data ?? {}; + const pageItems = Array.isArray(data.items) ? data.items : []; + items += pageItems.length; + + const stored = writeRaw(knowledgeDir, `${chatId}-p${String(pages).padStart(3, "0")}`, response.text); + outputs.push({ path: stored.path, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }); + ledgerEntries.push({ + id: `${CHANNEL}:${slug(chatId)}:p${String(pages).padStart(3, "0")}`, + kind: "raw", + origin: stored.relativePath, + source: CHANNEL, + fetched_at: now, + bytes: stored.bytes, + sha256: stored.sha256, + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + method: useUserToken && userToken ? "api-user-token" : "api-tenant-token", + items: pageItems.length, + warnings: [], + }); + + cursor = data.page_token ?? null; + hasMore = Boolean(data.has_more) && Boolean(cursor); + if (resume) { + writeCheckpoint( + { env, root, target: chatId }, + { channel: CHANNEL, target: chatId, cursor, pages, items, updated_at: now }, + ); + } + onProgress(`page ${pages}: ${pageItems.length} items, has_more=${hasMore}`); + } + + const ledger = appendLedger(knowledgeDir, ledgerEntries); + if (resume) clearCheckpoint({ env, root, target: chatId }); + + const receipt = { + ...base, + ok: true, + person: person ?? null, + chat_id: chatId, + pages, + items, + requests, + cursor, + resumed_from: checkpoint?.cursor ?? null, + outputs, + ledger: { path: ledger.path, added: ledger.added, total: ledger.total }, + unavailable: [], + }; + return { ok: true, exitCode: 0, receipt: scrub(receipt, secrets) }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + const result = fail(error); + // Keep whatever was already written; only the ledger is flushed for those pages. + if (ledgerEntries.length > 0) { + try { + const ledger = appendLedger(knowledgeDir, ledgerEntries); + result.receipt.ledger = { path: ledger.path, added: ledger.added, total: ledger.total }; + result.receipt.partial = true; + } catch { + result.receipt.warnings.push("could not register the partial pages in knowledge/index.json"); + } + } + return result; + } +} + +// ─── CLI ───────────────────────────────────────────────────────────────────── + +export const HELP = `distilly collect feishu — 飞书消息采集（只读）/ Feishu message collection (read-only) + +用法 (zh): + distilly collect feishu --chat-id [--person ] [--root ] + [--limit 50] [--max-pages 10] [--max-retries 4] + [--since ] [--no-resume] [--json] + +凭据：~/.distilly/${CONFIG_FILE}（app_id / app_secret，0600）或环境变量 + DISTILLY_FEISHU_APP_ID / DISTILLY_FEISHU_APP_SECRET。 + 错误信息只出现配置文件名，绝不出现值。缺凭据 → 非零退出 + 补救步骤。 +权限：im:message:readonly、im:chat:readonly；私聊另需 user_access_token。 +只读：本模块没有任何点赞/关注/发帖/私信调用，写操作在 assertReadOnly() 里被拒绝。 + +--- +## English + distilly collect feishu --chat-id [--person ] [--limit N] [--json] + +Credentials: ~/.distilly/${CONFIG_FILE} or DISTILLY_FEISHU_APP_ID / DISTILLY_FEISHU_APP_SECRET. +Raw pages land verbatim in knowledge/raw/feishu/ and are registered in knowledge/index.json. +Rate limits honour Retry-After; an interrupted run resumes from its checkpoint cursor. +`; + +export function parseFlags(argv) { + const flags = { _: [] }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (!arg.startsWith("--")) { + flags._.push(arg); + continue; + } + const name = arg.slice(2); + if (name === "json" || name === "help" || name === "no-resume" || name === "use-user-token") { + flags[name] = true; + continue; + } + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + flags[name] = value; + index += 1; + } + return flags; +} + +/** + * @param {string[]} argv arguments after `collect feishu` + * @param {{env?: object, stdout?: Function, stderr?: Function, fetch?: Function}} [io] + * @returns {Promise} exit code + */ +export async function runCollectCli(argv, io = {}) { + const out = io.stdout ?? ((line) => process.stdout.write(`${line}\n`)); + const err = io.stderr ?? ((line) => process.stderr.write(`${line}\n`)); + + let flags; + try { + flags = parseFlags(argv); + } catch (error) { + err(`Error: ${error.message}`); + return 1; + } + if (flags.help) { + out(HELP); + return 0; + } + + const result = await collect({ + fetch: io.fetch ?? globalThis.fetch, + env: io.env ?? process.env, + root: flags.root ?? process.cwd(), + person: flags.person, + family: flags.family, + chatId: flags["chat-id"], + limit: flags.limit ? Number(flags.limit) : undefined, + maxPages: flags["max-pages"] ? Number(flags["max-pages"]) : undefined, + maxRetries: flags["max-retries"] ? Number(flags["max-retries"]) : undefined, + since: flags.since, + resume: !flags["no-resume"], + sleep: io.sleep, + now: io.now, + baseUrl: flags["base-url"], + }); + + if (flags.json) out(JSON.stringify(result.receipt, null, 2)); + if (result.ok) { + out(`collected ${result.receipt.items} item(s) in ${result.receipt.pages} page(s) → ${result.receipt.outputs.length} raw file(s)`); + for (const warning of result.receipt.warnings) err(`warning: ${warning}`); + } else { + err(`Error: ${result.receipt.errors?.[0] ?? "collect failed"}`); + for (const entry of result.receipt.unavailable) { + err(`unavailable: ${entry.channel} — ${entry.reason}`); + for (const step of entry.remediation ?? []) err(` fix: ${step}`); + } + for (const warning of result.receipt.warnings) err(`warning: ${warning}`); + } + return result.exitCode; +} diff --git a/src/collect/gmail.mjs b/src/collect/gmail.mjs new file mode 100644 index 00000000..035111e7 --- /dev/null +++ b/src/collect/gmail.mjs @@ -0,0 +1,390 @@ +/** + * gmail.mjs — Gmail messages, read with an OAuth refresh token. + * + * The read path is deliberately thin: list message ids, fetch each one as **raw + * MIME**, and hand the bytes to the email parser that already exists + * (`src/parse/email.mjs`). Headers, charsets, HTML fallbacks and attachment + * warnings are therefore identical to a locally harvested `.eml` — one + * implementation, two doors. + * + * Read-only by construction: the allowlist has one POST (the OAuth token + * exchange) and GETs under `/gmail/v1/users/`. Nothing here can send, label, + * trash or modify a message. + */ + +import { join, resolve } from "node:path"; + +import { KnowledgeStore } from "../knowledge/store.mjs"; +import { loadLedger, recordDocument, saveLedger } from "../knowledge/ledger.mjs"; +import { SourceFile } from "../parse/common.mjs"; +import { parseEmail } from "../parse/email.mjs"; +import { + CollectFailure, + DEFAULT_MAX_RETRIES, + clearCheckpoint, + loadCredential, + readCheckpoint, + redact, + requestJson, + scrub, + writeCheckpoint, +} from "./kit.mjs"; + +export const CHANNEL = "gmail"; +export const CONFIG_FILE = "gmail_config.json"; +export const ENV_KEYS = [ + "DISTILLY_GMAIL_CLIENT_ID", + "GMAIL_CLIENT_ID", + "DISTILLY_GMAIL_CLIENT_SECRET", + "GMAIL_CLIENT_SECRET", + "DISTILLY_GMAIL_REFRESH_TOKEN", + "GMAIL_REFRESH_TOKEN", +]; +export const DEFAULT_BASE_URL = "https://gmail.googleapis.com/gmail/v1"; +export const DEFAULT_TOKEN_URL = "https://oauth2.googleapis.com/token"; +export const DEFAULT_PAGE_SIZE = 100; +export const DEFAULT_MAX_MESSAGES = 200; + +export const ALLOWED_CALLS = [ + { method: "POST", path: "/token" }, + { method: "GET", path: "/gmail/v1/users/" }, +]; + +const REMEDIATION = [ + "create an OAuth client (desktop) in Google Cloud, enable the Gmail API, and add scope gmail.readonly", + "run the consent flow once and keep the refresh token", + `write ~/.distilly/${CONFIG_FILE} as {"client_id": "…", "client_secret": "…", "refresh_token": "…"}`, +]; + +/** Gmail returns base64url; `Buffer` needs the padding restored. */ +export function decodeRawMessage(raw) { + const text = String(raw ?? "").replace(/-/g, "+").replace(/_/g, "/"); + const padded = text + "=".repeat((4 - (text.length % 4)) % 4); + return Buffer.from(padded, "base64"); +} + +/** + * @param {{fetch?: Function, env?: object, root?: string, person: string, family?: string, + * query?: string, limit?: number, maxMessages?: number, maxRetries?: number, + * sleep?: Function, now?: string, baseUrl?: string, tokenUrl?: string}} options + */ +export async function collect(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + query = "", + limit = DEFAULT_PAGE_SIZE, + maxMessages = DEFAULT_MAX_MESSAGES, + maxRetries = DEFAULT_MAX_RETRIES, + sleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_GMAIL_BASE_URL || DEFAULT_BASE_URL, + tokenUrl = env?.DISTILLY_GMAIL_TOKEN_URL || DEFAULT_TOKEN_URL, + } = options; + + const outputs = []; + const warnings = []; + const retries = []; + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + retries, + }; + let secrets = []; + const fail = (failure) => ({ + ok: false, + exitCode: failure.exitCode ?? 1, + receipt: scrub( + { + ...base, + ok: false, + person: person ?? null, + query: query || null, + errors: [redact(failure.message, secrets)], + unavailable: [{ channel: CHANNEL, reason: redact(`${failure.reason}: ${failure.message}`, secrets), remediation: failure.remediation ?? [] }], + }, + secrets, + ), + }); + + try { + if (!person) throw new CollectFailure("missing-person", "collect gmail needs --person ", { remediation: ["pass --person"] }); + const credential = loadCredential({ + env, + configFile: CONFIG_FILE, + envKeys: ENV_KEYS, + fields: ["client_id", "client_secret", "refresh_token"], + }); + secrets = [credential.values.client_secret, credential.values.refresh_token]; + base.credential_source = credential.source; + + const onRetry = (info) => { + retries.push(info); + warnings.push(`retry ${info.attempt} after ${info.reason} (waited ${info.delayMs}ms)`); + }; + + const exchange = async () => { + const response = await requestJson({ + fetchImpl, + url: tokenUrl, + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: credential.values.refresh_token, + client_id: credential.values.client_id, + client_secret: credential.values.client_secret, + }).toString(), + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + }); + const token = response.json?.access_token ?? null; + if (!token) throw new CollectFailure("auth-failed", "Google rejected the refresh token", { remediation: REMEDIATION }); + secrets = [credential.values.client_secret, credential.values.refresh_token, token]; + return token; + }; + + let token = await exchange(); + const headers = () => ({ authorization: `Bearer ${token}`, accept: "application/json" }); + + // A token can expire mid-run; one re-exchange keeps a long collection going. + const get = async (url) => { + try { + return await requestJson({ + fetchImpl, + url, + method: "GET", + headers: headers(), + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + onRetry, + }); + } catch (error) { + if (!(error instanceof CollectFailure) || error.reason !== "unauthorized") throw error; + warnings.push("the access token expired mid-run; the refresh token was exchanged again"); + token = await exchange(); + return await requestJson({ + fetchImpl, + url, + method: "GET", + headers: headers(), + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + onRetry, + }); + } + }; + + const store = new KnowledgeStore(join(resolve(root), "skills", family, person)); + const ledger = loadLedger(store); + const checkpoint = options.resume === false ? null : readCheckpoint({ env, root, channel: CHANNEL, target: query || "inbox" }); + const seen = new Set(checkpoint?.seen ?? []); + let pageToken = checkpoint?.page_token ?? null; + if (seen.size > 0) warnings.push(`resuming: ${seen.size} message(s) already fetched`); + + const pageSize = Math.min(Math.max(1, Number(limit) || DEFAULT_PAGE_SIZE), 500); + let pages = 0; + let fetched = 0; + let anchors = 0; + const textEntries = []; + + for (;;) { + if (fetched >= maxMessages) { + warnings.push(`stopped after --max-messages ${maxMessages}`); + break; + } + const params = new URLSearchParams({ maxResults: String(Math.min(pageSize, maxMessages - fetched)) }); + if (query) params.set("q", query); + if (pageToken) params.set("pageToken", pageToken); + const listing = await get(`${baseUrl}/users/me/messages?${params.toString()}`); + pages += 1; + const ids = (listing.json?.messages ?? []).map((entry) => entry.id).filter(Boolean); + pageToken = listing.json?.nextPageToken ?? null; + + for (const id of ids) { + if (seen.has(id)) continue; + const message = await get(`${baseUrl}/users/me/messages/${encodeURIComponent(id)}?format=raw`); + const bytes = decodeRawMessage(message.json?.raw); + if (bytes.length === 0) { + warnings.push(`message ${id} carried no raw MIME payload and was skipped`); + continue; + } + const file = new SourceFile({ path: `${id}.eml`, name: `${id}.eml`, raw: new Uint8Array(bytes) }); + let document; + try { + document = parseEmail(file, { + source: CHANNEL, + method: "api-oauth-refresh", + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + }); + } catch (error) { + warnings.push(`message ${id}: ${redact(error.message, secrets)}; the raw bytes are stored, the text is not`); + document = null; + } + const recorded = recordDocument( + store, + ledger, + document + ? { ...document, fetched_at: now } + : { + parser: "gmail", + format: "eml", + kind: "email", + method: "api-oauth-refresh", + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + source: CHANNEL, + origin: `gmail:${id}`, + files: [{ path: `${id}.eml`, name: `${id}.eml`, raw: new Uint8Array(bytes) }], + content: "", + segments: [], + entries: [], + warnings: ["the message could not be parsed; only the raw bytes are recorded"], + fetched_at: now, + }, + { fetched_at: now }, + ); + seen.add(id); + fetched += 1; + const raw = recorded.written.files[0]; + if (raw) outputs.push({ path: raw.relativePath, sha256: raw.sha256, bytes: raw.bytes, kind: "raw" }); + if (recorded.written.text) { + outputs.push({ + path: recorded.written.text.relativePath, + sha256: recorded.written.text.sha256, + bytes: recorded.written.text.bytes, + kind: "text", + }); + textEntries.push(recorded.entry?.id ?? null); + anchors += recorded.entry?.anchor_count ?? 0; + } + } + + writeCheckpoint( + { env, root, channel: CHANNEL, target: query || "inbox" }, + { channel: CHANNEL, target: query || "inbox", page_token: pageToken, pages, fetched, seen: [...seen].slice(-500), updated_at: now }, + ); + if (!pageToken || ids.length === 0) break; + } + + saveLedger(store, ledger); + clearCheckpoint({ env, root, channel: CHANNEL, target: query || "inbox" }); + + return { + ok: true, + exitCode: 0, + receipt: scrub( + { + ...base, + ok: true, + person, + query: query || null, + pages, + messages: fetched, + text_entries: textEntries.filter(Boolean), + anchors: { total: anchors, cited: 0 }, + ledger: { path: store.ledgerPath, total: ledger.length }, + unavailable: [], + }, + secrets, + ), + }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + return fail(error); + } +} + +/* ------------------------------------------------------------------ CLI */ + +export function parseGmailArgs(argv) { + const options = { person: null, baseDir: process.cwd(), query: "", limit: DEFAULT_PAGE_SIZE, maxMessages: DEFAULT_MAX_MESSAGES, json: false }; + const takesValue = { "--person": "person", "--base-dir": "baseDir", "--query": "query", "--limit": "limit", "--max-messages": "maxMessages" }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--json") options.json = true; + else if (takesValue[arg]) { + const value = argv[index + 1]; + if (value === undefined) return { error: `${arg} requires a value` }; + options[takesValue[arg]] = /limit|maxMessages/.test(takesValue[arg]) ? Number(value) : value; + index += 1; + } else if (arg.startsWith("--")) return { error: `unknown option: ${arg}` }; + else if (options.query === "") options.query = arg; + else return { error: `unexpected argument: ${arg}` }; + } + if (!options.person) return { error: "--person is required" }; + return { options }; +} + +export async function runCollectCli(argv, io = {}) { + const out = typeof io.stdout === "function" ? io.stdout : (line) => process.stdout.write(`${line}\n`); + const err = typeof io.stderr === "function" ? io.stderr : (line) => process.stderr.write(`${line}\n`); + const parsed = parseGmailArgs(argv); + if (parsed.error) { + return { + ok: false, + exitCode: 2, + receipt: { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + person: null, + warnings: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + unavailable: [{ channel: CHANNEL, reason: parsed.error, remediation: ["distilly collect gmail --help"] }], + error: { code: "collect/usage", message: parsed.error, remedy: "distilly collect gmail --help" }, + }, + }; + } + const { options } = parsed; + const result = await collect({ + root: options.baseDir, + person: options.person, + query: options.query, + limit: options.limit, + maxMessages: options.maxMessages, + ...(io.fetch ? { fetch: io.fetch } : {}), + }); + if (!options.json) { + if (result.ok) { + out(`collect gmail: ${result.receipt.messages} message(s) in ${result.receipt.pages} page(s), ${result.receipt.anchors.total} anchor(s)`); + for (const output of result.receipt.outputs) out(` ${output.kind}: ${output.path}`); + } + for (const warning of result.receipt.warnings ?? []) err(` warning: ${warning}`); + for (const failure of result.receipt.unavailable ?? []) { + err(` unavailable: ${failure.reason}`); + for (const step of failure.remediation ?? []) err(` ${step}`); + } + } + return result; +} + +export { REMEDIATION as GMAIL_REMEDIATION }; diff --git a/src/collect/kit.mjs b/src/collect/kit.mjs new file mode 100644 index 00000000..6797aec1 --- /dev/null +++ b/src/collect/kit.mjs @@ -0,0 +1,315 @@ +/** + * kit.mjs — the plumbing every credentialed collector needs, in one place. + * + * `src/collect/feishu.mjs` grew this by hand first (credential lookup, redaction, + * retry/backoff, a read-only allowlist, checkpointed cursors, verbatim raw storage, + * ledger registration). The channels added later share it from here instead of + * copying it, so one fix — a leak in a log line, a retry that ignores + * `Retry-After` — lands everywhere at once. + * + * Nothing here talks to a specific API: the caller supplies the base URL, the + * allowlist and the parser. What this module guarantees is the discipline in + * `docs/v2/CONTRACT.md`: credentials never reach stdout/stderr/a receipt (only the + * config *file name* does), every request is checked against a read-only + * allowlist before it is sent, and an interrupted run resumes from its cursor. + */ + +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +/** A failure a collector reports as a receipt, never as a stack trace. */ +export class CollectFailure extends Error { + constructor(reason, message, options = {}) { + super(message); + this.name = "CollectFailure"; + this.reason = reason; + this.exitCode = options.exitCode ?? 1; + this.remediation = options.remediation ?? []; + } +} + +export const DEFAULT_MAX_RETRIES = 4; +export const DEFAULT_MAX_BACKOFF_MS = 60_000; +/** 分页上限：给一个有限的默认值，避免一次采集把配额跑光。 */ +export const DEFAULT_MAX_PAGES = 10; +/** 每页条数。Discord / Notion / Reddit 三个 API 的上限都是 100。 */ +export const DEFAULT_PAGE_SIZE = 100; +/** 单次采集的消息上限。 */ +export const DEFAULT_MAX_MESSAGES = 200; + +export const sha256Hex = (bytes) => createHash("sha256").update(Buffer.from(bytes)).digest("hex"); + +export const slug = (text, fallback = "target") => { + const cleaned = String(text ?? "") + .trim() + .replace(/[^\p{L}\p{N}._-]+/gu, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80); + return cleaned === "" ? fallback : cleaned; +}; + +/** Replace every occurrence of a secret with a fixed marker. */ +export function redact(text, secrets = []) { + let output = String(text ?? ""); + for (const secret of secrets) { + if (typeof secret !== "string" || secret.length < 4) continue; + output = output.split(secret).join("***"); + } + return output; +} + +/** Redact recursively, so no receipt field can carry a credential value. */ +export function scrub(value, secrets = []) { + if (typeof value === "string") return redact(value, secrets); + if (Array.isArray(value)) return value.map((item) => scrub(item, secrets)); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, scrub(item, secrets)])); + } + return value; +} + +export function distillyHome(env = process.env) { + return env.DISTILLY_HOME && env.DISTILLY_HOME.trim() !== "" + ? env.DISTILLY_HOME + : join(homedir(), ".distilly"); +} + +/** `~/.distilly/`, with the pre-rename location as a fallback. */ +export function credentialPaths(configFile, env = process.env) { + return { + primary: join(distillyHome(env), configFile), + legacy: join(env.HOME ?? homedir(), ".colleague-skill", configFile), + }; +} + +/** + * Read a channel credential from the environment or the config file. + * + * @param {{configFile: string, envKeys: string[], fields: string[], env?: object, readFile?: Function}} input + * @returns {{values: object, source: string, configFile: string, path: string|null}} + */ +export function loadCredential(input) { + const { configFile, envKeys, fields, env = process.env, readFile = readFileSync } = input; + const pick = (name) => { + const value = env?.[name]; + return typeof value === "string" && value.trim() !== "" ? value.trim() : null; + }; + const fromEnv = {}; + let envComplete = true; + for (const [index, field] of fields.entries()) { + const value = pick(envKeys[index]); + if (value === null) envComplete = false; + else fromEnv[field] = value; + } + if (envComplete) return { values: fromEnv, source: "env", configFile, path: null }; + + const { primary, legacy } = credentialPaths(configFile, env); + for (const [path, source] of [ + [primary, "config"], + [legacy, "legacy-config"], + ]) { + if (!existsSync(path)) continue; + let parsed; + try { + parsed = JSON.parse(readFile(path, "utf8")); + } catch (error) { + throw new CollectFailure("bad-credential-file", `${configFile} is not valid JSON (${redact(error.message)}); rewrite it as {${fields.map((f) => `"${f}": "…"`).join(", ")}}`, { + remediation: [`write ~/.distilly/${configFile}`], + }); + } + const values = {}; + let complete = true; + for (const field of fields) { + const camel = field.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); + const value = parsed[field] ?? parsed[camel] ?? null; + if (typeof value !== "string" || value.trim() === "") complete = false; + else values[field] = value.trim(); + } + if (!complete) { + throw new CollectFailure("incomplete-credential", `${configFile} is missing ${fields.join(" / ")}`, { + remediation: [`write ~/.distilly/${configFile} with ${fields.join(", ")}`], + }); + } + return { values, source, configFile, path }; + } + + throw new CollectFailure("no-credential", `no credential for this channel: set ${envKeys.join(" / ")} or write ~/.distilly/${configFile}`, { + remediation: [`write ~/.distilly/${configFile}`, `or export ${envKeys[0]}`], + }); +} + +/** + * Refuse anything outside the channel's read-only allowlist, before it is sent. + * + * @param {string} url + * @param {string} method + * @param {Array<{method: string, path: string|RegExp}>} allowlist + * @param {string} channel + */ +export function assertReadOnly(url, method, allowlist, channel) { + const verb = String(method ?? "GET").toUpperCase(); + const path = (() => { + try { + return new URL(url).pathname; + } catch { + return String(url); + } + })(); + const allowed = allowlist.some((entry) => { + if (entry.method !== verb) return false; + return entry.path instanceof RegExp ? entry.path.test(path) : path.startsWith(entry.path); + }); + if (!allowed) { + throw new CollectFailure("write-refused", `${channel} collection is read-only: ${verb} ${path} is not on the allowlist`, { + remediation: ["this build never posts, edits or deletes anything"], + }); + } +} + +/** Seconds from a `Retry-After` header (or a body's `retry_after`). */ +export function parseRetryAfter(headerValue, nowMs = Date.now()) { + if (headerValue === undefined || headerValue === null || headerValue === "") return null; + const text = String(headerValue).trim(); + if (/^\d+(\.\d+)?$/.test(text)) return Math.min(Number(text) * 1000, 15 * 60_000); + const at = Date.parse(text); + if (Number.isNaN(at)) return null; + return Math.max(0, Math.min(at - nowMs, 15 * 60_000)); +} + +export function backoffDelay(attempt, retryAfterMs = null, options = {}) { + const maxMs = options.maxMs ?? DEFAULT_MAX_BACKOFF_MS; + if (retryAfterMs !== null) return Math.min(retryAfterMs, maxMs); + const base = options.baseMs ?? 500; + return Math.min(base * 2 ** Math.max(0, attempt - 1), maxMs); +} + +export const defaultSleep = (ms) => new Promise((settle) => setTimeout(settle, ms)); + +/** + * One request with retry, backoff and redaction. + * + * @returns {Promise<{status: number, text: string, json: any, attempts: number}>} + */ +export async function requestJson(options) { + const { + fetchImpl, + url, + method = "GET", + headers = {}, + body, + maxRetries = DEFAULT_MAX_RETRIES, + sleep = defaultSleep, + secrets = [], + allowlist = null, + channel = "collect", + onRetry = () => {}, + acceptStatus = [], + parse = "json", + } = options; + + if (allowlist) assertReadOnly(url, method, allowlist, channel); + + let attempt = 0; + for (;;) { + attempt += 1; + const init = { method, headers, ...(body === undefined ? {} : { body: typeof body === "string" ? body : JSON.stringify(body) }) }; + const response = await fetchImpl(url, init); + const status = response.status; + if (status === 429 || (status >= 500 && status !== 501)) { + if (attempt > maxRetries) { + throw new CollectFailure("retry-exhausted", `${channel} still answering HTTP ${status} after ${maxRetries} retries`, { + remediation: ["retry later; the pages already fetched are on disk"], + }); + } + const retryAfterHeader = response.headers?.get?.("retry-after") ?? null; + const text = await response.text().catch(() => ""); + let bodyRetryAfter = null; + if (text) { + try { + const parsed = JSON.parse(text); + bodyRetryAfter = parsed?.retry_after ?? parsed?.error?.retry_after ?? null; + } catch { + bodyRetryAfter = null; + } + } + const retryAfterMs = + parseRetryAfter(retryAfterHeader) ?? + (typeof bodyRetryAfter === "number" ? Math.min(bodyRetryAfter * (bodyRetryAfter < 1000 ? 1000 : 1), 15 * 60_000) : null); + const delayMs = backoffDelay(attempt, retryAfterMs); + onRetry({ attempt, status, delayMs, reason: `HTTP ${status}` }); + await sleep(delayMs); + continue; + } + const text = await response.text(); + if (status >= 400 && !acceptStatus.includes(status)) { + throw new CollectFailure(status === 401 || status === 403 ? "unauthorized" : "http-error", `${channel} HTTP ${status}: ${redact(text.slice(0, 200), secrets)}`, { + remediation: ["check the credential and its scopes"], + }); + } + let json = null; + if (parse === "json" && text.trim() !== "") { + try { + json = JSON.parse(text); + } catch { + throw new CollectFailure("invalid-json", `${channel} returned a non-JSON body`, { + remediation: ["retry later; if it persists the endpoint may have changed"], + }); + } + } + return { status, text, json, attempts: attempt }; + } +} + +/* ------------------------------------------------------------------ storage */ + +/** `/skills///knowledge` — the Skill's evidence root. */ +export function knowledgeRoot({ root = process.cwd(), person, family = "colleague" } = {}) { + return person + ? join(resolve(root), "skills", slug(family), slug(person), "knowledge") + : join(resolve(root), "knowledge"); +} + +/** Store raw bytes verbatim, creating the bucket only now. */ +export function writeRaw(knowledgeDir, bucket, name, bytes) { + const dir = join(knowledgeDir, "raw", slug(bucket)); + mkdirSync(dir, { recursive: true }); + const path = join(dir, `${slug(name, "page")}.json`); + const staging = `${path}.${process.pid}.tmp`; + const buffer = Buffer.from(bytes); + writeFileSync(staging, buffer); + renameSync(staging, path); + const readBack = readFileSync(path); + if (!readBack.equals(buffer)) throw new CollectFailure("write-verify", `raw bytes changed on disk: ${path}`, {}); + return { path, relativePath: `raw/${slug(bucket)}/${slug(name, "page")}.json`, bytes: buffer.length, sha256: sha256Hex(buffer) }; +} + +/** A JSON cursor file under `$DISTILLY_HOME/state`, so an interrupted run resumes. */ +export function statePath({ env = process.env, root = process.cwd(), channel, target }) { + const key = sha256Hex(Buffer.from(`${resolve(root)}\n${target ?? ""}`, "utf8")).slice(0, 12); + return join(distillyHome(env), "state", `${slug(channel)}-${key}.json`); +} + +export function readCheckpoint({ env = process.env, root = process.cwd(), channel, target }) { + const path = statePath({ env, root, channel, target }); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +export function writeCheckpoint({ env = process.env, root = process.cwd(), channel, target }, value) { + const path = statePath({ env, root, channel, target }); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`, "utf8"); + return path; +} + +export function clearCheckpoint({ env = process.env, root = process.cwd(), channel, target }) { + const path = statePath({ env, root, channel, target }); + if (existsSync(path)) rmSync(path, { force: true }); + return path; +} diff --git a/src/collect/notion.mjs b/src/collect/notion.mjs new file mode 100644 index 00000000..69a74cd4 --- /dev/null +++ b/src/collect/notion.mjs @@ -0,0 +1,418 @@ +/** + * notion.mjs — Notion pages, read with an internal integration token. + * + * Notion's read path is unusual: the API has almost no GETs — searching and + * listing blocks are `POST`s that change nothing. The allowlist therefore names + * those two POSTs explicitly and refuses everything else, which is the same + * guarantee the GET-only channels get: there is no code path here that can create, + * edit, delete or share anything. + * + * A fetched page becomes one ledger entry: the raw API responses verbatim under + * `knowledge/raw/notion/`, and the block text as anchored paragraphs — the same + * shape a harvested document has, so `retrospect` reads it without special cases. + */ + +import { join, resolve } from "node:path"; + +import { KnowledgeStore } from "../knowledge/store.mjs"; +import { loadLedger, recordDocument, saveLedger } from "../knowledge/ledger.mjs"; +import { SourceFile, buildDocument, recordsFromCharSpans } from "../parse/common.mjs"; +import { + CollectFailure, + DEFAULT_MAX_PAGES, + DEFAULT_MAX_RETRIES, + loadCredential, + readCheckpoint, + redact, + requestJson, + scrub, + writeCheckpoint, + clearCheckpoint, + writeRaw, +} from "./kit.mjs"; + +export const CHANNEL = "notion"; +export const CONFIG_FILE = "notion_config.json"; +export const ENV_KEYS = ["DISTILLY_NOTION_TOKEN", "NOTION_TOKEN"]; +export const DEFAULT_BASE_URL = "https://api.notion.com/v1"; +export const NOTION_VERSION = "2022-06-28"; + +/** + * The only calls this module makes. `POST /v1/search` and `POST + * /v1/databases/*\/query` are documented as read operations — they take a filter + * and return rows; nothing is written. + */ +export const ALLOWED_CALLS = [ + { method: "GET", path: "/v1/blocks/" }, + { method: "GET", path: "/v1/pages/" }, + { method: "GET", path: "/v1/users/" }, + { method: "POST", path: "/v1/search" }, + { method: "POST", path: /^\/v1\/databases\/[^/]+\/query$/ }, +]; + +const REMEDIATION = [ + "create an internal integration at https://www.notion.so/my-integrations and copy its token", + "share the page with the integration (⋯ → Connections)", + `write ~/.distilly/${CONFIG_FILE} as {"integration_token": "…"}`, +]; + +/** Block types whose text is dialogue or prose; everything else is named. */ +const TEXT_BLOCKS = new Map([ + ["paragraph", null], + ["heading_1", null], + ["heading_2", null], + ["heading_3", null], + ["bulleted_list_item", null], + ["numbered_list_item", null], + ["quote", null], + ["callout", null], + ["toggle", null], + ["to_do", null], + ["code", null], + ["template", null], +]); + +/** Flatten `rich_text[]` (and a code block's `caption`) into plain text. */ +export function blockText(block) { + const type = block?.type; + if (!type || !TEXT_BLOCKS.has(type)) return null; + const payload = block[type] ?? {}; + const rich = Array.isArray(payload.rich_text) ? payload.rich_text : []; + const text = rich.map((part) => part?.plain_text ?? part?.text?.content ?? "").join(""); + const caption = Array.isArray(payload.caption) ? payload.caption.map((part) => part?.plain_text ?? "").join("") : ""; + return `${text}${caption}`.trim(); +} + +/** How a page's title is rendered: the first rich-text of its title property. */ +export function pageTitle(page) { + const properties = page?.properties ?? {}; + for (const value of Object.values(properties)) { + if (value?.type === "title" && Array.isArray(value.title)) { + const text = value.title.map((part) => part?.plain_text ?? "").join("").trim(); + if (text !== "") return text; + } + } + return page?.id ?? "untitled"; +} + +/** + * Walk a page's children (breadth-first, depth-limited) and collect paragraphs. + * + * @returns {{paragraphs: string[], blocks: number, skipped: object, warnings: string[]}} + */ +export async function readBlocks(options) { + const { fetchImpl, baseUrl, headers, rootBlockId, maxDepth = 2, maxBlocks = 500, maxRetries, sleep, secrets, onRetry, depth = 0 } = options; + const paragraphs = []; + const warnings = []; + const skipped = {}; + let blocks = 0; + let cursor = null; + + do { + const params = new URLSearchParams({ page_size: "100" }); + if (cursor) params.set("start_cursor", cursor); + const url = `${baseUrl}/blocks/${encodeURIComponent(rootBlockId)}/children?${params.toString()}`; + const response = await requestJson({ + fetchImpl, + url, + method: "GET", + headers, + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + onRetry, + }); + for (const block of response.json?.results ?? []) { + blocks += 1; + if (blocks > maxBlocks) break; + const text = blockText(block); + if (text !== null) { + if (text !== "") paragraphs.push(text); + continue; + } + if (block?.type === "child_page" && depth < maxDepth) { + const child = await readBlocks({ ...options, rootBlockId: block.id, depth: depth + 1, maxBlocks: maxBlocks - blocks }); + paragraphs.push(...child.paragraphs); + Object.assign(skipped, child.skipped); + warnings.push(...child.warnings); + blocks += child.blocks; + continue; + } + const type = block?.type ?? "unknown"; + skipped[type] = (skipped[type] ?? 0) + 1; + } + cursor = response.json?.has_more ? response.json?.next_cursor ?? null : null; + } while (cursor); + + if (Object.keys(skipped).length > 0) { + warnings.push( + `blocks without dialogue text were not anchored: ${Object.entries(skipped) + .map(([type, count]) => `${type}×${count}`) + .join(", ")}`, + ); + } + return { paragraphs, blocks, skipped, warnings }; +} + +/** + * @param {{fetch?: Function, env?: object, root?: string, person: string, family?: string, + * pageId: string, maxDepth?: number, maxRetries?: number, sleep?: Function, + * now?: string, baseUrl?: string}} options + */ +export async function collect(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + pageId, + maxDepth = 2, + maxRetries = DEFAULT_MAX_RETRIES, + sleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_NOTION_BASE_URL || DEFAULT_BASE_URL, + } = options; + + const outputs = []; + const warnings = []; + const retries = []; + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + retries, + }; + let secrets = []; + const fail = (failure) => ({ + ok: false, + exitCode: failure.exitCode ?? 1, + receipt: scrub( + { + ...base, + ok: false, + person: person ?? null, + page_id: pageId ?? null, + errors: [redact(failure.message, secrets)], + unavailable: [{ channel: CHANNEL, reason: redact(`${failure.reason}: ${failure.message}`, secrets), remediation: failure.remediation ?? [] }], + }, + secrets, + ), + }); + + try { + if (!person) throw new CollectFailure("missing-person", "collect notion needs --person ", { remediation: ["pass --person"] }); + if (!pageId) { + throw new CollectFailure("missing-target", "collect notion needs --page-id ", { + remediation: ["copy the page id from its URL (32 hex characters before the ?)"], + }); + } + const credential = loadCredential({ env, configFile: CONFIG_FILE, envKeys: ENV_KEYS, fields: ["integration_token"] }); + secrets = [credential.values.integration_token]; + base.credential_source = credential.source; + + const headers = { + authorization: `Bearer ${credential.values.integration_token}`, + "notion-version": NOTION_VERSION, + accept: "application/json", + }; + const id = normalisePageId(pageId); + const onRetry = (info) => { + retries.push(info); + warnings.push(`retry ${info.attempt} after ${info.reason} (waited ${info.delayMs}ms)`); + }; + + const pageResponse = await requestJson({ + fetchImpl, + url: `${baseUrl}/pages/${id}`, + method: "GET", + headers, + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + onRetry, + }); + const title = pageTitle(pageResponse.json); + + const knowledgeDir = join(resolve(root), "skills", family, person, "knowledge"); + const stored = writeRaw(knowledgeDir, CHANNEL, `${id}-page`, pageResponse.text); + outputs.push({ path: stored.relativePath, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }); + + const read = await readBlocks({ + fetchImpl, + baseUrl, + headers, + rootBlockId: id, + maxDepth, + maxRetries, + sleep, + secrets, + onRetry, + }); + warnings.push(...read.warnings); + + const text = [`# ${title}`, "", ...read.paragraphs].join("\n"); + if (read.paragraphs.length === 0) { + throw new CollectFailure("no-text", `page ${id} has no paragraph blocks this client can read`, { + remediation: ["share the page with the integration, or check that it has text blocks"], + }); + } + + const file = new SourceFile({ path: `${id}.md`, name: `${id}.md`, raw: new Uint8Array(Buffer.from(text, "utf8")) }); + const spans = []; + const pattern = /[^\n]+/g; + let match; + while ((match = pattern.exec(text)) !== null) { + spans.push({ text: match[0], charStart: match.index, charEnd: match.index + match[0].length, kind: "paragraph" }); + } + const document = buildDocument({ + parser: "notion", + format: "blocks", + kind: "doc", + method: "api-integration-token", + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + source: CHANNEL, + origin: `https://www.notion.so/${id}`, + files: [file], + records: recordsFromCharSpans(file, spans), + meta: { title, blocks: read.blocks, page_id: id }, + }); + + const store = new KnowledgeStore(join(resolve(root), "skills", family, person)); + const ledger = loadLedger(store); + const recorded = recordDocument(store, ledger, { ...document, fetched_at: now }, { fetched_at: now }); + saveLedger(store, ledger); + if (recorded.written.text) { + outputs.push({ + path: recorded.written.text.relativePath, + sha256: recorded.written.text.sha256, + bytes: recorded.written.text.bytes, + kind: "text", + }); + } + + return { + ok: true, + exitCode: 0, + receipt: scrub( + { + ...base, + ok: true, + person, + page_id: id, + title, + blocks: read.blocks, + paragraphs: read.paragraphs.length, + text_entries: [recorded.entry?.id ?? null].filter(Boolean), + anchors: { total: recorded.entry?.anchor_count ?? 0, cited: 0 }, + ledger: { path: store.ledgerPath, total: ledger.length }, + unavailable: [], + }, + secrets, + ), + }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + return fail(error); + } +} + +/** `https://www.notion.so/Title-<32 hex>?v=…` → `<32 hex>` (dashed ids accepted). */ +export function normalisePageId(input) { + const text = String(input ?? "").trim(); + const dashed = text.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i); + if (dashed) return dashed[0]; + const bare = text.match(/[0-9a-f]{32}/i); + if (bare) { + const value = bare[0]; + return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`; + } + if (/^[0-9a-f-]{36}$/i.test(text)) return text; + throw new CollectFailure("bad-target", `cannot read a Notion page id out of "${text.slice(0, 60)}"`, { + remediation: ["pass the page URL or its 32-character id"], + }); +} + +/* ------------------------------------------------------------------ CLI */ + +export function parseNotionArgs(argv) { + const options = { person: null, baseDir: process.cwd(), pageId: null, maxDepth: 2, json: false }; + const takesValue = { "--person": "person", "--base-dir": "baseDir", "--page-id": "pageId", "--url": "pageId", "--max-depth": "maxDepth" }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--json") options.json = true; + else if (takesValue[arg]) { + const value = argv[index + 1]; + if (value === undefined) return { error: `${arg} requires a value` }; + options[takesValue[arg]] = takesValue[arg] === "maxDepth" ? Number(value) : value; + index += 1; + } else if (arg.startsWith("--")) return { error: `unknown option: ${arg}` }; + else if (!options.pageId) options.pageId = arg; + else return { error: `unexpected argument: ${arg}` }; + } + if (!options.person) return { error: "--person is required" }; + if (!options.pageId) return { error: "collect notion needs --page-id " }; + return { options }; +} + +export async function runCollectCli(argv, io = {}) { + const out = typeof io.stdout === "function" ? io.stdout : (line) => process.stdout.write(`${line}\n`); + const err = typeof io.stderr === "function" ? io.stderr : (line) => process.stderr.write(`${line}\n`); + const parsed = parseNotionArgs(argv); + if (parsed.error) { + return { + ok: false, + exitCode: 2, + receipt: { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + person: null, + warnings: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + unavailable: [{ channel: CHANNEL, reason: parsed.error, remediation: ["distilly collect notion --help"] }], + error: { code: "collect/usage", message: parsed.error, remedy: "distilly collect notion --help" }, + }, + }; + } + const { options } = parsed; + const result = await collect({ + root: options.baseDir, + person: options.person, + pageId: options.pageId, + maxDepth: options.maxDepth, + ...(io.fetch ? { fetch: io.fetch } : {}), + }); + if (!options.json) { + if (result.ok) { + out(`collect notion: "${result.receipt.title}" — ${result.receipt.paragraphs} paragraph(s), ${result.receipt.anchors.total} anchor(s)`); + for (const output of result.receipt.outputs) out(` ${output.kind}: ${output.path}`); + } + for (const warning of result.receipt.warnings ?? []) err(` warning: ${warning}`); + for (const failure of result.receipt.unavailable ?? []) { + err(` unavailable: ${failure.reason}`); + for (const step of failure.remediation ?? []) err(` ${step}`); + } + } + return result; +} + +export { REMEDIATION as NOTION_REMEDIATION }; diff --git a/src/collect/reddit.mjs b/src/collect/reddit.mjs new file mode 100644 index 00000000..7d37a240 --- /dev/null +++ b/src/collect/reddit.mjs @@ -0,0 +1,405 @@ +/** + * reddit.mjs — Reddit comments, read with an OAuth client credential. + * + * The only mutation this module performs is the OAuth token exchange — a POST that + * changes nothing a user can see, exactly like Feishu's tenant-token call. Every + * other request is a GET against a small allowlist, so there is no path here that + * can post, vote, edit or delete. + * + * A listing page is stored verbatim under `knowledge/raw/reddit/` and the comments + * it contains become anchored paragraphs (author + UTC time + body), which is the + * shape the derivation reads. `[deleted]` and `[removed]` bodies are skipped by + * name, never anchored as if someone had said them. + */ + +import { join, resolve } from "node:path"; + +import { KnowledgeStore } from "../knowledge/store.mjs"; +import { loadLedger, recordDocument, saveLedger } from "../knowledge/ledger.mjs"; +import { SourceFile, buildDocument, recordsFromCharSpans } from "../parse/common.mjs"; +import { + CollectFailure, + DEFAULT_MAX_PAGES, + DEFAULT_MAX_RETRIES, + clearCheckpoint, + loadCredential, + readCheckpoint, + redact, + requestJson, + scrub, + writeCheckpoint, + writeRaw, +} from "./kit.mjs"; + +export const CHANNEL = "reddit"; +export const CONFIG_FILE = "reddit_config.json"; +export const ENV_KEYS = ["DISTILLY_REDDIT_CLIENT_ID", "REDDIT_CLIENT_ID", "DISTILLY_REDDIT_CLIENT_SECRET", "REDDIT_CLIENT_SECRET"]; +export const DEFAULT_BASE_URL = "https://oauth.reddit.com"; +export const DEFAULT_TOKEN_URL = "https://www.reddit.com/api/v1/access_token"; +export const DEFAULT_PAGE_SIZE = 100; +export const USER_AGENT = "distilly/1.0 (read-only collector)"; + +/** The token exchange, plus GETs. Reddit's read API never needs another verb. */ +export const ALLOWED_CALLS = [ + { method: "POST", path: "/api/v1/access_token" }, + { method: "GET", path: "/api/v1/me" }, + { method: "GET", path: "/r/" }, + { method: "GET", path: "/user/" }, + { method: "GET", path: "/comments/" }, +]; + +const REMEDIATION = [ + "create a script app at https://www.reddit.com/prefs/apps (type: script) and copy the id and secret", + `write ~/.distilly/${CONFIG_FILE} as {"client_id": "…", "client_secret": "…"}`, +]; + +/** `[deleted]` / `[removed]` are placeholders, not something a person said. */ +export const isPlaceholder = (body) => { + const text = String(body ?? "").trim(); + return text === "" || text === "[deleted]" || text === "[removed]"; +}; + +/** A listing's comment children, flattened, with placeholders removed. */ +export function commentsFromListing(json) { + const children = json?.data?.children ?? []; + const comments = []; + const skipped = {}; + const walk = (nodes, depth = 0) => { + for (const node of nodes) { + if (node?.kind === "more") { + skipped.more = (skipped.more ?? 0) + 1; + continue; + } + const data = node?.data ?? node; + if (!data || typeof data !== "object") continue; + if (typeof data.body === "string") { + if (isPlaceholder(data.body)) skipped.placeholder = (skipped.placeholder ?? 0) + 1; + else { + comments.push({ + id: data.id ?? null, + author: data.author ?? "unknown", + body: data.body, + created_utc: typeof data.created_utc === "number" ? data.created_utc : null, + permalink: data.permalink ?? null, + depth, + score: typeof data.score === "number" ? data.score : null, + }); + } + } + const replies = data.replies?.data?.children; + if (Array.isArray(replies)) walk(replies, depth + 1); + } + }; + walk(children); + return { comments, skipped, after: json?.data?.after ?? null }; +} + +/** One anchor per comment: ` ：`. */ +export function redditDocument({ comments, file, options }) { + const text = comments.map((comment) => { + const at = comment.created_utc === null ? "" : new Date(comment.created_utc * 1000).toISOString(); + return `${at} ${comment.author}：${comment.body}`; + }).join("\n"); + const spans = []; + let offset = 0; + for (const [index, comment] of comments.entries()) { + const line = text.split("\n")[index]; + spans.push({ text: line, charStart: offset, charEnd: offset + line.length, kind: "comment", label: comment.id ?? undefined }); + offset += line.length + 1; + } + void file; + return { content: text, spans, options }; +} + +/** + * @param {{fetch?: Function, env?: object, root?: string, person: string, family?: string, + * target: string, kind?: "subreddit"|"user", limit?: number, maxPages?: number, + * maxRetries?: number, sleep?: Function, now?: string, baseUrl?: string, tokenUrl?: string}} options + */ +export async function collect(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + target, + limit = DEFAULT_PAGE_SIZE, + maxPages = DEFAULT_MAX_PAGES, + maxRetries = DEFAULT_MAX_RETRIES, + sleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_REDDIT_BASE_URL || DEFAULT_BASE_URL, + tokenUrl = env?.DISTILLY_REDDIT_TOKEN_URL || DEFAULT_TOKEN_URL, + } = options; + + const outputs = []; + const warnings = []; + const retries = []; + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + retries, + }; + let secrets = []; + const fail = (failure) => ({ + ok: false, + exitCode: failure.exitCode ?? 1, + receipt: scrub( + { + ...base, + ok: false, + person: person ?? null, + target: target ?? null, + errors: [redact(failure.message, secrets)], + unavailable: [{ channel: CHANNEL, reason: redact(`${failure.reason}: ${failure.message}`, secrets), remediation: failure.remediation ?? [] }], + }, + secrets, + ), + }); + + try { + if (!person) throw new CollectFailure("missing-person", "collect reddit needs --person ", { remediation: ["pass --person"] }); + if (!target) { + throw new CollectFailure("missing-target", "collect reddit needs --target ", { + remediation: ["pass --target programming, or --target someuser with --kind user"], + }); + } + const kind = options.kind ?? "subreddit"; + if (!["subreddit", "user"].includes(kind)) { + throw new CollectFailure("bad-kind", `--kind must be subreddit or user (got ${kind})`, { remediation: ["pass --kind subreddit|user"] }); + } + + const credential = loadCredential({ + env, + configFile: CONFIG_FILE, + envKeys: ENV_KEYS, + fields: ["client_id", "client_secret"], + }); + secrets = [credential.values.client_secret]; + base.credential_source = credential.source; + + const basic = Buffer.from(`${credential.values.client_id}:${credential.values.client_secret}`).toString("base64"); + const tokenResponse = await requestJson({ + fetchImpl, + url: tokenUrl, + method: "POST", + headers: { authorization: `Basic ${basic}`, "content-type": "application/x-www-form-urlencoded", "user-agent": USER_AGENT }, + body: "grant_type=client_credentials", + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + }); + const token = tokenResponse.json?.access_token ?? null; + if (!token) { + throw new CollectFailure("auth-failed", "Reddit rejected the client credential", { remediation: REMEDIATION }); + } + secrets = [...secrets, token]; + + const knowledgeDir = join(resolve(root), "skills", family, person, "knowledge"); + const store = new KnowledgeStore(join(resolve(root), "skills", family, person)); + const ledger = loadLedger(store); + const checkpoint = options.resume === false ? null : readCheckpoint({ env, root, channel: CHANNEL, target: `${kind}:${target}` }); + let cursor = checkpoint?.cursor ?? null; + if (cursor) warnings.push(`resuming from checkpoint (${checkpoint.pages ?? 0} page(s) done)`); + + const pageSize = Math.min(Math.max(1, Number(limit) || DEFAULT_PAGE_SIZE), 100); + const headers = { authorization: `Bearer ${token}`, "user-agent": USER_AGENT, accept: "application/json" }; + const path = kind === "subreddit" ? `/r/${encodeURIComponent(target)}/comments` : `/user/${encodeURIComponent(target)}/comments`; + + let pages = 0; + let items = 0; + let comments = 0; + let anchors = 0; + const textEntries = []; + + for (;;) { + if (pages >= maxPages) { + warnings.push(`stopped after --max-pages ${maxPages}; rerun to continue from the cursor`); + break; + } + const params = new URLSearchParams({ limit: String(pageSize), raw_json: "1" }); + if (cursor) params.set("after", cursor); + const response = await requestJson({ + fetchImpl, + url: `${baseUrl}${path}?${params.toString()}`, + method: "GET", + headers, + maxRetries, + ...(sleep ? { sleep } : {}), + secrets, + allowlist: ALLOWED_CALLS, + channel: CHANNEL, + onRetry: (info) => { + retries.push(info); + warnings.push(`retry ${info.attempt} after ${info.reason} (waited ${info.delayMs}ms)`); + }, + }); + pages += 1; + const listing = commentsFromListing(response.json); + items += response.json?.data?.children?.length ?? 0; + comments += listing.comments.length; + for (const [type, count] of Object.entries(listing.skipped)) warnings.push(`${count} ${type} entr(ies) were skipped`); + + const name = `${kind}-${target}-p${String(pages).padStart(3, "0")}`; + const stored = writeRaw(knowledgeDir, CHANNEL, name, response.text); + outputs.push({ path: stored.relativePath, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }); + + if (listing.comments.length > 0) { + const text = listing.comments + .map((comment) => `${comment.created_utc === null ? "" : new Date(comment.created_utc * 1000).toISOString()} ${comment.author}：${comment.body}`) + .join("\n"); + const file = new SourceFile({ path: `${name}.txt`, name: `${name}.txt`, raw: new Uint8Array(Buffer.from(text, "utf8")) }); + const spans = []; + let offset = 0; + for (const line of text.split("\n")) { + spans.push({ text: line, charStart: offset, charEnd: offset + line.length, kind: "comment" }); + offset += line.length + 1; + } + const document = buildDocument({ + parser: "reddit", + format: "listing", + kind: "chat", + method: "api-oauth-client", + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + source: CHANNEL, + origin: `https://www.reddit.com${path}`, + files: [file], + records: recordsFromCharSpans(file, spans), + meta: { target, kind, comments: listing.comments.length }, + }); + const recorded = recordDocument(store, ledger, { ...document, fetched_at: now }, { fetched_at: now }); + if (recorded.written.text) { + outputs.push({ + path: recorded.written.text.relativePath, + sha256: recorded.written.text.sha256, + bytes: recorded.written.text.bytes, + kind: "text", + }); + textEntries.push(recorded.entry?.id ?? null); + anchors += recorded.entry?.anchor_count ?? 0; + } + } + + cursor = listing.after; + if (!cursor) break; + if (options.resume !== false) { + writeCheckpoint({ env, root, channel: CHANNEL, target: `${kind}:${target}` }, { channel: CHANNEL, target, cursor, pages, items, updated_at: now }); + } + } + + saveLedger(store, ledger); + clearCheckpoint({ env, root, channel: CHANNEL, target: `${kind}:${target}` }); + + return { + ok: true, + exitCode: 0, + receipt: scrub( + { + ...base, + ok: true, + person, + target, + kind, + pages, + items, + comments, + cursor, + text_entries: textEntries.filter(Boolean), + anchors: { total: anchors, cited: 0 }, + ledger: { path: store.ledgerPath, total: ledger.length }, + unavailable: [], + }, + secrets, + ), + }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + return fail(error); + } +} + +/* ------------------------------------------------------------------ CLI */ + +export function parseRedditArgs(argv) { + const options = { person: null, baseDir: process.cwd(), target: null, kind: "subreddit", limit: DEFAULT_PAGE_SIZE, maxPages: DEFAULT_MAX_PAGES, json: false }; + const takesValue = { "--person": "person", "--base-dir": "baseDir", "--target": "target", "--kind": "kind", "--limit": "limit", "--max-pages": "maxPages" }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--json") options.json = true; + else if (takesValue[arg]) { + const value = argv[index + 1]; + if (value === undefined) return { error: `${arg} requires a value` }; + options[takesValue[arg]] = /limit|maxPages/.test(takesValue[arg]) ? Number(value) : value; + index += 1; + } else if (arg.startsWith("--")) return { error: `unknown option: ${arg}` }; + else if (!options.target) options.target = arg; + else return { error: `unexpected argument: ${arg}` }; + } + if (!options.person) return { error: "--person is required" }; + if (!options.target) return { error: "collect reddit needs --target " }; + return { options }; +} + +export async function runCollectCli(argv, io = {}) { + const out = typeof io.stdout === "function" ? io.stdout : (line) => process.stdout.write(`${line}\n`); + const err = typeof io.stderr === "function" ? io.stderr : (line) => process.stderr.write(`${line}\n`); + const parsed = parseRedditArgs(argv); + if (parsed.error) { + return { + ok: false, + exitCode: 2, + receipt: { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + person: null, + warnings: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + unavailable: [{ channel: CHANNEL, reason: parsed.error, remediation: ["distilly collect reddit --help"] }], + error: { code: "collect/usage", message: parsed.error, remedy: "distilly collect reddit --help" }, + }, + }; + } + const { options } = parsed; + const result = await collect({ + root: options.baseDir, + person: options.person, + target: options.target, + kind: options.kind, + limit: options.limit, + maxPages: options.maxPages, + ...(io.fetch ? { fetch: io.fetch } : {}), + }); + if (!options.json) { + if (result.ok) { + out(`collect reddit: ${result.receipt.comments} comment(s) in ${result.receipt.pages} page(s), ${result.receipt.anchors.total} anchor(s)`); + for (const output of result.receipt.outputs) out(` ${output.kind}: ${output.path}`); + } + for (const warning of result.receipt.warnings ?? []) err(` warning: ${warning}`); + for (const failure of result.receipt.unavailable ?? []) { + err(` unavailable: ${failure.reason}`); + for (const step of failure.remediation ?? []) err(` ${step}`); + } + } + return result; +} + +export { REMEDIATION as REDDIT_REMEDIATION }; diff --git a/src/collect/slack.mjs b/src/collect/slack.mjs new file mode 100644 index 00000000..9dd35cb8 --- /dev/null +++ b/src/collect/slack.mjs @@ -0,0 +1,719 @@ +/** + * slack.mjs — credentialed collection from the Slack Web API. + * + * Legacy ported: `tools/slack_auto_collector.py` (722 lines, `slack_sdk`). The + * credential file, the retry policy and the two error classes (missing scope, + * invalid token) keep their meaning; the SDK is replaced by an injected `fetch`, + * so the channel is testable without a workspace and without a network. + * + * Slack differences from Feishu worth knowing: + * - the token is a bot token (`xoxb-…`) used directly — there is no exchange + * step, so this module makes **no non-GET request at all**; + * - rate limiting arrives twice: as HTTP 429 with `Retry-After`, and as HTTP 200 + * with `{"ok": false, "error": "ratelimited"}`. Both back off; + * - pagination is `cursor` → `response_metadata.next_cursor`, and the empty + * string means "no more pages". + * + * Everything else follows the discipline described in `feishu.mjs`: raw bytes + * verbatim into `knowledge/raw/slack/`, ledger upsert, resume checkpoints, and + * no credential value in stdout / stderr / receipts. + */ + +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +export const CHANNEL = "slack"; +export const CONFIG_FILE = "slack_config.json"; +export const LEGACY_CONFIG_FILE = join(".colleague-skill", CONFIG_FILE); +export const DEFAULT_BASE_URL = "https://slack.com/api"; +export const DEFAULT_PAGE_SIZE = 200; +export const DEFAULT_MAX_PAGES = 10; +export const DEFAULT_MAX_RETRIES = 4; +export const DEFAULT_MAX_BACKOFF_MS = 60_000; + +export const ENV_KEYS = { + botToken: ["DISTILLY_SLACK_BOT_TOKEN", "SLACK_BOT_TOKEN"], +}; + +/** Slack is read-only here: no method may be anything but GET. */ +export const ALLOWED_MUTATIONS = []; + +const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); +const REMEDIATION_SETUP = [ + `create ~/.distilly/${CONFIG_FILE} (chmod 600) with {"bot_token": "xoxb-…"}`, + " app: https://api.slack.com/apps → OAuth & Permissions → Bot Token Scopes:", + " channels:history, groups:history, mpim:history, im:history, channels:read, groups:read, users:read", + "or export DISTILLY_SLACK_BOT_TOKEN for this shell only", + "then invite the bot to every channel you want collected: /invite @your-bot", +]; + +export class CollectFailure extends Error { + constructor(reason, message, { remediation = [], exitCode = 1, kind = "failure" } = {}) { + super(message); + this.name = "CollectFailure"; + this.reason = reason; + this.remediation = remediation; + this.exitCode = exitCode; + this.kind = kind; + } +} + +export function redact(text, secrets = []) { + let output = String(text ?? ""); + for (const secret of secrets) { + if (typeof secret !== "string" || secret.length < 4) continue; + output = output.split(secret).join("[redacted]"); + } + return output; +} + +export function scrub(value, secrets = []) { + return JSON.parse( + JSON.stringify(value, (_key, item) => (typeof item === "string" ? redact(item, secrets) : item)), + ); +} + +export function distillyHome(env = process.env) { + const override = env?.DISTILLY_HOME; + return override ? resolve(String(override)) : join(homedir(), ".distilly"); +} + +export function credentialPaths(env = process.env) { + // `env.HOME ?? homedir()`, matching `kit.mjs`: the pre-rename + // `~/.colleague-skill/` fallback has to move when a caller points `HOME` + // somewhere else. Reading the real home here ignored `env.HOME` — so an isolated + // run (a test, a sandboxed collect) could still pick up the credential sitting in + // the developer's own home, and the two channels disagreed about which file they + // had just read. + return { + primary: join(distillyHome(env), CONFIG_FILE), + legacy: join(env?.HOME ?? homedir(), LEGACY_CONFIG_FILE), + }; +} + +export function loadCredential({ env = process.env, readFile = readFileSync } = {}) { + for (const name of ENV_KEYS.botToken) { + const value = env?.[name]; + if (typeof value === "string" && value.trim() !== "") { + return { + ok: true, + source: "env", + configFile: CONFIG_FILE, + path: null, + values: { bot_token: value.trim() }, + }; + } + } + + const { primary, legacy } = credentialPaths(env); + for (const [path, source] of [ + [primary, "config"], + [legacy, "legacy-config"], + ]) { + if (!existsSync(path)) continue; + let parsed; + try { + parsed = JSON.parse(readFile(path, "utf8")); + } catch (error) { + throw new CollectFailure( + "bad-credential-file", + `${CONFIG_FILE} is not valid JSON (${redact(error.message)}); rewrite it with {"bot_token": "xoxb-…"}`, + { remediation: REMEDIATION_SETUP }, + ); + } + const token = parsed.bot_token ?? parsed.botToken ?? parsed.token ?? null; + if (!token) { + throw new CollectFailure("incomplete-credential", `${CONFIG_FILE} is missing bot_token`, { + remediation: REMEDIATION_SETUP, + }); + } + return { ok: true, source, configFile: CONFIG_FILE, path, values: { bot_token: token } }; + } + + throw new CollectFailure("no-credential", `no credential at ~/.distilly/${CONFIG_FILE}`, { + remediation: REMEDIATION_SETUP, + }); +} + +export function assertReadOnly(url, method = "GET") { + const verb = String(method).toUpperCase(); + if (!MUTATING_METHODS.has(verb)) return true; + const path = (() => { + try { + return new URL(url).pathname; + } catch { + return String(url); + } + })(); + const allowed = ALLOWED_MUTATIONS.some((entry) => entry.method === verb && path.endsWith(entry.path)); + if (!allowed) { + throw new CollectFailure( + "write-operation-refused", + `refusing ${verb} ${path}: this collector never writes to ${CHANNEL}`, + { remediation: ["collectors are read-only; remove the mutating call instead of allowlisting it"] }, + ); + } + return true; +} + +export function parseRetryAfter(headerValue, nowMs = Date.now()) { + if (headerValue === null || headerValue === undefined || headerValue === "") return null; + const seconds = Number(headerValue); + if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000); + const date = Date.parse(String(headerValue)); + if (Number.isFinite(date)) return Math.max(0, date - nowMs); + return null; +} + +export function backoffDelay(attempt, retryAfterMs = null, options = {}) { + const { baseMs = 500, maxMs = DEFAULT_MAX_BACKOFF_MS } = options; + if (Number.isFinite(retryAfterMs) && retryAfterMs !== null) return Math.min(retryAfterMs, maxMs); + return Math.min(baseMs * 2 ** Math.max(0, attempt - 1), maxMs); +} + +export function defaultSleep(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +export async function requestJson(options) { + const { + fetchImpl, + url, + method = "GET", + headers = {}, + body, + maxRetries = DEFAULT_MAX_RETRIES, + sleep = defaultSleep, + secrets = [], + onRetry = () => {}, + authRemediation = REMEDIATION_SETUP, + } = options; + + assertReadOnly(url, method); + if (typeof fetchImpl !== "function") { + throw new CollectFailure("no-fetch", "no fetch implementation available", { + remediation: ["run on Node >= 20, or pass an injected fetch"], + }); + } + + let attempt = 0; + for (;;) { + attempt += 1; + let response; + try { + response = await fetchImpl(url, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); + } catch (error) { + const message = redact(error?.message ?? String(error), secrets); + if (attempt > maxRetries) { + throw new CollectFailure("network-error", `request failed: ${message}`, { + remediation: ["check the network/proxy and retry", ...authRemediation], + }); + } + await sleep(backoffDelay(attempt)); + onRetry({ attempt, status: null, delayMs: backoffDelay(attempt), reason: message }); + continue; + } + + const status = Number(response?.status ?? 0); + const headerBag = response?.headers; + const retryAfterMs = parseRetryAfter(headerBag?.get?.("retry-after") ?? null); + + if (status === 429 || status >= 500) { + if (attempt > maxRetries) { + throw new CollectFailure( + status === 429 ? "rate-limited" : "server-error", + status === 429 + ? `rate limited (HTTP 429) after ${maxRetries} retries` + : `server error (HTTP ${status}) after ${maxRetries} retries`, + { + remediation: [ + "retry later; already-fetched pages stay on disk and the cursor is checkpointed", + `lower --limit / --max-pages to stay under the ${CHANNEL} quota`, + ], + }, + ); + } + const delayMs = backoffDelay(attempt, retryAfterMs); + onRetry({ attempt, status, delayMs, reason: `HTTP ${status}` }); + await sleep(delayMs); + continue; + } + + const text = await response.text(); + if (status === 401 || status === 403) { + throw new CollectFailure("unauthorized", `HTTP ${status} from ${CHANNEL}; bot token rejected`, { + remediation: [ + "the credential in ~/.distilly/" + CONFIG_FILE + " was rejected — reinstall the app and copy a fresh bot token", + "check the bot token scopes at https://api.slack.com/apps → OAuth & Permissions", + ], + }); + } + if (status >= 400) { + throw new CollectFailure("http-error", `HTTP ${status}: ${redact(text.slice(0, 200), secrets)}`, { + remediation: ["check the request parameters", ...authRemediation], + }); + } + + let json = null; + try { + json = JSON.parse(text); + } catch { + throw new CollectFailure("invalid-json", `${CHANNEL} returned a non-JSON body`, { + remediation: ["retry later; if it persists the endpoint may have changed"], + }); + } + return { status, text, json, attempts: attempt, headers: headerBag }; + } +} + +/** Map a Slack `{ok:false, error}` body onto a loud failure with real steps. */ +export function slackError(error, secrets = []) { + const code = redact(String(error ?? "unknown_error"), secrets); + const byCode = { + invalid_auth: { + remediation: [ + "the bot token is invalid or revoked — copy a fresh one from https://api.slack.com/apps → OAuth & Permissions", + `then rewrite ~/.distilly/${CONFIG_FILE} with {"bot_token": "xoxb-…"}`, + ], + }, + token_revoked: { remediation: ["the token was revoked — reinstall the app and update the config file"] }, + account_inactive: { remediation: ["the Slack account is inactive; ask a workspace admin"] }, + missing_scope: { + remediation: [ + "add the missing scope: https://api.slack.com/apps → OAuth & Permissions → Bot Token Scopes", + "needed: channels:history, groups:history, mpim:history, channels:read, groups:read, users:read", + "then reinstall the app so the token carries the new scope", + ], + }, + not_in_channel: { + remediation: ["invite the bot to the channel: /invite @your-bot, then rerun"], + }, + channel_not_found: { + remediation: ["check --channel ; list channels with conversations.list (channels:read)"], + }, + }; + const entry = byCode[code] ?? { remediation: ["unexpected Slack error; retry or check the app configuration"] }; + return new CollectFailure("api-error", `Slack error: ${code}`, { remediation: entry.remediation }); +} + +export function sha256Hex(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function slug(text, fallback = "target") { + const slugged = String(text ?? "") + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^[-._]+|[-._]+$/g, "") + .slice(0, 64); + return slugged || fallback; +} + +export function knowledgeRoot({ root = process.cwd(), person, family = "colleague" } = {}) { + return person ? join(resolve(root), "skills", slug(family), slug(person), "knowledge") : join(resolve(root), "knowledge"); +} + +export function writeRaw(knowledgeDir, name, bytes) { + const dir = join(knowledgeDir, "raw", CHANNEL); + mkdirSync(dir, { recursive: true }); + const path = join(dir, `${slug(name, "page")}.json`); + const staging = `${path}.${process.pid}.tmp`; + const buffer = Buffer.from(bytes); + try { + writeFileSync(staging, buffer); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { + path, + relativePath: `raw/${CHANNEL}/${slug(name, "page")}.json`, + bytes: buffer.length, + sha256: sha256Hex(buffer), + }; +} + +export function appendLedger(knowledgeDir, entries) { + if (entries.length === 0) return { path: join(knowledgeDir, "index.json"), added: 0, total: 0, existed: false }; + const path = join(knowledgeDir, "index.json"); + let existing = []; + if (existsSync(path)) { + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + existing = Array.isArray(parsed) ? parsed : []; + } catch (error) { + throw new CollectFailure("bad-ledger", `knowledge/index.json is not valid JSON: ${redact(error.message)}`, { + remediation: ["repair or remove knowledge/index.json, then rerun the collect"], + }); + } + } + const byId = new Map(existing.filter((e) => e && typeof e === "object").map((e) => [e.id, e])); + let added = 0; + for (const entry of entries) { + if (!byId.has(entry.id)) added += 1; + byId.set(entry.id, entry); + } + const merged = [...byId.values()].sort((a, b) => String(a.id).localeCompare(String(b.id))); + const staging = `${path}.${process.pid}.tmp`; + try { + writeFileSync(staging, `${JSON.stringify(merged, null, 2)}\n`); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { path, added, total: merged.length, existed: existing.length > 0 }; +} + +export function statePath({ env = process.env, root = process.cwd(), target } = {}) { + const key = sha256Hex(Buffer.from(`${resolve(root)}\n${target ?? ""}`, "utf8")).slice(0, 12); + return join(distillyHome(env), "state", `${CHANNEL}-${key}.json`); +} + +export function readCheckpoint(options) { + const path = statePath(options); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +export function writeCheckpoint(options, value) { + const path = statePath(options); + mkdirSync(dirname(path), { recursive: true }); + const staging = `${path}.${process.pid}.tmp`; + writeFileSync(staging, `${JSON.stringify(value, null, 2)}\n`); + renameSync(staging, path); + return path; +} + +export function clearCheckpoint(options) { + const path = statePath(options); + if (existsSync(path)) rmSync(path, { force: true }); + return path; +} + +/** + * Collect history from one Slack channel. + * + * @param {object} options `fetch`, `env`, `root`, `person`, `channel` (required), + * `limit`, `maxPages`, `maxRetries`, `since` (cursor), `resume`, `sleep`, `now` + */ +export async function collect(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + channel, + limit = DEFAULT_PAGE_SIZE, + maxPages = DEFAULT_MAX_PAGES, + maxRetries = DEFAULT_MAX_RETRIES, + since, + resume = true, + sleep = defaultSleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_SLACK_BASE_URL || DEFAULT_BASE_URL, + onProgress = () => {}, + } = options; + + const knowledgeDir = knowledgeRoot({ root, person, family }); + const outputs = []; + const warnings = []; + const ledgerEntries = []; + const retries = []; + const onRetry = (info) => { + retries.push(info); + warnings.push(`retry ${info.attempt} after ${info.reason} (waited ${info.delayMs}ms)`); + onProgress(`retry ${info.attempt}: ${info.reason}`); + }; + + let secrets = []; + let credential = null; + let pages = 0; + let items = 0; + let requests = 0; + let cursor = since ?? null; + let checkpoint = null; + + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + retries, + }; + const fail = (failure) => { + const receipt = { + ...base, + ok: false, + person: person ?? null, + channel_id: channel ?? null, + pages, + items, + requests, + cursor, + resumed_from: checkpoint?.cursor ?? null, + errors: [redact(failure.message, secrets)], + unavailable: [ + { + channel: CHANNEL, + reason: redact(`${failure.reason}: ${failure.message}`, secrets), + remediation: failure.remediation ?? [], + }, + ], + }; + return { ok: false, exitCode: failure.exitCode ?? 1, receipt: scrub(receipt, secrets) }; + }; + + try { + if (!channel) { + throw new CollectFailure("missing-target", "collect slack needs --channel ", { + remediation: [ + "channel ids start with C (public), G (private) or D (dm); find them in the Slack URL", + ], + }); + } + + credential = loadCredential({ env }); + secrets = [credential.values.bot_token]; + base.credential_source = credential.source; + + if (resume) checkpoint = readCheckpoint({ env, root, target: channel }); + if (since === undefined && checkpoint?.cursor) { + cursor = checkpoint.cursor; + pages = Number(checkpoint.pages ?? 0); + warnings.push(`resuming from checkpoint cursor (page ${pages} done)`); + } + + let hasMore = true; + const pageSize = Math.min(Math.max(1, Number(limit) || DEFAULT_PAGE_SIZE), 200); + while (hasMore) { + if (pages >= maxPages) { + warnings.push(`stopped after --max-pages ${maxPages}; rerun to continue from the cursor`); + break; + } + + const url = new URL(`${baseUrl}/conversations.history`); + url.searchParams.set("channel", channel); + url.searchParams.set("limit", String(pageSize)); + if (cursor) url.searchParams.set("cursor", cursor); + + // Slack can rate limit with HTTP 200 + {"ok": false, "error": "ratelimited"}. + let response; + for (let pageAttempt = 1; ; pageAttempt += 1) { + response = await requestJson({ + fetchImpl, + url: url.toString(), + headers: { authorization: `Bearer ${credential.values.bot_token}` }, + maxRetries, + sleep, + secrets, + onRetry, + }); + requests += 1; + if (response.json?.ok === false && response.json?.error === "ratelimited" && pageAttempt <= maxRetries) { + const delayMs = backoffDelay( + pageAttempt, + parseRetryAfter(response.headers?.get?.("retry-after") ?? null), + ); + warnings.push(`retry ${pageAttempt} after slack ratelimited (waited ${delayMs}ms)`); + retries.push({ attempt: pageAttempt, status: 200, delayMs, reason: "ratelimited" }); + await sleep(delayMs); + continue; + } + break; + } + + if (response.json?.ok !== true) throw slackError(response.json?.error, secrets); + + pages += 1; + const pageItems = Array.isArray(response.json?.messages) ? response.json.messages : []; + items += pageItems.length; + + const stored = writeRaw(knowledgeDir, `${channel}-p${String(pages).padStart(3, "0")}`, response.text); + outputs.push({ path: stored.path, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }); + ledgerEntries.push({ + id: `${CHANNEL}:${slug(channel)}:p${String(pages).padStart(3, "0")}`, + kind: "raw", + origin: stored.relativePath, + source: CHANNEL, + fetched_at: now, + bytes: stored.bytes, + sha256: stored.sha256, + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + method: "api-bot-token", + items: pageItems.length, + warnings: [], + }); + + const next = response.json?.response_metadata?.next_cursor; + cursor = typeof next === "string" && next !== "" ? next : null; + hasMore = Boolean(cursor); + if (resume) { + writeCheckpoint( + { env, root, target: channel }, + { channel: CHANNEL, target: channel, cursor, pages, items, updated_at: now }, + ); + } + onProgress(`page ${pages}: ${pageItems.length} messages, has_more=${hasMore}`); + } + + const ledger = appendLedger(knowledgeDir, ledgerEntries); + if (resume) clearCheckpoint({ env, root, target: channel }); + + const receipt = { + ...base, + ok: true, + person: person ?? null, + channel_id: channel, + pages, + items, + requests, + cursor, + resumed_from: checkpoint?.cursor ?? null, + outputs, + ledger: { path: ledger.path, added: ledger.added, total: ledger.total }, + unavailable: [], + }; + return { ok: true, exitCode: 0, receipt: scrub(receipt, secrets) }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + const result = fail(error); + if (ledgerEntries.length > 0) { + try { + const ledger = appendLedger(knowledgeDir, ledgerEntries); + result.receipt.ledger = { path: ledger.path, added: ledger.added, total: ledger.total }; + result.receipt.partial = true; + } catch { + result.receipt.warnings.push("could not register the partial pages in knowledge/index.json"); + } + } + return result; + } +} + +export const HELP = `distilly collect slack — Slack 频道消息采集（只读）/ Slack channel history (read-only) + +用法 (zh): + distilly collect slack --channel [--person ] [--root ] + [--limit 200] [--max-pages 10] [--max-retries 4] + [--since ] [--no-resume] [--json] + +凭据：~/.distilly/${CONFIG_FILE}（bot_token，0600）或 DISTILLY_SLACK_BOT_TOKEN。 + 错误信息只出现配置文件名，绝不出现值。缺凭据 → 非零退出 + 补救步骤。 +权限：channels:history / groups:history / channels:read / users:read；机器人必须先被 /invite 进频道。 +限流：HTTP 429 与 {"ok":false,"error":"ratelimited"} 都按 Retry-After / 指数退避重试，超上限则保留已落盘页并非零退出。 +只读：本模块没有任何点赞/关注/发帖/私信调用；所有请求都是 GET（ALLOWED_MUTATIONS 为空）。 + +--- +## English + distilly collect slack --channel [--person ] [--limit N] [--json] + +Credentials: ~/.distilly/${CONFIG_FILE} or DISTILLY_SLACK_BOT_TOKEN. GET only. +Rate limits honour Retry-After; partial runs keep their pages and resume from a checkpoint. +`; + +export function parseFlags(argv) { + const flags = { _: [] }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (!arg.startsWith("--")) { + flags._.push(arg); + continue; + } + const name = arg.slice(2); + if (name === "json" || name === "help" || name === "no-resume") { + flags[name] = true; + continue; + } + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + flags[name] = value; + index += 1; + } + return flags; +} + +/** + * @param {string[]} argv arguments after `collect slack` + * @param {{env?: object, stdout?: Function, stderr?: Function, fetch?: Function}} [io] + * @returns {Promise} exit code + */ +export async function runCollectCli(argv, io = {}) { + const out = io.stdout ?? ((line) => process.stdout.write(`${line}\n`)); + const err = io.stderr ?? ((line) => process.stderr.write(`${line}\n`)); + + let flags; + try { + flags = parseFlags(argv); + } catch (error) { + err(`Error: ${error.message}`); + return 1; + } + if (flags.help) { + out(HELP); + return 0; + } + + const result = await collect({ + fetch: io.fetch ?? globalThis.fetch, + env: io.env ?? process.env, + root: flags.root ?? process.cwd(), + person: flags.person, + family: flags.family, + channel: flags.channel, + limit: flags.limit ? Number(flags.limit) : undefined, + maxPages: flags["max-pages"] ? Number(flags["max-pages"]) : undefined, + maxRetries: flags["max-retries"] ? Number(flags["max-retries"]) : undefined, + since: flags.since, + resume: !flags["no-resume"], + sleep: io.sleep, + now: io.now, + baseUrl: flags["base-url"], + }); + + if (flags.json) out(JSON.stringify(result.receipt, null, 2)); + if (result.ok) { + out(`collected ${result.receipt.items} message(s) in ${result.receipt.pages} page(s) → ${result.receipt.outputs.length} raw file(s)`); + for (const warning of result.receipt.warnings) err(`warning: ${warning}`); + } else { + err(`Error: ${result.receipt.errors?.[0] ?? "collect failed"}`); + for (const entry of result.receipt.unavailable) { + err(`unavailable: ${entry.channel} — ${entry.reason}`); + for (const step of entry.remediation ?? []) err(` fix: ${step}`); + } + for (const warning of result.receipt.warnings) err(`warning: ${warning}`); + } + return result.exitCode; +} diff --git a/src/collect/x.mjs b/src/collect/x.mjs new file mode 100644 index 00000000..6b625367 --- /dev/null +++ b/src/collect/x.mjs @@ -0,0 +1,968 @@ +/** + * x.mjs — X (Twitter) collection, two ways. + * + * api X API v2 with a Bearer token: `GET /2/users/by/username/:handle` + * then `GET /2/users/:id/tweets` with `pagination_token` (page cursor) + * and `since_id` (incremental re-collection). Read-only, GET only. + * browser computer use, which this module does **not** perform. It owns the + * consent gate (`src/consent.mjs`, scope `collect:x:browser`), the + * verbatim sink under `knowledge/raw/x/` and the ledger registration + * of whatever the host captured. Without `--consent ` (or with + * an expired one) the command exits 2 and the receipt says the run is + * waiting for user consent. + * + * Legacy context: `tools/research/xquik_public_posts.py` collected public posts + * through the Xquik aggregator (`XQUIK_API_KEY`). That third-party route is *not* + * migrated here — this module talks to X directly — and the aggregator remains a + * known gap in `docs/evidence/pr-07-collect-consent.md`. + * + * Read-only by construction: `ALLOWED_MUTATIONS` is empty, so `assertReadOnly()` + * refuses every non-GET request this file could ever issue. There is no code + * path that likes, follows, reposts, posts or DMs. + */ + +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +import { consentTokenFingerprint, verify as verifyConsent } from "../consent.mjs"; + +export const CHANNEL = "x"; +export const CONFIG_FILE = "x_config.json"; +export const LEGACY_CONFIG_FILE = join(".colleague-skill", CONFIG_FILE); +export const DEFAULT_BASE_URL = "https://api.x.com/2"; +export const DEFAULT_PAGE_SIZE = 100; +export const DEFAULT_MAX_PAGES = 10; +export const DEFAULT_MAX_RETRIES = 4; +export const DEFAULT_MAX_BACKOFF_MS = 60_000; +export const BROWSER_SCOPE = "collect:x:browser"; + +export const ENV_KEYS = { + bearer: ["DISTILLY_X_BEARER_TOKEN", "X_BEARER_TOKEN", "TWITTER_BEARER_TOKEN"], +}; + +/** X API v2 reads only. Any non-GET request is refused before it is sent. */ +export const ALLOWED_MUTATIONS = []; + +const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); +const REMEDIATION_SETUP = [ + `create ~/.distilly/${CONFIG_FILE} (chmod 600) with {"bearer_token": "…"}`, + " app: https://developer.x.com → Project & Apps → Keys and tokens → Bearer Token", + "or export DISTILLY_X_BEARER_TOKEN for this shell only", + "browser mode does not need this token: it needs consent — distilly consent grant --scope collect:x:browser", +]; + +export class CollectFailure extends Error { + constructor(reason, message, { remediation = [], exitCode = 1, kind = "failure" } = {}) { + super(message); + this.name = "CollectFailure"; + this.reason = reason; + this.remediation = remediation; + this.exitCode = exitCode; + this.kind = kind; + } +} + +export function redact(text, secrets = []) { + let output = String(text ?? ""); + for (const secret of secrets) { + if (typeof secret !== "string" || secret.length < 4) continue; + output = output.split(secret).join("[redacted]"); + } + return output; +} + +export function scrub(value, secrets = []) { + return JSON.parse( + JSON.stringify(value, (_key, item) => (typeof item === "string" ? redact(item, secrets) : item)), + ); +} + +export function distillyHome(env = process.env) { + const override = env?.DISTILLY_HOME; + return override ? resolve(String(override)) : join(homedir(), ".distilly"); +} + +export function credentialPaths(env = process.env) { + return { + primary: join(distillyHome(env), CONFIG_FILE), + legacy: join(homedir(), LEGACY_CONFIG_FILE), + }; +} + +export function loadCredential({ env = process.env, readFile = readFileSync } = {}) { + for (const name of ENV_KEYS.bearer) { + const value = env?.[name]; + if (typeof value === "string" && value.trim() !== "") { + return { + ok: true, + source: "env", + configFile: CONFIG_FILE, + path: null, + values: { bearer_token: value.trim() }, + }; + } + } + + const { primary, legacy } = credentialPaths(env); + for (const [path, source] of [ + [primary, "config"], + [legacy, "legacy-config"], + ]) { + if (!existsSync(path)) continue; + let parsed; + try { + parsed = JSON.parse(readFile(path, "utf8")); + } catch (error) { + throw new CollectFailure( + "bad-credential-file", + `${CONFIG_FILE} is not valid JSON (${redact(error.message)}); rewrite it with {"bearer_token": "…"}`, + { remediation: REMEDIATION_SETUP }, + ); + } + const token = parsed.bearer_token ?? parsed.bearerToken ?? parsed.bearer ?? parsed.api_key ?? null; + if (!token) { + throw new CollectFailure("incomplete-credential", `${CONFIG_FILE} is missing bearer_token`, { + remediation: REMEDIATION_SETUP, + }); + } + return { ok: true, source, configFile: CONFIG_FILE, path, values: { bearer_token: token } }; + } + + throw new CollectFailure("no-credential", `no credential at ~/.distilly/${CONFIG_FILE}`, { + remediation: REMEDIATION_SETUP, + }); +} + +export function assertReadOnly(url, method = "GET") { + const verb = String(method).toUpperCase(); + if (!MUTATING_METHODS.has(verb)) return true; + const path = (() => { + try { + return new URL(url).pathname; + } catch { + return String(url); + } + })(); + const allowed = ALLOWED_MUTATIONS.some((entry) => entry.method === verb && path.endsWith(entry.path)); + if (!allowed) { + throw new CollectFailure( + "write-operation-refused", + `refusing ${verb} ${path}: this collector never writes to ${CHANNEL}`, + { remediation: ["collectors are read-only; remove the mutating call instead of allowlisting it"] }, + ); + } + return true; +} + +export function parseRetryAfter(headerValue, nowMs = Date.now()) { + if (headerValue === null || headerValue === undefined || headerValue === "") return null; + const seconds = Number(headerValue); + if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000); + const date = Date.parse(String(headerValue)); + if (Number.isFinite(date)) return Math.max(0, date - nowMs); + return null; +} + +/** X sends `x-rate-limit-reset` as epoch seconds when the window resets. */ +export function parseRateLimitReset(headerValue, nowMs = Date.now()) { + const epochSeconds = Number(headerValue); + if (!Number.isFinite(epochSeconds) || epochSeconds <= 0) return null; + return Math.max(0, epochSeconds * 1000 - nowMs); +} + +export function backoffDelay(attempt, retryAfterMs = null, options = {}) { + const { baseMs = 500, maxMs = DEFAULT_MAX_BACKOFF_MS } = options; + if (Number.isFinite(retryAfterMs) && retryAfterMs !== null) return Math.min(retryAfterMs, maxMs); + return Math.min(baseMs * 2 ** Math.max(0, attempt - 1), maxMs); +} + +export function defaultSleep(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +export async function requestJson(options) { + const { + fetchImpl, + url, + method = "GET", + headers = {}, + body, + maxRetries = DEFAULT_MAX_RETRIES, + sleep = defaultSleep, + secrets = [], + onRetry = () => {}, + authRemediation = REMEDIATION_SETUP, + } = options; + + assertReadOnly(url, method); + if (typeof fetchImpl !== "function") { + throw new CollectFailure("no-fetch", "no fetch implementation available", { + remediation: ["run on Node >= 20, or pass an injected fetch"], + }); + } + + let attempt = 0; + for (;;) { + attempt += 1; + let response; + try { + response = await fetchImpl(url, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); + } catch (error) { + const message = redact(error?.message ?? String(error), secrets); + if (attempt > maxRetries) { + throw new CollectFailure("network-error", `request failed: ${message}`, { + remediation: ["check the network/proxy and retry", ...authRemediation], + }); + } + await sleep(backoffDelay(attempt)); + onRetry({ attempt, status: null, delayMs: backoffDelay(attempt), reason: message }); + continue; + } + + const status = Number(response?.status ?? 0); + const headerBag = response?.headers; + const retryAfterMs = + parseRetryAfter(headerBag?.get?.("retry-after") ?? null) ?? + (status === 429 ? parseRateLimitReset(headerBag?.get?.("x-rate-limit-reset") ?? null) : null); + + if (status === 429 || status >= 500) { + if (attempt > maxRetries) { + throw new CollectFailure( + status === 429 ? "rate-limited" : "server-error", + status === 429 + ? `rate limited (HTTP 429) after ${maxRetries} retries` + : `server error (HTTP ${status}) after ${maxRetries} retries`, + { + remediation: [ + "retry later; already-fetched pages stay on disk and the cursor is checkpointed", + "X API v2 windows are 15 minutes; lower --limit / --max-pages", + ], + }, + ); + } + const delayMs = backoffDelay(attempt, retryAfterMs); + onRetry({ attempt, status, delayMs, reason: `HTTP ${status}` }); + await sleep(delayMs); + continue; + } + + const text = await response.text(); + if (status === 401 || status === 403) { + throw new CollectFailure("unauthorized", `HTTP ${status} from ${CHANNEL}; bearer token rejected or not entitled`, { + remediation: [ + `the bearer token in ~/.distilly/${CONFIG_FILE} was rejected — regenerate it in the X developer portal`, + "check that your access level includes user tweet lookup", + ], + }); + } + if (status >= 400) { + throw new CollectFailure("http-error", `HTTP ${status}: ${redact(text.slice(0, 200), secrets)}`, { + remediation: ["check the request parameters", ...authRemediation], + }); + } + + let json = null; + try { + json = JSON.parse(text); + } catch { + throw new CollectFailure("invalid-json", `${CHANNEL} returned a non-JSON body`, { + remediation: ["retry later; if it persists the endpoint may have changed"], + }); + } + return { status, text, json, attempts: attempt, headers: headerBag }; + } +} + +export function sha256Hex(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function slug(text, fallback = "target") { + const slugged = String(text ?? "") + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^[-._]+|[-._]+$/g, "") + .slice(0, 64); + return slugged || fallback; +} + +export function knowledgeRoot({ root = process.cwd(), person, family = "colleague" } = {}) { + return person ? join(resolve(root), "skills", slug(family), slug(person), "knowledge") : join(resolve(root), "knowledge"); +} + +export function writeRaw(knowledgeDir, name, bytes) { + const dir = join(knowledgeDir, "raw", CHANNEL); + mkdirSync(dir, { recursive: true }); + const path = join(dir, `${slug(name, "page")}.json`); + const staging = `${path}.${process.pid}.tmp`; + const buffer = Buffer.from(bytes); + try { + writeFileSync(staging, buffer); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { + path, + relativePath: `raw/${CHANNEL}/${slug(name, "page")}.json`, + bytes: buffer.length, + sha256: sha256Hex(buffer), + }; +} + +export function appendLedger(knowledgeDir, entries) { + if (entries.length === 0) return { path: join(knowledgeDir, "index.json"), added: 0, total: 0, existed: false }; + const path = join(knowledgeDir, "index.json"); + let existing = []; + if (existsSync(path)) { + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + existing = Array.isArray(parsed) ? parsed : []; + } catch (error) { + throw new CollectFailure("bad-ledger", `knowledge/index.json is not valid JSON: ${redact(error.message)}`, { + remediation: ["repair or remove knowledge/index.json, then rerun the collect"], + }); + } + } + const byId = new Map(existing.filter((e) => e && typeof e === "object").map((e) => [e.id, e])); + let added = 0; + for (const entry of entries) { + if (!byId.has(entry.id)) added += 1; + byId.set(entry.id, entry); + } + const merged = [...byId.values()].sort((a, b) => String(a.id).localeCompare(String(b.id))); + const staging = `${path}.${process.pid}.tmp`; + try { + writeFileSync(staging, `${JSON.stringify(merged, null, 2)}\n`); + renameSync(staging, path); + } catch (error) { + if (existsSync(staging)) rmSync(staging, { force: true }); + throw error; + } + return { path, added, total: merged.length, existed: existing.length > 0 }; +} + +/** + * Run state for X: unlike the other channels the file survives a successful run, + * because `since_id` is what makes the *next* run incremental. + */ +export function statePath({ env = process.env, root = process.cwd(), target } = {}) { + const key = sha256Hex(Buffer.from(`${resolve(root)}\n${target ?? ""}`, "utf8")).slice(0, 12); + return join(distillyHome(env), "state", `${CHANNEL}-${key}.json`); +} + +export function readState(options) { + const path = statePath(options); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +export function writeState(options, value) { + const path = statePath(options); + mkdirSync(dirname(path), { recursive: true }); + const staging = `${path}.${process.pid}.tmp`; + writeFileSync(staging, `${JSON.stringify(value, null, 2)}\n`); + renameSync(staging, path); + return path; +} + +/** + * Resolve `@handle` → numeric user id (X API v2). + * @returns {Promise} the numeric id + */ +export async function resolveUserId(handle, { fetchImpl, baseUrl, headers, maxRetries, sleep, secrets, onRetry }) { + const clean = String(handle).replace(/^@/, "").trim(); + if (!/^[A-Za-z0-9_]{1,15}$/.test(clean)) { + throw new CollectFailure("bad-username", `invalid X username: ${JSON.stringify(handle)}`, { + remediation: ["use 1-15 letters, digits or underscores, with or without the leading @", ...REMEDIATION_SETUP], + }); + } + const response = await requestJson({ + fetchImpl, + url: `${baseUrl}/users/by/username/${encodeURIComponent(clean)}`, + headers, + maxRetries, + sleep, + secrets, + onRetry, + }); + const id = response.json?.data?.id; + if (!id) { + throw new CollectFailure("user-not-found", `X has no user @${clean}`, { + remediation: ["check the handle; deleted or suspended accounts return no data", ...REMEDIATION_SETUP], + }); + } + return String(id); +} + +/** + * api mode: collect one user's recent posts with cursor pagination and + * `since_id` incremental collection. + * + * @param {object} options `fetch`, `env`, `root`, `person`, `userId` or `username`, + * `limit`, `maxPages`, `maxRetries`, `sinceId`, `paginationToken`, `resume`, `sleep`, `now` + */ +export async function collectApi(options = {}) { + const { + fetch: fetchImpl = globalThis.fetch, + env = process.env, + root = process.cwd(), + person, + family = "colleague", + userId, + username, + limit = DEFAULT_PAGE_SIZE, + maxPages = DEFAULT_MAX_PAGES, + maxRetries = DEFAULT_MAX_RETRIES, + sinceId, + paginationToken, + resume = true, + sleep = defaultSleep, + now = new Date().toISOString(), + baseUrl = env?.DISTILLY_X_BASE_URL || DEFAULT_BASE_URL, + tweetFields = "created_at,public_metrics,lang,author_id", + onProgress = () => {}, + } = options; + + const knowledgeDir = knowledgeRoot({ root, person, family }); + const outputs = []; + const warnings = []; + const ledgerEntries = []; + const retries = []; + const onRetry = (info) => { + retries.push(info); + warnings.push(`retry ${info.attempt} after ${info.reason} (waited ${info.delayMs}ms)`); + onProgress(`retry ${info.attempt}: ${info.reason}`); + }; + + let secrets = []; + let credential = null; + let pages = 0; + let items = 0; + let requests = 0; + let cursor = paginationToken ?? null; + let since = sinceId ?? null; + let prior = null; + let target = userId ? String(userId) : username ? `@${String(username).replace(/^@/, "")}` : null; + + const base = { + command: "collect", + channel: CHANNEL, + mode: "api", + ok: false, + inputs: [], + outputs, + warnings, + unavailable: [], + credential_file: CONFIG_FILE, + retries, + }; + const fail = (failure) => ({ + ok: false, + exitCode: failure.exitCode ?? 1, + receipt: scrub( + { + ...base, + ok: false, + person: person ?? null, + target, + pages, + items, + requests, + cursor, + since_id: since, + errors: [redact(failure.message, secrets)], + unavailable: [ + { + channel: CHANNEL, + reason: redact(`${failure.reason}: ${failure.message}`, secrets), + remediation: failure.remediation ?? [], + }, + ], + }, + secrets, + ), + }); + + try { + if (!userId && !username) { + throw new CollectFailure("missing-target", "collect x needs --user-id or --username ", { + remediation: [ + "find the numeric id with `distilly collect x --username --dry-run` or from the profile URL", + ], + }); + } + + credential = loadCredential({ env }); + secrets = [credential.values.bearer_token]; + base.credential_source = credential.source; + const headers = { authorization: `Bearer ${credential.values.bearer_token}` }; + + // Resolve the handle first so the run state is always keyed by the numeric id. + let resolvedUserId = userId ? String(userId) : null; + if (!resolvedUserId) { + resolvedUserId = await resolveUserId(username, { fetchImpl, baseUrl, headers, maxRetries, sleep, secrets, onRetry }); + requests += 1; + } + target = resolvedUserId; + + if (resume) prior = readState({ env, root, target }); + if (since === undefined || since === null) { + if (prior?.since_id) { + since = prior.since_id; + warnings.push(`incremental since_id ${since} from the previous run`); + } + } + if (cursor === null && prior?.cursor && prior?.completed !== true) { + cursor = prior.cursor; + pages = Number(prior.pages ?? 0); + warnings.push(`resuming from checkpoint cursor (page ${pages} done)`); + } + const resumedFrom = cursor; + const sinceFromCheckpoint = prior?.since_id ?? null; + + let hasMore = true; + let newestId = prior?.newest_id ?? null; + const pageSize = Math.min(Math.max(10, Number(limit) || DEFAULT_PAGE_SIZE), 100); + while (hasMore) { + if (pages >= maxPages) { + warnings.push(`stopped after --max-pages ${maxPages}; rerun to continue from the cursor`); + break; + } + + const url = new URL(`${baseUrl}/users/${encodeURIComponent(resolvedUserId)}/tweets`); + url.searchParams.set("max_results", String(pageSize)); + url.searchParams.set("tweet.fields", tweetFields); + if (cursor) url.searchParams.set("pagination_token", cursor); + if (since) url.searchParams.set("since_id", String(since)); + + const response = await requestJson({ + fetchImpl, + url: url.toString(), + headers, + maxRetries, + sleep, + secrets, + onRetry, + }); + requests += 1; + + if (response.json?.errors?.length) { + const detail = redact(JSON.stringify(response.json.errors).slice(0, 200), secrets); + throw new CollectFailure("api-error", `X API error: ${detail}`, { + remediation: ["check the user id / access level, then retry", ...REMEDIATION_SETUP], + }); + } + + pages += 1; + const pageItems = Array.isArray(response.json?.data) ? response.json.data : []; + const meta = response.json?.meta ?? {}; + items += pageItems.length; + if (meta.newest_id) newestId = String(meta.newest_id); + + const stored = writeRaw(knowledgeDir, `${target}-p${String(pages).padStart(3, "0")}`, response.text); + outputs.push({ path: stored.path, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }); + ledgerEntries.push({ + id: `${CHANNEL}:${slug(target)}:p${String(pages).padStart(3, "0")}`, + kind: "raw", + origin: stored.relativePath, + source: CHANNEL, + fetched_at: now, + bytes: stored.bytes, + sha256: stored.sha256, + credentialed: true, + credential_source: credential.source, + credential_file: CONFIG_FILE, + method: "api-v2-bearer", + since_id: since ? String(since) : null, + newest_id: meta.newest_id ? String(meta.newest_id) : null, + items: pageItems.length, + warnings: [], + }); + + const next = meta.next_token; + cursor = typeof next === "string" && next !== "" ? next : null; + const ids = pageItems.map((item) => String(item?.id ?? "")).filter(Boolean); + const allOlderThanSince = since ? ids.length > 0 && ids.every((id) => BigInt(id) <= BigInt(since)) : false; + hasMore = Boolean(cursor) && pageItems.length > 0 && !allOlderThanSince; + + if (resume) { + writeState( + { env, root, target }, + { + channel: CHANNEL, + target, + cursor, + pages, + items, + since_id: since ? String(since) : null, + newest_id: newestId, + updated_at: now, + completed: false, + }, + ); + } + onProgress(`page ${pages}: ${pageItems.length} posts, has_more=${hasMore}`); + } + + const ledger = appendLedger(knowledgeDir, ledgerEntries); + if (resume) { + writeState( + { env, root, target }, + { + channel: CHANNEL, + target, + cursor: null, + pages, + items, + since_id: newestId ?? (since ? String(since) : null), + newest_id: newestId, + updated_at: now, + completed: true, + }, + ); + } + + return { + ok: true, + exitCode: 0, + receipt: scrub( + { + ...base, + ok: true, + person: person ?? null, + target, + pages, + items, + requests, + cursor, + since_id: since ? String(since) : null, + newest_id: newestId, + resumed_from: resumedFrom, + since_from_checkpoint: sinceFromCheckpoint, + outputs, + ledger: { path: ledger.path, added: ledger.added, total: ledger.total }, + unavailable: [], + }, + secrets, + ), + }; + } catch (error) { + if (!(error instanceof CollectFailure)) { + throw new CollectFailure("unexpected", redact(error?.message ?? String(error), secrets), { + remediation: ["rerun with --json and report the receipt"], + }); + } + const result = fail(error); + if (ledgerEntries.length > 0) { + try { + const ledger = appendLedger(knowledgeDir, ledgerEntries); + result.receipt.ledger = { path: ledger.path, added: ledger.added, total: ledger.total }; + result.receipt.partial = true; + } catch { + result.receipt.warnings.push("could not register the partial pages in knowledge/index.json"); + } + } + return result; + } +} + +/** The plan the host must follow in browser mode. No browser runs here. */ +export function browserPlan({ target, window = "recent", now = new Date().toISOString() }) { + return { + generated_at: now, + target: target ?? "@handle or profile URL", + window, + host_steps: [ + "Host (computer use): open a browser you control, sign in as the user, and open the target profile or search.", + "Host: scroll/expand the requested window; do not click like, follow, repost, reply or send — this tool cannot, and the grant does not cover it.", + "Host: save the captured posts (JSON or text, verbatim) to a file.", + `Host: register the capture with: distilly collect x --mode browser --consent --capture `, + ], + boundary: [ + "This module never drives a browser, never injects into one and never sends input events.", + "It owns three things only: the consent gate, the verbatim sink under knowledge/raw/x/, and the ledger entry.", + "Browser automation itself is a host capability (Claude Code, Codex, …); see docs/v2/STATUS.md for the host matrix.", + ], + }; +} + +/** + * browser mode: consent gate + sink + ledger registration for a host capture. + */ +export function collectBrowser(options = {}) { + const { + env = process.env, + root = process.cwd(), + person, + family = "colleague", + scope = BROWSER_SCOPE, + consentToken, + capturePath, + urls = [], + label, + target, + producer = "host:computer-use", + now = new Date().toISOString(), + readFile = readFileSync, + } = options; + + const knowledgeDir = knowledgeRoot({ root, person, family }); + const base = { + command: "collect", + channel: CHANNEL, + mode: "browser", + ok: false, + inputs: [], + outputs: [], + warnings: [], + unavailable: [], + credential_file: null, + }; + + const verification = verifyConsent(consentToken, { env, scope }); + if (!verification.ok) { + return { + ok: false, + exitCode: 2, + receipt: { + ...base, + ok: false, + status: "waiting-for-user-consent", + person: person ?? null, + target: target ?? null, + errors: [`waiting for user consent (${verification.reason})`], + unavailable: [ + { + channel: CHANNEL, + reason: `waiting for user consent: ${verification.reason}`, + scope, + remediation: verification.remediation, + }, + ], + consent_scope: scope, + }, + }; + } + + const consentBlock = { + scope: verification.record.scope, + granted_at: verification.record.granted_at, + expires_at: verification.record.expires_at, + token_sha256_12: consentTokenFingerprint(consentToken), + }; + + if (!capturePath) { + return { + ok: true, + exitCode: 0, + receipt: { + ...base, + ok: true, + status: "awaiting-host-capture", + person: person ?? null, + target: target ?? null, + consent: consentBlock, + plan: browserPlan({ target, now }), + unavailable: [], + }, + }; + } + + let bytes; + try { + bytes = readFile(capturePath); + } catch (error) { + return { + ok: false, + exitCode: 1, + receipt: { + ...base, + ok: false, + person: person ?? null, + errors: [`cannot read --capture ${capturePath}: ${redact(error.message)}`], + unavailable: [ + { + channel: CHANNEL, + reason: `capture-unreadable: ${redact(error.message)}`, + remediation: ["point --capture at a readable file produced by the host"], + }, + ], + }, + }; + } + + const name = label ?? `browser-${String(now).slice(0, 10)}`; + const stored = writeRaw(knowledgeDir, name, bytes); + const entry = { + id: `${CHANNEL}:${slug(name)}:capture`, + kind: "raw", + origin: stored.relativePath, + source: CHANNEL, + fetched_at: now, + bytes: stored.bytes, + sha256: stored.sha256, + credentialed: false, + method: "browser-host", + // The page the host captured. One capture is one page, so `url` is exact; a + // multi-page hand-over keeps them in the receipt and leaves `url` null + // rather than naming an arbitrary one of them. + url: urls.length === 1 ? urls[0] : null, + provenance: { method: "browser-host", producer, confidence: "host-reported" }, + consent: consentBlock, + warnings: [], + }; + const ledger = appendLedger(knowledgeDir, [entry]); + + return { + ok: true, + exitCode: 0, + receipt: { + ...base, + ok: true, + status: "captured", + person: person ?? null, + target: target ?? null, + consent: consentBlock, + urls: [...urls], + outputs: [{ path: stored.path, sha256: stored.sha256, bytes: stored.bytes, kind: "raw" }], + ledger: { path: ledger.path, added: ledger.added, total: ledger.total }, + provenance: entry.provenance, + unavailable: [], + }, + }; +} + +export async function collect(options = {}) { + return options.mode === "browser" ? collectBrowser(options) : collectApi(options); +} + +export const HELP = `distilly collect x — X（Twitter）采集 / X collection + +用法 (zh): + distilly collect x --user-id [--person ] [--limit 100] [--max-pages 10] + [--since-id ] [--pagination-token ] [--no-resume] [--json] + distilly collect x --username […同上…] + distilly collect x --mode browser --consent [--capture ] [--json] + +凭据：~/.distilly/${CONFIG_FILE}（bearer_token，0600）或 DISTILLY_X_BEARER_TOKEN； + 错误信息只出现配置文件名，绝不出现值。 +分页/续采：pagination_token 翻页；成功后把 newest_id 记为 since_id，下次自动增量采集。 +限流：429 按 Retry-After（缺失时用 x-rate-limit-reset）退避，超上限保留已落盘页并非零退出。 +同意门：browser 模式必须带 --consent （distilly consent grant --scope ${BROWSER_SCOPE}）， + 无 token / 已过期 → exit 2 + 回执写“等待用户同意”；本工具不做浏览器自动化，只负责 + 同意门 + 原样落盘 + 账本登记，浏览器操作由宿主（computer use）完成。 +只读：ALLOWED_MUTATIONS 为空，任何非 GET 请求都会被 assertReadOnly() 拒绝；不存在点赞/关注/发帖/私信代码。 + +--- +## English + distilly collect x --user-id | --username [--since-id ] [--json] + X API v2, Bearer token, cursor pagination + since_id incremental collection. + distilly collect x --mode browser --consent [--capture ] [--json] + Requires a consent grant; without one the command exits 2 and the receipt says the + run is waiting for user consent. This tool does not automate a browser — the host + captures, this module stores bytes verbatim and registers them in the ledger. +`; + +export function parseFlags(argv) { + const flags = { _: [] }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (!arg.startsWith("--")) { + flags._.push(arg); + continue; + } + const name = arg.slice(2); + if (name === "json" || name === "help" || name === "no-resume") { + flags[name] = true; + continue; + } + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + flags[name] = value; + index += 1; + } + return flags; +} + +export async function runCollectCli(argv, io = {}) { + const out = io.stdout ?? ((line) => process.stdout.write(`${line}\n`)); + const err = io.stderr ?? ((line) => process.stderr.write(`${line}\n`)); + + let flags; + try { + flags = parseFlags(argv); + } catch (error) { + err(`Error: ${error.message}`); + return 1; + } + if (flags.help) { + out(HELP); + return 0; + } + + const result = await collect({ + fetch: io.fetch ?? globalThis.fetch, + env: io.env ?? process.env, + root: flags.root ?? process.cwd(), + person: flags.person, + family: flags.family, + mode: flags.mode ?? "api", + userId: flags["user-id"], + username: flags.username, + limit: flags.limit ? Number(flags.limit) : undefined, + maxPages: flags["max-pages"] ? Number(flags["max-pages"]) : undefined, + maxRetries: flags["max-retries"] ? Number(flags["max-retries"]) : undefined, + sinceId: flags["since-id"], + paginationToken: flags["pagination-token"], + resume: !flags["no-resume"], + consentToken: flags.consent, + capturePath: flags.capture, + // `--url` is repeatable: the host is told which pages to capture, and the + // receipt has to name them back, otherwise "what did this consent actually + // authorise" is unanswerable after the fact. + urls: flags.url === undefined ? [] : [].concat(flags.url), + label: flags.label, + target: flags.target, + sleep: io.sleep, + now: io.now, + baseUrl: flags["base-url"], + }); + + if (flags.json) out(JSON.stringify(result.receipt, null, 2)); + if (result.ok) { + const status = result.receipt.status; + if (status) out(`ok (${status})`); + else out(`collected ${result.receipt.items} post(s) in ${result.receipt.pages} page(s) → ${result.receipt.outputs.length} raw file(s)`); + for (const warning of result.receipt.warnings ?? []) err(`warning: ${warning}`); + for (const step of result.receipt.plan?.host_steps ?? []) err(`host: ${step}`); + } else { + err(`Error: ${result.receipt.errors?.[0] ?? "collect failed"}`); + for (const entry of result.receipt.unavailable ?? []) { + err(`unavailable: ${entry.channel} — ${entry.reason}`); + for (const step of entry.remediation ?? []) err(` fix: ${step}`); + } + for (const warning of result.receipt.warnings ?? []) err(`warning: ${warning}`); + } + return result.exitCode; +} diff --git a/src/commands/credentialed.mjs b/src/commands/credentialed.mjs new file mode 100644 index 00000000..b3666f82 --- /dev/null +++ b/src/commands/credentialed.mjs @@ -0,0 +1,195 @@ +/** Channels frozen in CONTRACT §1 that this build does not implement yet. */ +export const PENDING_CHANNELS = {}; + +/** + * Credentialed commands — `collect`, `consent`, `transcribe` (from ds/07). + * + * Those modules own their flags, receipts and exit codes; this adapter only + * registers them with the command registry and guarantees CONTRACT §3: in + * `--json` mode stdout carries exactly one JSON object, whatever the module + * prints while it works (hence the stdout capture). + */ + +import { register } from "./index.mjs"; + +const CHANNELS = { + feishu: () => import("../collect/feishu.mjs"), + slack: () => import("../collect/slack.mjs"), + dingtalk: () => import("../collect/dingtalk.mjs"), + x: () => import("../collect/x.mjs"), +}; + +/** Run `fn` with stdout/stderr captured, so the dispatcher owns the output. */ +async function capture(fn) { + const originalOut = process.stdout.write.bind(process.stdout); + const originalErr = process.stderr.write.bind(process.stderr); + let out = ""; + let err = ""; + process.stdout.write = (chunk) => { + out += String(chunk); + return true; + }; + process.stderr.write = (chunk) => { + err += String(chunk); + return true; + }; + try { + const result = await fn(); + return { result, out, err }; + } finally { + process.stdout.write = originalOut; + process.stderr.write = originalErr; + } +} + +/** + * The first JSON object in captured stdout, ignoring whatever follows it. + * + * A channel module emits its receipt and then keeps printing human lines + * (`granted …`, `token: …`). Parsing "from the first brace to the end of the + * buffer" therefore failed on the trailing prose, and the caller quietly fell + * back to an empty default receipt — so `consent grant --json` reported + * `ok: true` with no `action`, no `grants` and no `outputs`, while the real + * receipt had been written. Cutting at the matching brace keeps the receipt. + */ +function parseReceiptFrom(text) { + const start = text.indexOf("{"); + if (start === -1) return null; + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < text.length; index += 1) { + const character = text[index]; + if (inString) { + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') inString = false; + continue; + } + if (character === '"') inString = true; + else if (character === "{") depth += 1; + else if (character === "}") { + depth -= 1; + if (depth === 0) { + try { + return JSON.parse(text.slice(start, index + 1)); + } catch { + return null; + } + } + } + } + return null; +} + +function forward(text, write) { + for (const line of text.split("\n")) if (line.trim()) write(line); +} + +async function runModule(load, argv, { json, reporter }) { + const module = await load(); + const entry = module.runCollectCli ?? module.runConsentCli ?? module.runTranscribeCli; + if (typeof entry !== "function") throw new Error("module exposes no CLI entry point"); + // The dispatcher strips the global `--json` before handing argv to a command, + // but these modules parse their own flags and only emit a receipt when they see + // it. Without this, `--json` reached the CLI and never reached the module: it + // printed prose, `parseReceiptFrom` found no object, and the caller fell back to + // an empty default receipt (`ok: true`, no action, no grants, no outputs). + const moduleArgv = json && !argv.includes("--json") ? [...argv, "--json"] : argv; + const { result, out, err } = await capture(() => entry(moduleArgv, { json })); + // Human prose from a channel module goes to stderr in `--json` mode (stdout is + // the receipt alone), so stdout lines are only forwarded in prose mode. Stderr + // is forwarded either way: that is where the module puts the remedy + // ("waiting for user consent … fix: distilly consent grant"), and swallowing it + // left a failure with a non-zero exit and no explanation. + if (!json) forward(out, (line) => reporter.line(line)); + forward(err, (line) => reporter.warn(line)); + const receipt = result?.receipt ?? parseReceiptFrom(out) ?? undefined; + // A channel module signals an early failure by returning the **exit code** (a + // number) rather than a result object — `runCollectCli` returns 1 when the key + // is missing. Reading only `result?.exitCode` turned every one of those into a + // success: `collect feishu` with no credential exited 0 with an empty receipt, + // which is the opposite of "fail loudly with a remedy". + const numeric = typeof result === "number" ? result : null; + const exitCode = numeric ?? result?.exitCode ?? (result?.ok === false ? 1 : 0); + return { receipt, exitCode }; +} + +const collectHelp = { + zh: [ + "用法 / Usage:", + " distilly collect [options] [--json]", + " distilly collect x --mode browser --consent # computer use，需显式同意", + "", + "需要凭据的渠道由脚本负责鉴权/分页/限流/续采；缺 key 或缺同意 → 响亮失败并给补救步骤，", + "回执与日志里只出现配置文件名，永不出现 key 值。", + ].join("\n"), + en: [ + "Distilly collect — credentialed channels (feishu, slack, dingtalk, x).", + "Browser collection requires `--consent `; without it the command exits 2.", + "Receipts and logs name the credential file, never its contents.", + ].join("\n"), +}; + +register("collect", { + summary: "需要凭据的渠道采集 / credentialed collection", + usage: "distilly collect [options] [--json]", + ...collectHelp, + async run({ argv, json, reporter }) { + const [channel, ...rest] = argv; + if (!channel || channel === "--help" || channel === "help") { + reporter.line(`Usage: distilly collect <${Object.keys(CHANNELS).join("|")}> [options] [--json]`); + return { receipt: undefined, exitCode: channel ? 0 : 2 }; + } + const load = CHANNELS[channel]; + if (!load) { + return { + receipt: { + command: "collect", + person: null, + ok: false, + inputs: [], + outputs: [], + anchors: { total: 0, cited: 0 }, + warnings: [], + unavailable: [], + error: { code: "collect/unknown-channel", message: `unsupported channel: ${channel}`, remedy: `known channels: ${Object.keys(CHANNELS).join(", ")}` }, + }, + exitCode: 2, + }; + } + return runModule(load, rest, { json, reporter }); + }, +}); + +register("consent", { + summary: "computer-use 同意管理 / consent tokens", + usage: "distilly consent [options] [--json]", + zh: [ + "用法 / Usage:", + " distilly consent [--json]", + "", + "子命令 / Subcommands:", + " distilly consent grant --scope collect:x:browser [--ttl ] [--note ]", + " distilly consent list", + " distilly consent verify --token --scope collect:x:browser", + " distilly consent revoke --token ", + " distilly consent prune", + "", + "同意令牌存在 ~/.distilly/consent.json；没有令牌时 collect --mode browser 退出码 2。", + ].join("\n"), + en: ["Distilly consent — grant, list, verify, revoke or prune the computer-use consent tokens kept in ~/.distilly/consent.json."].join("\n"), + async run({ argv, json, reporter }) { + return runModule(() => import("../consent.mjs"), argv, { json, reporter }); + }, +}); + +register("transcribe", { + summary: "可选的转写后端 / optional transcription backend", + usage: "distilly transcribe [options] [--json]", + zh: ["用法 / Usage:", " distilly transcribe [--json]", "", "没有可用后端时明确报 unavailable，不静默降级；产物带 provenance{method,producer,confidence}。"].join("\n"), + en: ["Distilly transcribe — optional backend (OpenAI-compatible HTTP or a host capability). Reports `unavailable` instead of degrading silently, and records provenance with every transcript."].join("\n"), + async run({ argv, json, reporter }) { + return runModule(() => import("../optional/transcribe.mjs"), argv, { json, reporter }); + }, +}); diff --git a/src/commands/doctor.mjs b/src/commands/doctor.mjs new file mode 100644 index 00000000..f7911f76 --- /dev/null +++ b/src/commands/doctor.mjs @@ -0,0 +1,423 @@ +/** + * `distilly doctor` — inventory health check (minimal, honest version). + * + * Checks what is actually on disk: + * 1. every host in the shared matrix: is Distilly installed there, at which + * version, and does the directory look like a Distilly install; + * 2. generated Skills under `skills//` (version, corrections); + * 3. ledger coverage whenever `knowledge/index.json` exists; + * 4. capabilities this build cannot run yet are listed in `unavailable` + * (CONTRACT §3: nothing is silently skipped). + */ + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; + +import { register, PLANNED } from "./index.mjs"; +import { createReceipt, describeFile, displayPath } from "../cli/receipt.mjs"; +import { parseArgs } from "../cli/args.mjs"; +import { listAgents } from "../hosts/agents.mjs"; +import { inspectInstall, repoInstallDir } from "../install/hosts.mjs"; +import { resolveSkillsRoot } from "../cli/paths.mjs"; +import { parseAnchor } from "../knowledge/anchors.mjs"; +import { CHARACTER_PRESETS } from "../skill/presets.mjs"; +import { listSkills } from "../skill/writer.mjs"; + +function doctorHelp(binary = "distilly") { + const zh = [ + "用法：", + ` ${binary} doctor [--base-dir ] [--json]`, + "", + "检查项（最小版）：", + " 1. 宿主矩阵里每个宿主是否已安装 Distilly（路径、SKILL.md 版本）；", + " 2. 生成的 Skill 清单（skills//，含版本与 corrections）；", + " 3. 账本覆盖率：有 knowledge/index.json 时统计条目与字节数；", + " 4. 未实现能力（parse/view/collect 等）写进回执的 unavailable，不静默跳过。", + ].join("\n"); + const en = [ + "Usage:", + ` ${binary} doctor [--base-dir ] [--json]`, + "", + "Checks (minimal):", + " 1. whether Distilly is installed for each host in the shared matrix (path, SKILL.md version);", + " 2. the generated Skill inventory (skills// with version and corrections);", + " 3. ledger coverage: entry and byte counts whenever knowledge/index.json exists;", + " 4. capabilities this build cannot run yet (parse/view/collect …) are reported in the receipt's unavailable list, never skipped silently.", + ].join("\n"); + return { zh, en }; +} + +const OPTIONS = { + "base-dir": { type: "string", value: "dir" }, + // Opt-in: make a corpus-shape FAIL affect `ok` and the exit code. The shape verdict + // is always *reported*; it only becomes a gate when the caller says so, because + // reading a corpus that cannot carry a person is a legitimate thing to do while + // collecting more material. + "require-shape": { type: "boolean" }, +}; + +/** + * Every anchor string an `evidence/derived/*.json` file cites. + * + * The derived layer is prose-plus-claims: a claim carries `evidence: ["k0001"]`, + * a boundary carries anchors too. Walking the JSON for `k00NN`-shaped strings + * finds them all without coupling to one schema, which is what "doctor checks + * what is on disk" means here. + */ +function citedAnchors(skillDir) { + const derivedDir = join(skillDir, "evidence", "derived"); + if (!existsSync(derivedDir)) return []; + const found = new Set(); + const walk = (value) => { + if (typeof value === "string") { + if (parseAnchor(value)) found.add(value); + return; + } + if (Array.isArray(value)) { + for (const item of value) walk(item); + return; + } + if (value && typeof value === "object") { + for (const item of Object.values(value)) walk(item); + } + }; + for (const name of readdirSync(derivedDir)) { + if (!name.endsWith(".json")) continue; + try { + walk(JSON.parse(readFileSync(join(derivedDir, name), "utf8"))); + } catch { + // A malformed derived file is reported by `view check`, not here. + } + } + return [...found]; +} + +/** + * Every anchor string the **deliverable** cites — the generated Skill and the page. + * + * `citedAnchors` above walks `evidence/derived/` only, which is the intermediate + * layer nobody reads. What this product ships is `SKILL.md` / `work.md` / + * `persona.md` / `work_skill.md` / `persona_skill.md` (and the rendered view). A + * Skill whose every rule cites evidence used to contribute **zero** to the `cited` + * figure, so the number described the middle of the pipeline while the artifact at + * the end of it went unexamined. + */ +function deliveredAnchors(skillDir) { + const found = new Set(); + const collect = (text) => { + for (const match of String(text).matchAll(/\[(k\d{4}(?::t\d+)?)\]/g)) found.add(match[1]); + }; + for (const name of ["SKILL.md", "work.md", "persona.md", "work_skill.md", "persona_skill.md"]) { + const path = join(skillDir, name); + if (existsSync(path)) collect(readFileSync(path, "utf8")); + } + const viewsDir = join(skillDir, "views"); + if (existsSync(viewsDir)) { + for (const name of readdirSync(viewsDir)) { + if (name.endsWith(".view.json")) collect(readFileSync(join(viewsDir, name), "utf8")); + } + } + return found; +} + +/** Every anchor a skill's ledger knows about, so a citation can be checked. */ +function resolvedAnchors(skillDir) { + const ledgerPath = join(skillDir, "knowledge", "index.json"); + if (!existsSync(ledgerPath)) return new Set(); + let entries = []; + try { + const parsed = JSON.parse(readFileSync(ledgerPath, "utf8")); + entries = Array.isArray(parsed) ? parsed : (parsed.entries ?? []); + } catch { + return new Set(); + } + const known = new Set(); + for (const entry of entries) { + for (const anchor of entry?.anchors ?? []) { + const id = typeof anchor === "string" ? anchor : anchor?.anchor ?? anchor?.id; + if (id) known.add(id); + } + for (const detail of entry?.anchor_detail ?? []) { + const id = detail?.anchor ?? detail?.id; + if (id) known.add(id); + } + } + return known; +} + +/** + * A pre-flight read on the **corpus shape**: is this material able to carry a + * person at all? + * + * Nothing in the pipeline asked that question before Step 4, and the failure is not + * hypothetical — a 47-minute multi-speaker floor proceeding was run through the whole + * five-step mainline, producing a portrait of a room instead of a person, because the + * only judge of "is this the right material" was the model's own judgement at the end. + * The numbers here come from what `retrospect` already derived, so this is a reading, + * not a second derivation: + * + * - `units` anchors the ledger records (how much material there is); + * - `speakers` distinct speakers the voice derivation could attribute units to; + * - `top_share` the busiest speaker's share of the attributed units. + * + * Verdict rules, deliberately blunt and stated in the receipt: + * - fewer than 20 citable units → FAIL (nothing to be a person *from*); + * - two or more speakers but under 40% of units attributable, or no speaker with at + * least 20% → FAIL (a meeting, not a person: ask for that person's own material); + * - one or no speaker labels → PASS with a note (a single-voice source is legitimate; + * absence of labels is not evidence of a crowd). + */ +function corpusShape(skillDir, ledger) { + const reasons = []; + const notes = []; + const units = ledger.anchors; + const voicePath = join(skillDir, "evidence", "derived", "voice.json"); + let bySpeaker = null; + if (existsSync(voicePath)) { + try { + const voice = JSON.parse(readFileSync(voicePath, "utf8")); + const claim = (voice.claims ?? []).find((item) => item?.id === "voice.sentence_length"); + bySpeaker = claim?.value?.by_speaker ?? null; + } catch { + bySpeaker = null; + } + } + const speakers = bySpeaker ? Object.keys(bySpeaker) : []; + const attributed = bySpeaker ? Object.values(bySpeaker).reduce((total, item) => total + (item?.samples ?? 0), 0) : 0; + const top = bySpeaker + ? Object.entries(bySpeaker).sort(([, a], [, b]) => (b?.samples ?? 0) - (a?.samples ?? 0))[0] + : null; + const topShare = top && attributed > 0 ? (top[1]?.samples ?? 0) / attributed : null; + + if (units < 20) reasons.push(`可引用单元只有 ${units} 个，低于 20：材料量不足以支撑一个人物画像`); + if (speakers.length >= 2) { + const share = units > 0 ? attributed / units : 0; + if (share < 0.4) { + reasons.push( + `这是多人材料，但只有 ${(share * 100).toFixed(0)}% 的单元能归到某个说话人（阈值 40%）：` + + "先补这个人自己的一手产出（本人访谈/演讲字幕、本人文章），不要从会议流水里切人", + ); + } + if (topShare !== null && topShare < 0.2) { + reasons.push(`最活跃的说话人只占已归属单元的 ${(topShare * 100).toFixed(0)}%（阈值 20%）：没有哪个人是这份材料的主角`); + } + } else if (speakers.length === 0) { + notes.push("材料里没有说话人标注：按单一对象处理（对本人文章/邮件/单人口述是正常的）"); + } else { + notes.push("只有一位说话人：按单一对象处理"); + } + + return { + units, + sources: ledger.entries, + speakers: speakers.length, + attributed_units: attributed, + top_speaker: top ? top[0] : null, + top_share: topShare === null ? null : Number(topShare.toFixed(4)), + verdict: reasons.length === 0 ? "PASS" : "FAIL", + reasons, + notes, + }; +} + +function readLedger(skillDir) { + const ledgerPath = join(skillDir, "knowledge", "index.json"); + if (!existsSync(ledgerPath)) { + return { path: ledgerPath, entries: 0, bytes: 0, anchors: 0, present: false }; + } + const text = readFileSync(ledgerPath, "utf8"); + let entries = []; + try { + const parsed = JSON.parse(text); + entries = Array.isArray(parsed) ? parsed : (parsed.entries ?? []); + } catch { + entries = []; + } + const anchors = entries.reduce((total, entry) => { + const list = entry?.anchors; + return total + (Array.isArray(list) ? list.length : 0); + }, 0); + return { + path: ledgerPath, + entries: entries.length, + bytes: statSync(ledgerPath).size, + anchors, + present: true, + }; +} + +register("doctor", { + summary: "体检宿主与 Skill 库存 / Health-check hosts and skill inventory", + usage: "distilly doctor [--base-dir ] [--require-shape] [--json]", + options: OPTIONS, + ...doctorHelp(), + run({ argv, reporter }) { + const { flags } = parseArgs(argv, OPTIONS); + const requireShape = Boolean(flags["require-shape"]); + const warnings = []; + const inputs = []; + const outputs = []; + + reporter.line("Hosts / 宿主:"); + const hostRows = []; + for (const id of listAgents()) { + const target = repoInstallDir(id); + const state = inspectInstall(target); + hostRows.push({ host: id, path: displayPath(target), installed: state.installed, version: state.version }); + reporter.line( + ` ${state.installed ? "installed" : "missing "} ${id.padEnd(18)} ${displayPath(target)}${state.version ? ` (v${state.version})` : ""}`, + ); + if (state.installed) { + const described = describeFile(join(target, "SKILL.md")); + if (described) inputs.push(described); + } + } + + reporter.line(""); + reporter.line("Skills / 人物 Skill:"); + const familyBase = flags["base-dir"]; + let skillCount = 0; + const shapes = []; + let anchorTotal = 0; + let citedTotal = 0; + const dangling = []; + for (const [family, preset] of Object.entries(CHARACTER_PRESETS)) { + if (preset.character !== family) continue; + // One resolver for all three `--base-dir` spellings: a bare directory must + // not be read as if a `skills/` level were underneath it. + // `--base-dir <...>/skills/colleague` names ONE family's storage root. The + // other families must not read the same directory again — doing so counted + // the same skill three times over ("inspected once, not once per family"). + if (familyBase && Object.hasOwn(CHARACTER_PRESETS, basename(familyBase)) && basename(familyBase) !== family) { + continue; + } + const resolved = familyBase + ? resolveSkillsRoot({ baseDir: familyBase, family }) + : { root: preset.storage_root ?? preset.legacy_storage_root, warning: null }; + if (resolved.warning && !warnings.includes(resolved.warning)) warnings.push(resolved.warning); + const baseDir = resolved.root; + // The corpus check is a **pre-flight** reading: it has to fire before Step 4 has + // produced a `SKILL.md`, so it walks every person directory that has a ledger + // rather than only the finished Skills `listSkills` returns. (Enumerating + // finished Skills made the check silently inapplicable in exactly the situation + // it exists for — right after Collect, before Distill.) + if (existsSync(baseDir)) { + for (const entry of readdirSync(baseDir).sort()) { + const personDir = join(baseDir, entry); + const ledgerPath = join(personDir, "knowledge", "index.json"); + if (!existsSync(ledgerPath)) continue; + shapes.push({ slug: `${family}/${entry}`, ...corpusShape(personDir, readLedger(personDir)) }); + } + } + const skills = listSkills(baseDir); + for (const skill of skills) { + skillCount += 1; + const skillDir = join(baseDir, skill.slug); + const ledger = readLedger(skillDir); + anchorTotal += ledger.anchors; + // `cited` counts the citations that actually resolve; a dangling one is + // reported in `warnings`, never folded into the number — "we cite two + // anchors" and "one of the two is broken" are different claims. + const known = resolvedAnchors(skillDir); + // Both layers count now: the derived claims *and* what the generated Skill and + // the page actually cite. A citation that does not resolve is a broken promise + // — "every conclusion can be traced back" — so it is collected for the verdict + // below, not only for a warning line. + const citedInSkill = new Set([...citedAnchors(skillDir), ...deliveredAnchors(skillDir)]); + for (const anchor of citedInSkill) { + if (known.has(anchor)) citedTotal += 1; + else dangling.push(`${family}/${skill.slug}: ${anchor}`); + } + reporter.line( + ` ${family}/${skill.slug} ${skill.version} corrections=${skill.corrections_count} ` + + `knowledge=${ledger.present ? `${ledger.entries} entries / ${ledger.bytes} bytes` : "none"}`, + ); + if (ledger.present) { + const described = describeFile(ledger.path); + if (described) outputs.push(described); + } + const skillFile = describeFile(join(skillDir, "SKILL.md")); + if (skillFile) outputs.push(skillFile); + } + } + if (skillCount === 0) { + reporter.line(" none found (run `distilly skill create` first)"); + warnings.push("no generated skills found"); + } + + if (dangling.length > 0) { + warnings.push( + `${dangling.length} anchor(s) cited by evidence/derived cannot be resolved against knowledge/index.json: ` + + dangling.join(", "), + ); + } + + reporter.line(""); + reporter.line("Corpus shape / 语料体检:"); + if (shapes.length === 0) reporter.line(" （没有可检查的 Skill）"); + for (const shape of shapes) { + reporter.line( + ` ${shape.slug} ${shape.verdict} units=${shape.units} sources=${shape.sources} ` + + `speakers=${shape.speakers} attributed=${shape.attributed_units}` + + (shape.top_speaker ? ` top=${shape.top_speaker}(${((shape.top_share ?? 0) * 100).toFixed(0)}%)` : ""), + ); + for (const reason of shape.reasons) reporter.line(` ✗ ${reason}`); + for (const note of shape.notes) reporter.line(` · ${note}`); + } + + const coverage = anchorTotal === 0 ? null : citedTotal / anchorTotal; + reporter.line(""); + reporter.line( + `Ledger coverage / 账本：${skillCount} skills, ${anchorTotal} anchors recorded, ${citedTotal} cited ` + + `by derived evidence or the generated Skill` + + (coverage === null ? "" : ` (${(coverage * 100).toFixed(0)}% of the ledger)`) + + (dangling.length > 0 ? `, ${dangling.length} DANGLING` : ", 0 dangling"), + ); + // The gate is **dangling = 0**, not a coverage percentage: a Skill is not obliged + // to cite every cue in the corpus, but it is obliged to not cite evidence that + // does not exist. Coverage stays in the receipt as information for a human. + const shapeFailed = shapes.filter((shape) => shape.verdict === "FAIL"); + if (dangling.length > 0) { + reporter.line("Verdict / 判定：FAIL —— 有引用回指不到账本，交付物在承诺它没有的证据"); + } else if (requireShape && shapeFailed.length > 0) { + reporter.line("Verdict / 判定：FAIL —— 语料形状撑不起一个人（见上），先补材料再蒸馏"); + } else if (shapeFailed.length > 0) { + reporter.line( + `Verdict / 判定：PASS（引用完整）；但语料体检 FAIL（${shapeFailed.length} 个）——` + + "加 --require-shape 会让它成为硬门槛", + ); + } else { + reporter.line(`Verdict / 判定：PASS —— 0 悬空引用${coverage === null ? "" : `；账本覆盖 ${(coverage * 100).toFixed(0)}%`}`); + } + + const unavailable = Object.entries(PLANNED).map(([command, branch]) => ({ + channel: command, + reason: `not implemented in this build; delivered by ${branch}`, + })); + reporter.line(""); + reporter.line(`Unavailable / 未实现：${unavailable.map((item) => item.channel).join(", ")}`); + + // `createReceipt` keeps exactly the eight contract fields, so anything command + // specific has to be attached to the object afterwards. + const receipt = createReceipt("doctor", { + inputs, + outputs, + // `ok: false` when a citation cannot be followed back: the CLI derives its exit + // code from this, so doctor now *fails* instead of printing a warning beside a + // green `ok: true`. Coverage stays a number, not a verdict — see above. + ok: dangling.length === 0 && (!requireShape || shapes.every((shape) => shape.verdict !== "FAIL")), + anchors: { total: anchorTotal, cited: citedTotal, dangling: dangling.length }, + warnings, + unavailable, + }); + receipt.skills = skillCount; + receipt.verdict = + dangling.length > 0 || (requireShape && shapes.some((shape) => shape.verdict === "FAIL")) ? "FAIL" : "PASS"; + receipt.shape = shapes; + // The host inventory belongs in the receipt too: it is the part of `doctor` + // a caller most often wants to read programmatically (`--json` is the + // machine interface), and it was reachable only through the internal `extra`. + receipt.hosts = hostRows.map(({ host, installed, path }) => ({ host, installed, path })); + return { receipt, extra: { hosts: hostRows, skills: skillCount } }; + }, +}); diff --git a/src/commands/harvest.mjs b/src/commands/harvest.mjs new file mode 100644 index 00000000..bdc6fffb --- /dev/null +++ b/src/commands/harvest.mjs @@ -0,0 +1,252 @@ +/** + * `distilly harvest` — zero-credential intake: files in, `knowledge/` out. + * + * This is the glue ds/02 could not finish before its session ended: it walks the + * given paths, picks a parser by extension, and records every document through + * the shared ledger (raw bytes + anchored text + append-only index). + * + * Honest by construction: an unsupported or unreadable file is reported in + * `warnings` and in the receipt, never skipped silently, and a re-import of + * identical bytes appends nothing (the ledger dedupes by sha256). + */ + +import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { basename, dirname, extname, join, relative, resolve } from "node:path"; + +import { register } from "./index.mjs"; +import { KnowledgeStore, sha256Hex } from "../knowledge/store.mjs"; +import { loadLedger, recordDocument, saveLedger, ledgerStats } from "../knowledge/ledger.mjs"; +import { IDENTITY_FILE, applyIdentity, loadIdentity } from "../knowledge/identity.mjs"; +import { SourceFile, buildDocument, recordsFromCharSpans } from "../parse/common.mjs"; +import { parseSubtitle } from "../parse/subtitle.mjs"; +import { parseChat } from "../parse/chat.mjs"; +import { detectFeishuFormat, parseFeishu } from "../parse/feishu.mjs"; + +const SUBTITLE_EXTENSIONS = new Set([".srt", ".vtt"]); +const TEXT_EXTENSIONS = new Set([".md", ".txt", ".text"]); +const CHAT_EXTENSIONS = new Set([".json"]); + +const help = { + zh: [ + "用法 / Usage:", + " distilly harvest --person [--base-dir ] [--source