From 127085ad2f104ec50d33f8698163a1646c2257d4 Mon Sep 17 00:00:00 2001 From: jeffapple Date: Fri, 9 Oct 2026 14:31:58 +0000 Subject: [PATCH] fix(digest): drop per-user rankings and lock English output with an e2e test The digest prints team-wide totals only; the unused per-user `ranked` arrays in summarizeInterventions / summarizeConversation are removed so no caller can bring individual rankings back. A real-CLI e2e test asserts the digest is all English and names no member; usage-guide docs updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01R5AmspzzVhLVupACwm4W3e --- docs/usage-guide.md | 4 +- docs/usage-guide.zh-CN.md | 4 +- node_modules | 1 + src/__tests__/conversation-token-e2e.test.ts | 2 +- .../conversation-token-metrics.test.ts | 2 +- src/__tests__/digest-interventions.test.ts | 11 +- src/__tests__/digest-output-e2e.test.ts | 159 ++++++++++++++++++ src/__tests__/intervention-e2e.test.ts | 2 +- src/digest.ts | 30 +--- 9 files changed, 181 insertions(+), 34 deletions(-) create mode 120000 node_modules create mode 100644 src/__tests__/digest-output-e2e.test.ts diff --git a/docs/usage-guide.md b/docs/usage-guide.md index cf189e229..2849e9e16 100644 --- a/docs/usage-guide.md +++ b/docs/usage-guide.md @@ -2031,7 +2031,7 @@ The prompt is checked when the `UserPromptSubmit` hook captures it, so a change Matching normalizes both the prompt and keywords to Unicode NFC. For example, `réessaye` matches `re\u0301essaye`, where `\u0301` is a combining acute accent. Accents remain significant, so `reessaye` does not match. Normalization applies only to matching and does not change the 60-second correction window. Correction detection uses the original prompt in memory; the original is then discarded, while the locally stored summary is secret-redacted and capped at 200 characters. -Intervention data is automatically aggregated and reported to the team's `stats/.yaml` during `teamai pull`, and shown in the "Session Autonomy" leaderboard of `teamai digest`, with team averages and per-person intervention rate rankings — useful for verifying whether a skill/rule reduces intervention rates after rollout. Tools without a transcript (e.g. Cursor) degrade gracefully, tracking only `correction`. +Intervention data is automatically aggregated and reported to the team's `stats/.yaml` during `teamai pull`, and shown in the "Session Autonomy" section of `teamai digest`, with the team average and a breakdown by intervention type (team-wide totals only, no per-person ranking) — useful for verifying whether a skill/rule reduces intervention rates after rollout. Tools without a transcript (e.g. Cursor) degrade gracefully, tracking only `correction`. #### Conversation Volume & Token Usage @@ -2044,7 +2044,7 @@ Each session row also shows two columns; Details retains secret-redacted capture > Privacy: shared turn/token metrics contain counts only. Redacted prompt summaries and output in dashboard details remain on this machine. -These two metrics are likewise aggregated into `stats/.yaml` (as `prompts` and `tokens` fields) during `teamai pull`, and shown in the "Conversation Volume & Token Usage" section of `teamai digest`, with team-wide totals, bucketed token totals, and per-person token usage rankings. Tools without transcript access (e.g. Cursor) degrade gracefully: turn counts are still tracked, while tokens show as 0 / N/A. +These two metrics are likewise aggregated into `stats/.yaml` (as `prompts` and `tokens` fields) during `teamai pull`, and shown in the "Conversation Volume & Token Usage" section of `teamai digest`, with team-wide prompt and bucketed token totals (team-wide totals only, no per-person ranking). Tools without transcript access (e.g. Cursor) degrade gracefully: turn counts are still tracked, while tokens show as 0 / N/A. #### Daily Session Trends & Estimated Cost diff --git a/docs/usage-guide.zh-CN.md b/docs/usage-guide.zh-CN.md index b46d2aadf..84dd05fff 100644 --- a/docs/usage-guide.zh-CN.md +++ b/docs/usage-guide.zh-CN.md @@ -1869,7 +1869,7 @@ sharing: 匹配时，prompt 和纠偏词都会转换为 Unicode NFC 形式。例如，`réessaye` 可以匹配 `re\u0301essaye`，其中 `\u0301` 是组合尖音符。重音符号仍有区别，因此 `reessaye` 不匹配。规范化仅用于匹配，不会改变 60 秒的纠偏时间窗口。纠偏检测在内存中使用原始 prompt，随后丢弃原文；本机仅保存经密钥脱敏且最长 200 个字符的摘要。 -干预数据会随 `teamai pull` 自动聚合上报到团队 `stats/.yaml`，并在 `teamai digest` 的「会话自主性」榜单中给出团队均值与人均干预率排行，可用于验证某个 skill / rule 上线后干预率是否下降。无 transcript 的工具（如 Cursor）会优雅降级，只统计 `correction`。 +干预数据会随 `teamai pull` 自动聚合上报到团队 `stats/.yaml`，并在 `teamai digest` 的「会话自主性」板块中给出团队均值与干预类型分解（只有团队汇总，不展示个人排行），可用于验证某个 skill / rule 上线后干预率是否下降。无 transcript 的工具（如 Cursor）会优雅降级，只统计 `correction`。 #### 对话量与 Token 用量 @@ -1882,7 +1882,7 @@ sharing: > 隐私：团队共享的轮数和 Token 指标仅含计数。Dashboard 详情中的脱敏输入摘要和输出保留在本机。 -这两项同样随 `teamai pull` 聚合到 `stats/.yaml`（`prompts` 与 `tokens` 字段），并在 `teamai digest` 的「对话量与 Token 用量」板块给出团队对话总轮数、token 总量（分桶）与人均 token 用量排行。拿不到 transcript 的工具（如 Cursor）会优雅降级：仍统计对话轮数，token 显示为 0 / N/A。 +这两项同样随 `teamai pull` 聚合到 `stats/.yaml`（`prompts` 与 `tokens` 字段），并在 `teamai digest` 的「对话量与 Token 用量」板块给出团队对话总轮数与 token 总量（分桶），只有团队汇总，不展示个人排行。拿不到 transcript 的工具（如 Cursor）会优雅降级：仍统计对话轮数，token 显示为 0 / N/A。 #### 每日会话趋势与估算成本 diff --git a/node_modules b/node_modules new file mode 120000 index 000000000..ca9311dd6 --- /dev/null +++ b/node_modules @@ -0,0 +1 @@ +/home/user/teamai-cli/.claude/worktrees/project-scope-usage/node_modules \ No newline at end of file diff --git a/src/__tests__/conversation-token-e2e.test.ts b/src/__tests__/conversation-token-e2e.test.ts index 1c99e8a2a..0bd13f1e4 100644 --- a/src/__tests__/conversation-token-e2e.test.ts +++ b/src/__tests__/conversation-token-e2e.test.ts @@ -142,7 +142,7 @@ describe('conversation + token metric — end to end', () => { const summary = summarizeConversation([userStats])!; expect(summary.totalPrompts).toBe(2); expect(summary.totalTokens).toBe(120 + 130 + 2500 + 200); - expect(summary.ranked[0].username).toBe('jeff'); + expect(JSON.stringify(summary)).not.toContain('jeff'); // team totals only, no per-user ranking // Idempotent: re-reporting the same events yields no new delta. const second = computePromptTokenDelta(metrics, nextReported); diff --git a/src/__tests__/conversation-token-metrics.test.ts b/src/__tests__/conversation-token-metrics.test.ts index 777858e60..711d6c06e 100644 --- a/src/__tests__/conversation-token-metrics.test.ts +++ b/src/__tests__/conversation-token-metrics.test.ts @@ -611,6 +611,6 @@ describe('summarizeConversation', () => { expect(s.totalPrompts).toBe(14); expect(s.tokens).toEqual({ input: 6000, output: 600, cacheRead: 0, cacheCreation: 0 }); expect(s.totalTokens).toBe(6600); - expect(s.ranked[0].username).toBe('bob'); // higher token usage first + expect(JSON.stringify(s)).not.toContain('bob'); // team totals only, no per-user ranking }); }); diff --git a/src/__tests__/digest-interventions.test.ts b/src/__tests__/digest-interventions.test.ts index fb0181786..2b34bdde7 100644 --- a/src/__tests__/digest-interventions.test.ts +++ b/src/__tests__/digest-interventions.test.ts @@ -30,14 +30,13 @@ describe('summarizeInterventions', () => { expect(s.avgPerSession).toBeCloseTo(14 / 15); }); - it('ranks users by intervention rate descending', () => { + it('carries no per-user ranking (team-wide totals only)', () => { const stats = [ - user('alice', { sessions: 10, interrupt: 2, toolReject: 1, correction: 1 }), // rate 0.4 - user('bob', { sessions: 5, interrupt: 5, toolReject: 0, correction: 5 }), // rate 2.0 + user('alice', { sessions: 10, interrupt: 2, toolReject: 1, correction: 1 }), + user('bob', { sessions: 5, interrupt: 5, toolReject: 0, correction: 5 }), ]; const s = summarizeInterventions(stats)!; - expect(s.ranked[0].username).toBe('bob'); - expect(s.ranked[0].rate).toBeCloseTo(2.0); - expect(s.ranked[1].username).toBe('alice'); + expect(JSON.stringify(s)).not.toContain('alice'); + expect(JSON.stringify(s)).not.toContain('bob'); }); }); diff --git a/src/__tests__/digest-output-e2e.test.ts b/src/__tests__/digest-output-e2e.test.ts new file mode 100644 index 000000000..68de72c33 --- /dev/null +++ b/src/__tests__/digest-output-e2e.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn, execFileSync } from 'node:child_process'; +import path from 'node:path'; +import fs from 'node:fs'; +import os from 'node:os'; +import { fileURLToPath } from 'node:url'; + +// ─── End-to-end: `teamai digest` output is English and names no member ────── +// +// A user-scope `git` provider team whose teamai-reports branch already holds +// two members' reported stats (skills, interventions, prompts, tokens). Runs +// the real CLI binary and checks the digest it prints: +// - every label is English (no CJK characters anywhere in the output), and +// - the autonomy / token sections report team-wide totals only — no +// per-user ranking lines naming individual members. + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, '..', '..'); +const CLI = path.join(ROOT, 'dist', 'index.js'); + +const GIT_ENV = { + GIT_AUTHOR_NAME: 'TeamAI CI', + GIT_AUTHOR_EMAIL: 'ci@teamai.test', + GIT_COMMITTER_NAME: 'TeamAI CI', + GIT_COMMITTER_EMAIL: 'ci@teamai.test', + GIT_TERMINAL_PROMPT: '0', +}; + +function runCLI(args: string[], homeDir: string, cwd: string): Promise<{ code: number | null; output: string }> { + return new Promise((resolve) => { + const child = spawn('node', [CLI, ...args], { + env: { ...process.env, ...GIT_ENV, FORCE_COLOR: '0', HOME: homeDir, USERPROFILE: homeDir }, + stdio: ['pipe', 'pipe', 'pipe'], + cwd, + }); + let output = ''; + child.stdout.on('data', (d: Buffer) => { output += d.toString(); }); + child.stderr.on('data', (d: Buffer) => { output += d.toString(); }); + child.stdin.end(); + child.on('close', (code) => resolve({ code, output })); + }); +} + +function git(args: string[], cwd: string): void { + execFileSync('git', args, { cwd, stdio: 'pipe', env: { ...process.env, ...GIT_ENV } }); +} + +const fwd = (value: string): string => value.split(path.sep).join('/'); + +// Distinctive member names: a ranking line would have to print one of them. +const MEMBERS = ['alice-rankme', 'bob-rankme'] as const; + +function reportedStats(username: string, uses: number, sessions: number, prompts: number, tokens: number): string { + return [ + `username: ${username}`, + 'updatedAt: 2026-06-01T00:00:00Z', + 'skills:', + ' deploy-helper:', + ` count: ${uses}`, + ' lastUsed: 2026-06-01T00:00:00Z', + 'interventions:', + ` sessions: ${sessions}`, + ` interrupt: ${sessions}`, + ' toolReject: 1', + ' correction: 2', + `prompts: ${prompts}`, + 'tokens:', + ` input: ${tokens}`, + ` output: ${tokens}`, + ' cacheRead: 0', + ' cacheCreation: 0', + '', + ].join('\n'); +} + +describe('digest output e2e', () => { + let sandbox: string; + let homeDir: string; + let cwd: string; + let digest: { code: number | null; output: string }; + + beforeAll(async () => { + if (!fs.existsSync(CLI)) { + throw new Error(`CLI binary not found at ${CLI}. Run "npm run build" first.`); + } + + sandbox = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-digest-e2e-')); + homeDir = path.join(sandbox, 'home'); + cwd = path.join(sandbox, 'cwd'); + const remote = path.join(sandbox, 'remote.git'); + const work = path.join(sandbox, 'work'); + const localPath = path.join(homeDir, '.teamai', 'team-repo'); + fs.mkdirSync(path.join(homeDir, '.teamai'), { recursive: true }); + fs.mkdirSync(cwd, { recursive: true }); + + // main: teamai.yaml only. + git(['init', '-q', '--bare', '-b', 'main', remote], sandbox); + fs.mkdirSync(work, { recursive: true }); + git(['init', '-q', '-b', 'main'], work); + fs.writeFileSync(path.join(work, 'teamai.yaml'), [ + 'team: digest-e2e', `repo: ${fwd(remote)}`, 'provider: git', '', + ].join('\n')); + git(['add', '-A'], work); + git(['commit', '-qm', 'team'], work); + git(['remote', 'add', 'origin', remote], work); + git(['push', '-q', 'origin', 'main'], work); + + // teamai-reports: two members' reported stats. + git(['checkout', '-q', '--orphan', 'teamai-reports'], work); + git(['rm', '-rq', '--cached', '.'], work); + fs.rmSync(path.join(work, 'teamai.yaml')); + fs.mkdirSync(path.join(work, 'stats'), { recursive: true }); + fs.writeFileSync(path.join(work, 'stats', `${MEMBERS[0]}.yaml`), reportedStats(MEMBERS[0], 3, 10, 40, 1000)); + fs.writeFileSync(path.join(work, 'stats', `${MEMBERS[1]}.yaml`), reportedStats(MEMBERS[1], 5, 4, 90, 5000)); + git(['add', '-A'], work); + git(['commit', '-qm', 'reports'], work); + git(['push', '-q', 'origin', 'teamai-reports'], work); + + git(['clone', '-q', remote, localPath], sandbox); + fs.writeFileSync(path.join(homeDir, '.teamai', 'config.yaml'), [ + 'repo:', + ` localPath: ${fwd(localPath)}`, + ` remote: ${fwd(remote)}`, + ' kind: git', + `username: ${MEMBERS[0]}`, + 'scope: user', + '', + ].join('\n')); + + digest = await runCLI(['digest'], homeDir, cwd); + }, 60_000); + + afterAll(() => { + if (sandbox) fs.rmSync(sandbox, { recursive: true, force: true }); + }); + + it('prints the team digest from the reported stats', () => { + expect(digest.code, digest.output).toBe(0); + expect(digest.output).toContain('Team AI Weekly Digest'); + expect(digest.output).toContain('Active members: 2'); + expect(digest.output).toContain('deploy-helper (8 uses)'); + }); + + it('labels every section in English', () => { + // No CJK characters anywhere in the output. + expect(digest.output).not.toMatch(/[㐀-鿿]/); + expect(digest.output).toContain('Session Autonomy (Human Intervention):'); + expect(digest.output).toContain('Team avg: 1.43 interventions/session (14 sessions, 20 interventions)'); + expect(digest.output).toContain('Breakdown: interrupt 14 · reject 2 · correction 4'); + expect(digest.output).toContain('Lifetime Conversation & Token Usage:'); + expect(digest.output).toContain('Lifetime human prompts: 130'); + expect(digest.output).toContain('Lifetime tokens: 12.0K (input 6.0K · output 6.0K · cache read 0 · cache write 0)'); + }); + + it('reports team-wide totals only, never a per-user ranking', () => { + for (const member of MEMBERS) expect(digest.output).not.toContain(member); + expect(digest.output).not.toMatch(/rank/i); + }); +}); diff --git a/src/__tests__/intervention-e2e.test.ts b/src/__tests__/intervention-e2e.test.ts index 4e2682348..a5d95c4c2 100644 --- a/src/__tests__/intervention-e2e.test.ts +++ b/src/__tests__/intervention-e2e.test.ts @@ -139,7 +139,7 @@ describe('Human Intervention metric — end to end', () => { expect(summary.totalSessions).toBe(1); expect(summary.totalInterventions).toBe(3); expect(summary.avgPerSession).toBeCloseTo(3); - expect(summary.ranked[0].username).toBe('jeff'); + expect(JSON.stringify(summary)).not.toContain('jeff'); // team totals only, no per-user ranking }); it('re-running the report on the same events yields no new delta (idempotent)', async () => { diff --git a/src/digest.ts b/src/digest.ts index 102aa3b07..632386aa7 100644 --- a/src/digest.ts +++ b/src/digest.ts @@ -274,13 +274,12 @@ export interface InterventionSummary { correction: number; /** Team-wide mean interventions per session. */ avgPerSession: number; - /** Per-user ranking by intervention rate (highest first = least autonomous). */ - ranked: Array<{ username: string; sessions: number; total: number; rate: number }>; } /** * Summarize the Human Intervention metric across all reported team stats. - * Returns null when no user has reported any interventions yet. + * Returns null when no user has reported any interventions yet. Team-wide + * totals only: the digest never ranks or names individual members. */ export function summarizeInterventions(teamStats: UserStats[]): InterventionSummary | null { const users = teamStats.filter((u) => u.interventions && u.interventions.sessions > 0); @@ -291,20 +290,13 @@ export function summarizeInterventions(teamStats: UserStats[]): InterventionSumm let toolReject = 0; let correction = 0; - const ranked = users.map((u) => { + for (const u of users) { const iv = u.interventions!; - const total = iv.interrupt + iv.toolReject + iv.correction; totalSessions += iv.sessions; interrupt += iv.interrupt; toolReject += iv.toolReject; correction += iv.correction; - return { - username: u.username, - sessions: iv.sessions, - total, - rate: iv.sessions > 0 ? total / iv.sessions : 0, - }; - }).sort((a, b) => b.rate - a.rate); + } const totalInterventions = interrupt + toolReject + correction; return { @@ -314,7 +306,6 @@ export function summarizeInterventions(teamStats: UserStats[]): InterventionSumm toolReject, correction, avgPerSession: totalSessions > 0 ? totalInterventions / totalSessions : 0, - ranked, }; } @@ -326,8 +317,6 @@ export interface ConversationSummary { tokens: TokenUsage; /** Grand total of all token buckets. */ totalTokens: number; - /** Per-user ranking by token usage (highest first). */ - ranked: Array<{ username: string; prompts: number; tokens: number }>; } /** Compact a token count into a human-friendly string (e.g. 12.3M, 4.5K). */ @@ -340,6 +329,7 @@ export function formatTokenCount(n: number): string { /** * Summarize the conversation-turn count and token usage across all reported team * stats. Returns null when no user has reported any prompts or tokens yet. + * Team-wide totals only: the digest never ranks or names individual members. */ export function summarizeConversation(teamStats: UserStats[]): ConversationSummary | null { const users = teamStats.filter( @@ -350,18 +340,16 @@ export function summarizeConversation(teamStats: UserStats[]): ConversationSumma let totalPrompts = 0; const tokens: TokenUsage = { input: 0, output: 0, cacheRead: 0, cacheCreation: 0 }; - const ranked = users.map((u) => { - const p = u.prompts ?? 0; + for (const u of users) { const t = u.tokens ?? { input: 0, output: 0, cacheRead: 0, cacheCreation: 0 }; - totalPrompts += p; + totalPrompts += u.prompts ?? 0; tokens.input += t.input; tokens.output += t.output; tokens.cacheRead += t.cacheRead; tokens.cacheCreation += t.cacheCreation; - return { username: u.username, prompts: p, tokens: totalTokens(t) }; - }).sort((a, b) => b.tokens - a.tokens); + } - return { totalPrompts, tokens, totalTokens: totalTokens(tokens), ranked }; + return { totalPrompts, tokens, totalTokens: totalTokens(tokens) }; } export function summarizeTeamTrends(teamStats: UserStats[], now = new Date()): ReturnType | null {