diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d16c9e..d149b09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,9 @@ jobs: - name: stdio MCP (real client) run: npm run verify:mcp:stdio + - name: HTTP MCP (auth / PKCE / lock / fail-closed) + run: npm run verify:mcp:http + semantic-real: name: semantic recall (real model) runs-on: ubuntu-latest diff --git a/package.json b/package.json index 22cbadc..8adadef 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "verify:wan:cross:selftest:isolated": "node scripts/wan-sync.mjs selftest-isolated", "bench:snapshot": "node scripts/bench-snapshot.mjs", "verify:mcp:stdio": "node scripts/verify-mcp-stdio.mjs", + "verify:mcp:http": "node scripts/verify-mcp-http.mjs", "dev": "node --loader ts-node/esm src/index.ts" }, "dependencies": { diff --git a/packages/mcp/bin/mebular.mjs b/packages/mcp/bin/mebular.mjs index 992c41b..dc3d8e9 100755 --- a/packages/mcp/bin/mebular.mjs +++ b/packages/mcp/bin/mebular.mjs @@ -1,17 +1,115 @@ #!/usr/bin/env node // mebular CLI（@mebular/mcp） // -// G6.2 实现 `mcp`（stdio MCP server）；其余命令按阶段补齐（G6.3+）。 +// G6.2：mcp（stdio）；G6.3：serve（Streamable HTTP + 鉴权 + 单实例）、token。 -const command = process.argv[2]; +import { randomUUID } from 'node:crypto'; +import { existsSync } from 'node:fs'; +import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +const argv = process.argv.slice(2); +const command = argv[0]; + +function parseFlags(list) { + const flags = {}; + for (let i = 0; i < list.length; i++) { + const token = list[i]; + if (token.startsWith('--')) { + const key = token.slice(2); + const next = list[i + 1]; + if (next === undefined || next.startsWith('--')) flags[key] = true; + else { + flags[key] = next; + i++; + } + } + } + return flags; +} + +function homeDir() { + return process.env.MEBULAR_HOME ?? join(process.cwd(), '.mebular'); +} +function tokensPath(flagValue) { + return flagValue ?? process.env.MEBULAR_TOKENS_FILE ?? join(homeDir(), 'auth', 'tokens.json'); +} + +async function runToken(action, flags) { + const path = tokensPath(typeof flags['tokens-file'] === 'string' ? flags['tokens-file'] : undefined); + let data = { tokens: [] }; + if (existsSync(path)) { + try { + data = JSON.parse(await readFile(path, 'utf-8')); + if (!Array.isArray(data.tokens)) data = { tokens: [] }; + } catch (error) { + console.error(`✗ tokens 文件损坏：${path}（${error.message}）`); + process.exit(2); + } + } + const { createHash } = await import('node:crypto'); + if (action === 'grant') { + const scope = String(flags.scope ?? 'memory.read').split(',').map((s) => s.trim()).filter(Boolean); + const token = `meb_${randomUUID().replace(/-/g, '')}${randomUUID().replace(/-/g, '')}`; + const id = randomUUID(); + data.tokens.push({ id, sha256: createHash('sha256').update(token).digest('hex'), scope, label: flags.label ?? undefined, createdAt: new Date().toISOString(), revoked: false }); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, JSON.stringify(data, null, 2), 'utf-8'); + await chmod(path, 0o600); + console.log(JSON.stringify({ id, token, scope, tokensFile: path }, null, 2)); + console.log('（token 仅显示一次，请立即保存）'); + return; + } + if (action === 'list') { + console.log(JSON.stringify(data.tokens.map(({ id, scope, label, createdAt, revoked }) => ({ id, scope, label, createdAt, revoked })), null, 2)); + return; + } + if (action === 'revoke') { + const id = flags.id; + const record = data.tokens.find((t) => t.id === id); + if (!record) { + console.error(`✗ 未找到 token：${id}`); + process.exit(2); + } + record.revoked = true; + await writeFile(path, JSON.stringify(data, null, 2), 'utf-8'); + console.log(`已吊销 token ${id}`); + return; + } + console.error('用法：mebular token [--scope a,b] [--id x] [--tokens-file path]'); + process.exit(2); +} async function main() { + const flags = parseFlags(argv.slice(1)); switch (command) { case 'mcp': { const { startStdioServer } = await import('../src/server.mjs'); await startStdioServer(); return; } + case 'serve': { + const { startServeServer } = await import('../src/server.mjs'); + try { + const result = await startServeServer({ + host: typeof flags.host === 'string' ? flags.host : undefined, + port: flags.port !== undefined ? Number(flags.port) : undefined, + auth: typeof flags.auth === 'string' ? flags.auth : undefined, + tlsKey: typeof flags['tls-key'] === 'string' ? flags['tls-key'] : undefined, + tlsCert: typeof flags['tls-cert'] === 'string' ? flags['tls-cert'] : undefined, + tokensFile: typeof flags['tokens-file'] === 'string' ? flags['tokens-file'] : undefined, + }); + console.log(`SERVE_READY ${JSON.stringify({ host: result.host, port: result.port, auth: result.auth, issuer: result.issuer })}`); + } catch (error) { + console.error(`✗ serve 启动失败（${error?.code ?? 'ERROR'}）：${error?.message ?? error}`); + process.exit(2); + } + return; + } + case 'token': { + await runToken(argv[1], flags); + return; + } case '--help': case '-h': case undefined: @@ -20,23 +118,18 @@ async function main() { '用法：mebular ', '', '命令：', - ' mcp 启动 stdio MCP server（stdio 传输）', - ' serve 启动 Streamable HTTP server（G6.3 计划）', - ' init 初始化 .mebular 配置（G6.4 计划）', - ' keygen 生成用户主密钥（G6.4 计划）', - ' print-config 打印各 client 接入片段（G6.4 计划）', - ' status 打印记忆状态（G6.3 计划）', - ' token 管理访问令牌（G6.3 计划）', + ' mcp 启动 stdio MCP server', + ' serve [--host --port --auth --tls-key --tls-cert --tokens-file] Streamable HTTP server', + ' token grant|list|revoke [--scope a,b] [--id x] [--tokens-file p] 访问令牌管理', + ' init / keygen / print-config / status （G6.4+ 计划）', ].join('\n'), ); process.exit(0); return; - case 'serve': case 'init': case 'keygen': case 'print-config': case 'status': - case 'token': console.error(`mebular ${command}：尚未实现（按 G6 计划补齐）`); process.exit(2); return; diff --git a/packages/mcp/src/config.mjs b/packages/mcp/src/config.mjs index b81f07e..65c8885 100644 --- a/packages/mcp/src/config.mjs +++ b/packages/mcp/src/config.mjs @@ -118,7 +118,7 @@ export async function createMebular() { }, }); await app.initialize(); - return { app, home, config }; + return { app, home, config, storagePath, deviceId }; } export { homedir }; diff --git a/packages/mcp/src/serve.mjs b/packages/mcp/src/serve.mjs new file mode 100644 index 0000000..e13e77d --- /dev/null +++ b/packages/mcp/src/serve.mjs @@ -0,0 +1,426 @@ +// Streamable HTTP server（G6.3）：/mcp、/healthz、OAuth 最小 AS+RS、单实例锁。 +// +// - 传输：SDK WebStandardStreamableHTTPServerTransport（stateful + JSON 响应）。 +// - 认证：none（stdio 除外；环回可无）/ bearer（tokensFile 存 sha256）/ oauth（内置最小 AS+RS）。 +// - fail closed：非环回必须 TLS 且 auth != none，否则拒绝启动。 +// - 单实例：/lock O_EXCL + PID 存活检测 + 陈旧回收。 + +import { createHash, randomUUID } from 'node:crypto'; +import { existsSync } from 'node:fs'; +import { chmod, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'; +import http from 'node:http'; +import https from 'node:https'; +import { dirname, join } from 'node:path'; +import { WebStandardStreamableHTTPServerTransport } from '@modelcontextprotocol/server'; +import { TOOL_SCOPES } from './tools.mjs'; + +const SCOPES = ['memory.read', 'memory.write', 'memory.admin']; +const SCOPE_RANK = { 'memory.read': 0, 'memory.write': 1, 'memory.admin': 2 }; +const ACCESS_TTL = 900; // 15min +const REFRESH_TTL = 30 * 24 * 3600; // 30d + +// ---------- 单实例锁 ---------- + +function pidAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +export async function acquireLock(home, storagePath) { + const path = join(home, 'lock'); + await mkdir(dirname(path), { recursive: true }); + const payload = JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString(), storagePath }); + const write = () => writeFile(path, payload, 'utf-8'); + const release = async () => { await unlink(path).catch(() => undefined); }; + + if (!existsSync(path)) { + await writeFile(path, payload, { encoding: 'utf-8', flag: 'wx' }); + return { path, release }; + } + let held = null; + try { + held = JSON.parse(await readFile(path, 'utf-8')); + } catch { + // 损坏锁按陈旧处理 + } + if (held?.pid && pidAlive(held.pid)) { + const error = new Error( + `存储已被占用：pid ${held.pid}（自 ${held.startedAt}，${held.storagePath}）。请复用该 serve 实例，或先停止它。`, + ); + error.code = 'MCP_STORAGE_LOCKED'; + throw error; + } + // 陈旧锁回收 + await write(); + return { path, release }; +} + +// ---------- bearer ---------- + +export function hashToken(token) { + return createHash('sha256').update(token).digest('hex'); +} + +export async function loadTokens(tokensFile) { + if (!tokensFile || !existsSync(tokensFile)) return []; + try { + const data = JSON.parse(await readFile(tokensFile, 'utf-8')); + return Array.isArray(data.tokens) ? data.tokens : []; + } catch (error) { + throw new Error(`tokensFile 损坏：${tokensFile}（${error.message}）`); + } +} + +async function verifyBearer(token, tokensFile) { + const digest = hashToken(token); + const tokens = await loadTokens(tokensFile); + const record = tokens.find((t) => t.sha256 === digest && !t.revoked); + if (!record) return null; + return { tokenId: record.id ?? 'bearer', scopes: record.scope ?? [] }; +} + +// ---------- OAuth 最小 AS/RS ---------- + +function b64url(bytes) { + return Buffer.from(bytes).toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} +function b64urlToBytes(s) { + return new Uint8Array(Buffer.from(s.replace(/-/g, '+').replace(/_/g, '/'), 'base64')); +} +function ab(bytes) { + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); +} + +async function loadOrCreateSigningKey(home) { + const path = join(home, 'auth', 'as-key.json'); + if (existsSync(path)) { + const data = JSON.parse(await readFile(path, 'utf-8')); + const privateKey = await crypto.subtle.importKey('pkcs8', ab(b64urlToBytes(data.privateKeyPkcs8)), { name: 'Ed25519' }, true, ['sign']); + return { privateKey, publicRaw: b64urlToBytes(data.publicKey), kid: data.kid }; + } + const kp = await crypto.subtle.generateKey({ name: 'Ed25519' }, true, ['sign', 'verify']); + const pkcs8 = new Uint8Array(await crypto.subtle.exportKey('pkcs8', kp.privateKey)); + const publicRaw = new Uint8Array(await crypto.subtle.exportKey('raw', kp.publicKey)); + const kid = randomUUID(); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, JSON.stringify({ privateKeyPkcs8: b64url(pkcs8), publicKey: b64url(publicRaw), kid }), 'utf-8'); + await chmod(path, 0o600); + return { privateKey: kp.privateKey, publicRaw, kid }; +} + +async function signJwt(signingKey, payload) { + const header = { alg: 'EdDSA', typ: 'JWT', kid: signingKey.kid }; + const body = `${b64url(Buffer.from(JSON.stringify(header)))}.${b64url(Buffer.from(JSON.stringify(payload)))}`; + const sig = new Uint8Array(await crypto.subtle.sign({ name: 'Ed25519' }, signingKey.privateKey, new TextEncoder().encode(body))); + return `${body}.${b64url(sig)}`; +} + +async function verifyJwt(signingKey, token, { issuer, resource, revoked }) { + const parts = String(token).split('.'); + if (parts.length !== 3) return null; + const [h, p, s] = parts; + const data = new TextEncoder().encode(`${h}.${p}`); + const key = await crypto.subtle.importKey('raw', ab(signingKey.publicRaw), { name: 'Ed25519' }, false, ['verify']); + const ok = await crypto.subtle.verify({ name: 'Ed25519' }, key, ab(b64urlToBytes(s)), data); + if (!ok) return null; + let payload; + try { + payload = JSON.parse(Buffer.from(b64urlToBytes(p), 'utf8').toString('utf-8')); + } catch { + return null; + } + const now = Math.floor(Date.now() / 1000); + if (payload.iss !== issuer) return null; + if (payload.aud !== resource) return null; + if (typeof payload.exp !== 'number' || payload.exp < now) return null; + if (revoked.has(payload.jti)) return null; + return payload; +} + +function metadataFor(issuer) { + const resource = `${issuer}/mcp`; + return { + protectedResource: { + resource, + authorization_servers: [issuer], + scopes_supported: SCOPES, + bearer_methods_supported: ['header'], + }, + authorizationServer: { + issuer, + authorization_endpoint: `${issuer}/authorize`, + token_endpoint: `${issuer}/token`, + registration_endpoint: `${issuer}/register`, + jwks_uri: `${issuer}/jwks`, + response_types_supported: ['code'], + grant_types_supported: ['authorization_code', 'refresh_token', 'client_credentials'], + code_challenge_methods_supported: ['S256'], + token_endpoint_auth_methods_supported: ['none'], + scopes_supported: SCOPES, + }, + }; +} + +// ---------- 请求/响应桥 ---------- + +function toWebRequest(req, body, origin) { + const headers = new Headers(); + for (const [k, v] of Object.entries(req.headers)) { + if (Array.isArray(v)) for (const item of v) headers.append(k, item); + else if (v !== undefined) headers.set(k, v); + } + return new Request(new URL(req.url, origin), { + method: req.method, + headers, + ...(body && body.length > 0 && req.method !== 'GET' && req.method !== 'HEAD' ? { body } : {}), + }); +} + +async function writeWebResponse(res, response) { + res.statusCode = response.status; + for (const [k, v] of response.headers) res.setHeader(k, v); + if (response.body) { + const reader = response.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + res.write(Buffer.from(value)); + } + } + res.end(); +} + +function sendJson(res, status, data) { + const body = JSON.stringify(data); + res.statusCode = status; + res.setHeader('content-type', 'application/json'); + res.end(body); +} + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + req.on('data', (c) => chunks.push(c)); + req.on('end', () => resolve(Buffer.concat(chunks))); + req.on('error', reject); + }); +} + +function requiredScopeForBody(body) { + let messages; + try { + const parsed = JSON.parse(body.toString('utf-8')); + messages = Array.isArray(parsed) ? parsed : [parsed]; + } catch { + return 'memory.read'; + } + let required = 'memory.read'; + for (const msg of messages) { + if (!msg || typeof msg.method !== 'string') continue; + if (msg.method === 'initialize' || msg.method.startsWith('notifications/') || msg.method === 'ping') continue; + let scope = 'memory.read'; + if (msg.method === 'tools/call') { + scope = TOOL_SCOPES[msg.params?.name] ?? 'memory.read'; + } else if (msg.method.startsWith('tools/') || msg.method.startsWith('resources/') || msg.method.startsWith('prompts/')) { + scope = 'memory.read'; + } + if ((SCOPE_RANK[scope] ?? 0) > (SCOPE_RANK[required] ?? 0)) required = scope; + } + return required; +} + +function hasScope(granted, required) { + const rank = SCOPE_RANK[required] ?? 0; + return (granted ?? []).some((s) => (SCOPE_RANK[s] ?? -1) >= rank); +} + +/** + * 启动 HTTP MCP server。 + * @returns {Promise<{ server: import('node:http').Server, host: string, port: number, auth: string, close: () => Promise }>} + */ +export async function startHttpServer({ home, service, buildServer, host = '127.0.0.1', port = 7331, auth = 'none', tls = false, tlsKey, tlsCert, tokensFile }) { + const isLoopback = host === '127.0.0.1' || host === 'localhost' || host === '::1'; + if (!isLoopback && (auth === 'none' || !tls)) { + const error = new Error( + `fail closed：非环回地址 ${host} 必须启用 TLS 且 auth != none（当前 auth=${auth}, tls=${tls}）`, + ); + error.code = 'MCP_INSECURE_CONFIG'; + throw error; + } + if (!isLoopback && tls && !tlsKey) { + const error = new Error('fail closed：TLS 需要提供证书（tlsKey/tlsCert）'); + error.code = 'MCP_INSECURE_CONFIG'; + throw error; + } + + const scheme = tls ? 'https' : 'http'; + // origin/issuer 在 listen 后按实际端口重算（支持 --port 0） + let origin = `${scheme}://${isLoopback ? '127.0.0.1' : host}:${port}`; + let issuer = process.env.MEBULAR_OAUTH_ISSUER ?? origin; + let metadata = metadataFor(issuer); + const signingKey = auth === 'oauth' ? await loadOrCreateSigningKey(home) : null; + const codes = new Map(); + const revoked = new Set(); + + // 单实例 transport（stateful + JSON 响应） + const mcpServer = buildServer(service); + const transport = new WebStandardStreamableHTTPServerTransport({ + sessionIdGenerator: () => randomUUID(), + enableJsonResponse: true, + }); + await mcpServer.connect(transport); + + async function authenticate(req, body) { + if (auth === 'none') return { ok: true }; + const header = req.headers['authorization']; + if (typeof header !== 'string' || !header.startsWith('Bearer ')) { + return { ok: false, status: 401, message: 'missing bearer token', challenge: 'Bearer' }; + } + const token = header.slice('Bearer '.length); + let grant = null; + if (auth === 'bearer') { + grant = await verifyBearer(token, tokensFile); + } else { + const payload = await verifyJwt(signingKey, token, { issuer, resource: `${issuer}/mcp`, revoked }); + if (payload && payload.type === 'access') grant = { tokenId: payload.jti, scopes: payload.scope ?? [] }; + } + if (!grant) { + return { ok: false, status: 401, message: 'invalid token', challenge: 'Bearer error="invalid_token"' }; + } + const required = requiredScopeForBody(body); + if (!hasScope(grant.scopes, required)) { + return { ok: false, status: 403, message: `insufficient scope: need ${required}`, challenge: `Bearer error="insufficient_scope", scope="${required}"`, tokenId: grant.tokenId }; + } + return { ok: true, tokenId: grant.tokenId, scopes: grant.scopes }; + } + + const handler = async (req, res) => { + try { + const url = new URL(req.url, origin); + const path = url.pathname; + const body = req.method === 'POST' || req.method === 'PUT' ? await readBody(req) : Buffer.alloc(0); + + if (path === '/healthz' && req.method === 'GET') { + return sendJson(res, 200, { status: 'ok', name: 'mebular', version: '0.1.0', auth, tls }); + } + if (auth === 'oauth') { + if (path === '/.well-known/oauth-protected-resource' && req.method === 'GET') { + return sendJson(res, 200, metadata.protectedResource); + } + if (path === '/.well-known/oauth-authorization-server' && req.method === 'GET') { + return sendJson(res, 200, metadata.authorizationServer); + } + if (path === '/jwks' && req.method === 'GET') { + return sendJson(res, 200, { + keys: [{ kty: 'OKP', crv: 'Ed25519', x: b64url(signingKey.publicRaw), kid: signingKey.kid, use: 'sig', alg: 'EdDSA' }], + }); + } + if (path === '/register' && req.method === 'POST') { + return sendJson(res, 201, { client_id: randomUUID(), token_endpoint_auth_method: 'none', grant_types: ['authorization_code', 'refresh_token'], response_types: ['code'] }); + } + if (path === '/authorize' && (req.method === 'GET' || req.method === 'POST')) { + const params = req.method === 'GET' ? url.searchParams : new URLSearchParams(body.toString('utf-8')); + const redirectUri = params.get('redirect_uri'); + const codeChallenge = params.get('code_challenge'); + const method = params.get('code_challenge_method'); + const state = params.get('state'); + if (!redirectUri || !codeChallenge || method !== 'S256') { + return sendJson(res, 400, { error: 'invalid_request', error_description: 'PKCE S256 required' }); + } + const code = randomUUID(); + codes.set(code, { + redirectUri, + codeChallenge, + scope: (params.get('scope') ?? 'memory.read').split(/\s+/).filter(Boolean), + exp: Date.now() + 300_000, + }); + const location = `${redirectUri}?code=${encodeURIComponent(code)}${state ? `&state=${encodeURIComponent(state)}` : ''}`; + res.statusCode = 302; + res.setHeader('location', location); + return res.end(); + } + if (path === '/token' && req.method === 'POST') { + const params = new URLSearchParams(body.toString('utf-8')); + const grantType = params.get('grant_type'); + const now = Math.floor(Date.now() / 1000); + let scope = ['memory.read']; + if (grantType === 'authorization_code') { + const record = codes.get(params.get('code') ?? ''); + if (!record || record.exp < Date.now()) return sendJson(res, 400, { error: 'invalid_grant' }); + const verifier = params.get('code_verifier') ?? ''; + const challenge = b64url(createHash('sha256').update(verifier).digest()); + if (challenge !== record.codeChallenge) return sendJson(res, 400, { error: 'invalid_grant', error_description: 'PKCE verification failed' }); + codes.delete(params.get('code')); + scope = record.scope; + } else if (grantType === 'refresh_token') { + const payload = await verifyJwt(signingKey, params.get('refresh_token') ?? '', { issuer, resource: `${issuer}/mcp`, revoked }); + if (!payload || payload.type !== 'refresh') return sendJson(res, 400, { error: 'invalid_grant' }); + revoked.add(payload.jti); // 轮换：旧 refresh 失效 + scope = payload.scope ?? ['memory.read']; + } else if (grantType === 'client_credentials') { + scope = (params.get('scope') ?? 'memory.read').split(/\s+/).filter((s) => SCOPES.includes(s)); + if (scope.length === 0) return sendJson(res, 400, { error: 'invalid_scope' }); + } else { + return sendJson(res, 400, { error: 'unsupported_grant_type' }); + } + const access = await signJwt(signingKey, { iss: issuer, aud: `${issuer}/mcp`, sub: 'user', scope, type: 'access', jti: randomUUID(), iat: now, exp: now + ACCESS_TTL }); + const refresh = await signJwt(signingKey, { iss: issuer, aud: `${issuer}/mcp`, sub: 'user', scope, type: 'refresh', jti: randomUUID(), iat: now, exp: now + REFRESH_TTL }); + return sendJson(res, 200, { access_token: access, token_type: 'Bearer', expires_in: ACCESS_TTL, refresh_token: refresh, scope: scope.join(' ') }); + } + if (path === '/token/revoke' && req.method === 'POST') { + const params = new URLSearchParams(body.toString('utf-8')); + const token = params.get('token') ?? ''; + const parts = token.split('.'); + if (parts.length === 3) { + try { + const payload = JSON.parse(Buffer.from(b64urlToBytes(parts[1]), 'utf8').toString('utf-8')); + if (payload.jti) revoked.add(payload.jti); + } catch { + // ignore malformed + } + } + return sendJson(res, 200, {}); + } + } + if (path === '/mcp') { + const check = await authenticate(req, body); + if (!check.ok) { + res.statusCode = check.status; + if (check.challenge) res.setHeader('www-authenticate', check.challenge); + return sendJson(res, check.status, { jsonrpc: '2.0', error: { code: -32001, message: check.message }, id: null }); + } + const webRequest = toWebRequest(req, body, origin); + const response = await transport.handleRequest(webRequest); + return writeWebResponse(res, response); + } + return sendJson(res, 404, { error: 'not_found', path }); + } catch (error) { + if (!res.headersSent) sendJson(res, 500, { error: 'internal_error', message: String(error?.message ?? error) }); + else res.end(); + } + }; + + let server; + if (tls) { + server = https.createServer({ key: await readFile(tlsKey, 'utf-8'), cert: await readFile(tlsCert, 'utf-8') }, handler); + } else { + server = http.createServer(handler); + } + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, host, () => resolve()); + }); + const actualPort = server.address().port; + origin = `${scheme}://${isLoopback ? '127.0.0.1' : host}:${actualPort}`; + issuer = process.env.MEBULAR_OAUTH_ISSUER ?? origin; + metadata = metadataFor(issuer); + const close = async () => { + await new Promise((resolve) => server.close(() => resolve())); + await mcpServer.close().catch(() => undefined); + }; + return { server, host, port: actualPort, auth, issuer, close }; +} diff --git a/packages/mcp/src/server.mjs b/packages/mcp/src/server.mjs index 5542897..2596122 100644 --- a/packages/mcp/src/server.mjs +++ b/packages/mcp/src/server.mjs @@ -6,6 +6,7 @@ import { serveStdio } from '@modelcontextprotocol/server/stdio'; import { MemoryService } from '@mebular/core'; import { createMebular } from './config.mjs'; import { registerTools } from './tools.mjs'; +import { startHttpServer, acquireLock } from './serve.mjs'; const MEMORY_POLICY = `# Mebular 记忆使用规约（memory_policy） 1. 先查后写：写入前先用 memory_query/memory_search 查重，避免重复。 @@ -15,7 +16,7 @@ const MEMORY_POLICY = `# Mebular 记忆使用规约（memory_policy） 5. 隐私：不要写入密钥、口令、完整身份证件等高敏感信息。 6. 无结果别编：召回为空就如实说明，不要臆造记忆。`; -function buildServer(service) { +export function buildServer(service) { const server = new McpServer({ name: 'mebular', version: '0.1.0' }); registerTools(server, service); server.registerPrompt( @@ -51,3 +52,41 @@ export async function startStdioServer() { process.on('SIGTERM', shutdown); return { handle, app, service }; } + +/** + * 启动 Streamable HTTP server（G6.3）。 + * 单实例：先取 /lock；被占抛 MCP_STORAGE_LOCKED。 + */ +export async function startServeServer(options = {}) { + const { app, home, storagePath } = await createMebular(); + const lock = await acquireLock(home, storagePath); + try { + const service = new MemoryService(app); + const http = await startHttpServer({ + home, + app, + service, + buildServer, + host: options.host, + port: options.port, + auth: options.auth, + tls: Boolean(options.tlsKey), + tlsKey: options.tlsKey, + tlsCert: options.tlsCert, + tokensFile: options.tokensFile, + }); + const shutdown = async () => { + await http.close().catch(() => undefined); + await lock.release(); + await app.shutdown().catch(() => undefined); + process.exit(0); + }; + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); + return { ...http, lock, app, service }; + } catch (error) { + await lock.release(); + await app.shutdown().catch(() => undefined); + throw error; + } +} diff --git a/packages/mcp/src/tools.mjs b/packages/mcp/src/tools.mjs index 98f8b52..f0288a6 100644 --- a/packages/mcp/src/tools.mjs +++ b/packages/mcp/src/tools.mjs @@ -5,6 +5,21 @@ import { z } from 'zod'; +/** 每个工具所需的 OAuth scope（D36） */ +export const TOOL_SCOPES = { + memory_write: 'memory.write', + memory_write_batch: 'memory.write', + memory_query: 'memory.read', + memory_search: 'memory.read', + memory_profile: 'memory.read', + memory_skills: 'memory.read', + memory_history: 'memory.read', + memory_graph: 'memory.read', + memory_import: 'memory.admin', + memory_status: 'memory.read', + memory_sync: 'memory.admin', +}; + export const TOOL_NAMES = [ 'memory_write', 'memory_write_batch', diff --git a/scripts/verify-mcp-http.mjs b/scripts/verify-mcp-http.mjs new file mode 100644 index 0000000..2d07ed4 --- /dev/null +++ b/scripts/verify-mcp-http.mjs @@ -0,0 +1,241 @@ +#!/usr/bin/env node +// G6.3 Streamable HTTP MCP 验证（真实 MCP client） +// +// 覆盖：POST /mcp、GET /healthz、OAuth well-known、bearer/scope 校验、 +// PKCE 授权码流程、单实例 lock 争用被拒、非环回无 TLS fail-closed。 +// 干净环境退出码 0。前置：npm run build（core dist）。 + +import { createHash, randomUUID } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Client } from '@modelcontextprotocol/client'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/client'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const rootDir = join(__dirname, '..'); +const bin = join(rootDir, 'packages', 'mcp', 'bin', 'mebular.mjs'); + +let passed = true; +const check = (label, ok, detail = '') => { + console.log(` ${ok ? '✓' : '✗'} ${label}${detail ? `（${detail}）` : ''}`); + if (!ok) passed = false; +}; +const base64url = (buf) => Buffer.from(buf).toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + +function spawnServe({ home, storage, args = [], env = {} }) { + const proc = spawn(process.execPath, [bin, 'serve', '--port', '0', ...args], { + env: { ...process.env, MEBULAR_HOME: home, MEBULAR_STORAGE_PATH: storage, MEBULAR_DEVICE_ID: 'device-http', ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let out = ''; + let err = ''; + proc.stdout.on('data', (d) => { out += d.toString(); }); + proc.stderr.on('data', (d) => { err += d.toString(); }); + return { proc, getOut: () => out, getErr: () => err }; +} + +function waitReady(handle, timeoutMs = 15000) { + return new Promise((resolve, reject) => { + const started = Date.now(); + const timer = setInterval(() => { + const m = handle.getOut().match(/SERVE_READY (\{.*\})/); + if (m) { + clearInterval(timer); + resolve(JSON.parse(m[1])); + } else if (Date.now() - started > timeoutMs) { + clearInterval(timer); + reject(new Error(`serve 未就绪：${handle.getErr()}`)); + } + }, 100); + }); +} + +function waitExit(handle, timeoutMs = 10000) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('serve 未在超时内退出')), timeoutMs); + handle.proc.on('exit', (code) => { + clearTimeout(timer); + resolve(code); + }); + }); +} + +async function stop(handle) { + if (handle?.proc && handle.proc.exitCode === null) { + handle.proc.kill('SIGTERM'); + await new Promise((r) => handle.proc.on('exit', r)); + } +} + +async function httpJson(url, init) { + const res = await fetch(url, init); + return { status: res.status, headers: res.headers, json: await res.json().catch(() => null), location: res.headers.get('location') }; +} + +async function mcpClient(port, headers = {}) { + const transport = new StreamableHTTPClientTransport(new URL(`http://127.0.0.1:${port}/mcp`), { + requestInit: { headers }, + }); + const client = new Client({ name: 'mebular-http-verify', version: '0.1.0' }); + await client.connect(transport); + return client; +} + +console.log('Mebular G6.3 HTTP MCP 验证'); +console.log('=========================='); + +const home = await mkdtemp(join(tmpdir(), 'mebular-mcp-http-')); +const servers = []; + +try { + // ---------- 1) auth=none：healthz + /mcp ---------- + { + const h = spawnServe({ home: join(home, 'none'), storage: join(home, 'none', 's.jsonl') }); + servers.push(h); + const ready = await waitReady(h); + check('serve 启动（SERVE_READY）', Number.isInteger(ready.port) && ready.port > 0, `port=${ready.port}`); + + const health = await httpJson(`http://127.0.0.1:${ready.port}/healthz`); + check('GET /healthz 200 + status ok', health.status === 200 && health.json?.status === 'ok'); + + const client = await mcpClient(ready.port); + const { tools } = await client.listTools(); + check('POST /mcp 真实 MCP client tools/list=11', tools.length === 11, `count=${tools.length}`); + const w = await client.callTool({ name: 'memory_write', arguments: { items: [{ type: 'fact', content: 'http-smoke' }] } }); + check('POST /mcp tools/call 落图', Array.isArray(w.structuredContent?.stored)); + await client.close(); + } + + // ---------- 2) bearer + scope ---------- + { + const bearerHome = join(home, 'bearer'); + const tokensFile = join(bearerHome, 'auth', 'tokens.json'); + const readToken = 'meb_readtoken'; + const writeToken = 'meb_writetoken'; + await mkdir(dirname(tokensFile), { recursive: true }); + await writeFile( + tokensFile, + JSON.stringify({ + tokens: [ + { id: 't-read', sha256: createHash('sha256').update(readToken).digest('hex'), scope: ['memory.read'], revoked: false }, + { id: 't-write', sha256: createHash('sha256').update(writeToken).digest('hex'), scope: ['memory.write'], revoked: false }, + ], + }), + 'utf-8', + ); + const h = spawnServe({ home: bearerHome, storage: join(bearerHome, 's.jsonl'), args: ['--auth', 'bearer', '--tokens-file', tokensFile] }); + servers.push(h); + const ready = await waitReady(h); + + const noToken = await httpJson(`http://127.0.0.1:${ready.port}/mcp`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }), + }); + check('无 token → 401', noToken.status === 401, `status=${noToken.status}`); + + const wrongScope = await httpJson(`http://127.0.0.1:${ready.port}/mcp`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${readToken}` }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'memory_write', arguments: { items: [] } } }), + }); + check('错误 scope（read 调 write）→ 403', wrongScope.status === 403, `status=${wrongScope.status}`); + + const client = await mcpClient(ready.port, { authorization: `Bearer ${readToken}` }); + const { tools } = await client.listTools(); + check('正确 read token → tools/list 成功', tools.length === 11); + await client.close(); + } + + // ---------- 3) oauth（well-known + PKCE） ---------- + { + const oauthHome = join(home, 'oauth'); + const h = spawnServe({ home: oauthHome, storage: join(oauthHome, 's.jsonl'), args: ['--auth', 'oauth'] }); + servers.push(h); + const ready = await waitReady(h); + const base = `http://127.0.0.1:${ready.port}`; + + const pr = await httpJson(`${base}/.well-known/oauth-protected-resource`); + check( + '/.well-known/oauth-protected-resource（resource 用实际端口）', + pr.status === 200 && pr.json?.resource === `${base}/mcp`, + pr.json?.resource, + ); + const as = await httpJson(`${base}/.well-known/oauth-authorization-server`); + check('/.well-known/oauth-authorization-server', as.status === 200 && Array.isArray(as.json?.code_challenge_methods_supported) && as.json.code_challenge_methods_supported.includes('S256')); + const jwks = await httpJson(`${base}/jwks`); + check('/jwks 提供 EdDSA 公钥', jwks.status === 200 && jwks.json?.keys?.[0]?.alg === 'EdDSA' && jwks.json.keys[0].crv === 'Ed25519'); + + // PKCE 授权码流程 + const verifier = randomUUID().replace(/-/g, '') + randomUUID().replace(/-/g, ''); + const challenge = base64url(createHash('sha256').update(verifier).digest()); + const redirectUri = 'http://127.0.0.1/callback'; + const authzRes = await fetch( + `${base}/authorize?response_type=code&client_id=verify-client&redirect_uri=${encodeURIComponent(redirectUri)}` + + `&code_challenge=${challenge}&code_challenge_method=S256&scope=${encodeURIComponent('memory.read memory.write')}&state=xyz`, + { redirect: 'manual' }, + ); + const location = authzRes.headers.get('location') ?? ''; + const code = new URL(location).searchParams.get('code'); + const state = new URL(location).searchParams.get('state'); + check('PKCE /authorize 302 带 code', authzRes.status === 302 && !!code && state === 'xyz', `status=${authzRes.status}`); + + const badVerifier = await httpJson(`${base}/token`, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ grant_type: 'authorization_code', code, redirect_uri: redirectUri, client_id: 'verify-client', code_verifier: 'wrong' }).toString(), + }); + check('PKCE 错误 verifier → invalid_grant', badVerifier.status === 400 && badVerifier.json?.error === 'invalid_grant'); + + const tokenRes = await httpJson(`${base}/token`, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ grant_type: 'authorization_code', code, redirect_uri: redirectUri, client_id: 'verify-client', code_verifier: verifier }).toString(), + }); + check('PKCE /token 颁发 access_token', tokenRes.status === 200 && typeof tokenRes.json?.access_token === 'string' && tokenRes.json?.token_type === 'Bearer'); + + const client = await mcpClient(ready.port, { authorization: `Bearer ${tokenRes.json.access_token}` }); + const { tools } = await client.listTools(); + check('oauth access token → /mcp tools/list 成功', tools.length === 11); + await client.close(); + } + + // ---------- 4) 单实例 lock 争用 ---------- + { + const lockHome = join(home, 'lock'); + const h1 = spawnServe({ home: lockHome, storage: join(lockHome, 's.jsonl') }); + servers.push(h1); + await waitReady(h1); + + const h2 = spawnServe({ home: lockHome, storage: join(lockHome, 's.jsonl') }); + const code = await waitExit(h2); + check('第二实例被拒（退出码非 0）', code !== 0, `code=${code}`); + check('错误含 MCP_STORAGE_LOCKED', h2.getErr().includes('MCP_STORAGE_LOCKED'), h2.getErr().split('\n')[0]); + } + + // ---------- 5) 非环回无 TLS fail-closed ---------- + { + const badHome = join(home, 'insecure'); + const h = spawnServe({ home: badHome, storage: join(badHome, 's.jsonl'), args: ['--host', '0.0.0.0', '--auth', 'none'] }); + const code = await waitExit(h); + check('非环回 + auth=none + 无 TLS → 拒绝启动', code !== 0, `code=${code}`); + check('错误含 MCP_INSECURE_CONFIG', h.getErr().includes('MCP_INSECURE_CONFIG')); + } +} catch (error) { + check('HTTP MCP 端到端', false, String(error?.message ?? error).substring(0, 400)); +} finally { + for (const s of servers) await stop(s); + await rm(home, { recursive: true, force: true }).catch(() => undefined); +} + +console.log('=========================='); +if (passed) { + console.log('✓ G6.3 HTTP MCP 验证通过'); + process.exit(0); +} else { + console.log('✗ G6.3 HTTP MCP 验证失败'); + process.exit(1); +}