From a21ef333875ed7bda5a6ca78c7651de05b02eb00 Mon Sep 17 00:00:00 2001 From: juice094 <160722440+juice094@users.noreply.github.com> Date: Thu, 14 May 2026 12:23:29 +0800 Subject: [PATCH 1/2] feat(devkit_index_health): add --repair parameter for automatic fix When repair=true: - Deletes orphan Tantivy documents via sync_index_to_db_at - Re-indexes missing repos one by one via KnowledgeClient::run_index - Returns repair_actions summary (orphans_deleted, missing_reindexed, missing_failed) Safety: - Releases pool connection (drop(conn)) before reindexing to avoid r2d2 deadlock - Gracefully handles reindex failures per-repo without aborting entire repair Verification: cargo test --lib 446 passed, clippy clean --- src/mcp/tools/index_health.rs | 76 ++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/src/mcp/tools/index_health.rs b/src/mcp/tools/index_health.rs index 6597edc..1988fee 100644 --- a/src/mcp/tools/index_health.rs +++ b/src/mcp/tools/index_health.rs @@ -2,9 +2,10 @@ // Copyright (c) 2026 juice094 //! MCP tool: devkit_index_health — Tantivy + SQLite 索引健康度诊断。 +use crate::clients::KnowledgeClient; use crate::mcp::McpTool; use crate::registry::ENTITY_TYPE_REPO; -use crate::search::list_indexed_repo_ids_at; +use crate::search::{list_indexed_repo_ids_at, sync_index_to_db_at}; use crate::storage::AppContext; use std::collections::HashSet; use tantivy::{Index, ReloadPolicy}; @@ -31,20 +32,28 @@ Use this when: - Before/after running devkit_index to verify consistency - Troubleshooting "missing repo" or "orphan document" issues -Parameters: none (inspects all registered indexes automatically)."#, +Parameters: +- repair: If true, automatically delete orphan documents and re-index missing repos. Default false (read-only diagnosis)."#, "inputSchema": { "type": "object", - "properties": {} + "properties": { + "repair": { + "type": "boolean", + "description": "If true, automatically repair detected inconsistencies (delete orphans, re-index missing repos)", + "default": false + } + } } }) } async fn invoke( &self, - _args: serde_json::Value, + args: serde_json::Value, ctx: &mut AppContext, ) -> anyhow::Result { - run_index_health(ctx) + let repair = args.get("repair").and_then(|v| v.as_bool()).unwrap_or(false); + run_index_health(ctx, repair) } } @@ -85,7 +94,7 @@ fn check_index_at( Ok((schema_valid, num_docs)) } -pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result { +pub fn run_index_health(ctx: &mut AppContext, repair: bool) -> anyhow::Result { let index_path = ctx.storage.index_path()?; let symbol_index_path = ctx.storage.symbol_index_path()?; @@ -120,7 +129,7 @@ pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result = match list_indexed_repo_ids_at(&index_path) { Ok(ids) => ids.into_iter().collect(), Err(_) => HashSet::new(), @@ -132,12 +141,49 @@ pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result 0 { + for repo_id in sqlite_ids.difference(&tantivy_ids) { + if let Ok(path) = conn.query_row( + "SELECT local_path FROM entities WHERE id = ?1", + [repo_id], + |row| row.get::<_, String>(0), + ) { + missing_paths.push(path); + } + } + } + (orphans, missing, missing_paths) }; - drop(conn); + // 6. Repair actions (if requested) + let mut repaired_orphans = 0usize; + let mut reindexed = 0usize; + let mut reindex_failed = 0usize; + + if repair { + // 6a. Delete orphan documents from Tantivy + repaired_orphans = sync_index_to_db_at(&index_path, &conn).unwrap_or(0); + + drop(conn); // Release pool connection before reindexing + + // 6b. Reindex missing repos one by one + for path in missing_paths { + match ctx.run_index(&path) { + Ok(_) => reindexed += 1, + Err(e) => { + tracing::warn!("Failed to reindex {}: {}", path, e); + reindex_failed += 1; + } + } + } + } else { + drop(conn); + } - // 6. Health score calculation + // 7. Health score calculation let mut score = 100i64; if !repo_schema_valid { score = 0; @@ -171,6 +217,16 @@ pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result Date: Thu, 14 May 2026 12:33:28 +0800 Subject: [PATCH 2/2] feat(devkit_vault_export): vault export with integrity validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds devkit_vault_export MCP tool (66th tool) and CLI command: - Copies all .md files preserving PARA directory structure - Validates wikilink targets exist (reports broken links) - Validates frontmatter YAML parseability - Copies non-markdown assets as-is - Returns statistics: exported_files, total_bytes, broken_links, frontmatter_errors New module: src/vault/export.rs - Zero external dependencies beyond walkdir (already in tree) - Reuses devbase_vault_wikilink::extract_wikilinks and frontmatter::extract_frontmatter CLI usage: devbase vault export → auto-named dir with timestamp devbase vault export ./my-backup → explicit output directory Also adds export_vault() to VaultClient trait and AppContext impl. Verification: cargo test --all-targets 464 passed, clippy -D warnings clean --- src/clients.rs | 1 + src/commands/knowledge.rs | 14 ++++++ src/main.rs | 6 +++ src/mcp/mod.rs | 6 +++ src/mcp/tests.rs | 3 +- src/mcp/tools/vault.rs | 49 ++++++++++++++++++ src/vault/export.rs | 101 ++++++++++++++++++++++++++++++++++++++ src/vault/mod.rs | 7 +++ 8 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 src/vault/export.rs diff --git a/src/clients.rs b/src/clients.rs index 2b94c4a..e00f058 100644 --- a/src/clients.rs +++ b/src/clients.rs @@ -144,4 +144,5 @@ pub trait VaultClient: Send + Sync { fn read_vault_note(&self, path: &str) -> Result; fn get_backlinks(&self, note_id: &str) -> Result; fn build_vault_graph(&self, repo_id: Option<&str>) -> Result; + fn export_vault(&self, output_dir: &str) -> Result; } diff --git a/src/commands/knowledge.rs b/src/commands/knowledge.rs index 4258b42..8344758 100644 --- a/src/commands/knowledge.rs +++ b/src/commands/knowledge.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 juice094 +use devbase::clients::VaultClient; use devbase::*; use tracing::info; @@ -134,6 +135,19 @@ pub async fn run_vault( } } } + crate::VaultCommands::Export { output_dir } => { + let out = if output_dir.is_empty() { + format!("devbase-vault-export-{}", chrono::Local::now().format("%Y%m%d-%H%M%S")) + } else { + output_dir + }; + let result = ctx.export_vault(&out)?; + println!("Vault exported to: {}", out); + println!(" Files: {}", result["exported_files"]); + println!(" Bytes: {}", result["total_bytes"]); + println!(" Broken links: {}", result["broken_links"]["count"]); + println!(" Frontmatter errors: {}", result["frontmatter_errors"]["count"]); + } } Ok(()) } diff --git a/src/main.rs b/src/main.rs index 2de18dc..5f608d7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -508,6 +508,12 @@ pub(crate) enum VaultCommands { #[arg(short, long, default_value_t = 20)] limit: usize, }, + /// Export vault notes to a directory with integrity validation + Export { + /// Output directory for the exported vault + #[arg(default_value = "")] + output_dir: String, + }, } #[derive(Subcommand)] diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index a28f933..a01f9d6 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -79,6 +79,7 @@ pub enum McpToolEnum { VaultBacklinks(DevkitVaultBacklinksTool), VaultDaily(DevkitVaultDailyTool), VaultGraph(DevkitVaultGraphTool), + VaultExport(DevkitVaultExportTool), ProjectContext(DevkitProjectContextTool), ProjectBrief(DevkitProjectBriefTool), ImpactAnalysis(DevkitImpactAnalysisTool), @@ -167,6 +168,7 @@ impl McpToolEnum { McpToolEnum::VaultBacklinks(_) => ToolTier::Beta, McpToolEnum::VaultDaily(_) => ToolTier::Beta, McpToolEnum::VaultGraph(_) => ToolTier::Beta, + McpToolEnum::VaultExport(_) => ToolTier::Beta, McpToolEnum::NaturalLanguageQuery(_) => ToolTier::Beta, McpToolEnum::GithubInfo(_) => ToolTier::Beta, // Experimental: new, behavior may change, pending prod validation @@ -244,6 +246,7 @@ impl McpTool for McpToolEnum { McpToolEnum::VaultBacklinks(t) => t.name(), McpToolEnum::VaultDaily(t) => t.name(), McpToolEnum::VaultGraph(t) => t.name(), + McpToolEnum::VaultExport(t) => t.name(), McpToolEnum::ProjectContext(t) => t.name(), McpToolEnum::ProjectBrief(t) => t.name(), McpToolEnum::ImpactAnalysis(t) => t.name(), @@ -314,6 +317,7 @@ impl McpTool for McpToolEnum { McpToolEnum::VaultBacklinks(t) => t.schema(), McpToolEnum::VaultDaily(t) => t.schema(), McpToolEnum::VaultGraph(t) => t.schema(), + McpToolEnum::VaultExport(t) => t.schema(), McpToolEnum::ProjectContext(t) => t.schema(), McpToolEnum::ProjectBrief(t) => t.schema(), McpToolEnum::ImpactAnalysis(t) => t.schema(), @@ -388,6 +392,7 @@ impl McpTool for McpToolEnum { McpToolEnum::VaultBacklinks(t) => t.invoke(args, ctx).await, McpToolEnum::VaultDaily(t) => t.invoke(args, ctx).await, McpToolEnum::VaultGraph(t) => t.invoke(args, ctx).await, + McpToolEnum::VaultExport(t) => t.invoke(args, ctx).await, McpToolEnum::ProjectContext(t) => t.invoke(args, ctx).await, McpToolEnum::ProjectBrief(t) => t.invoke(args, ctx).await, McpToolEnum::ImpactAnalysis(t) => t.invoke(args, ctx).await, @@ -652,6 +657,7 @@ pub fn build_server_with_tiers(tiers: Option<&HashSet>) -> McpServer { McpToolEnum::VaultBacklinks(DevkitVaultBacklinksTool), McpToolEnum::VaultDaily(DevkitVaultDailyTool), McpToolEnum::VaultGraph(DevkitVaultGraphTool), + McpToolEnum::VaultExport(DevkitVaultExportTool), McpToolEnum::ProjectContext(DevkitProjectContextTool), McpToolEnum::ProjectBrief(DevkitProjectBriefTool), McpToolEnum::ImpactAnalysis(DevkitImpactAnalysisTool), diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index f5e757d..d8c0807 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -39,9 +39,10 @@ async fn test_tools_list() { let (mut ctx, _tmp) = test_ctx(); let resp = server.handle_request(req, &mut ctx).await.unwrap(); let tools = resp.get("result").unwrap().get("tools").unwrap().as_array().unwrap(); - assert_eq!(tools.len(), 65); + assert_eq!(tools.len(), 66); let names: Vec<&str> = tools.iter().map(|t| t.get("name").unwrap().as_str().unwrap()).collect(); assert!(names.contains(&"devkit_index_health")); + assert!(names.contains(&"devkit_vault_export")); assert!(names.contains(&"devkit_session_save")); assert!(names.contains(&"devkit_session_list")); assert!(names.contains(&"devkit_session_resume")); diff --git a/src/mcp/tools/vault.rs b/src/mcp/tools/vault.rs index f2461de..0be40ff 100644 --- a/src/mcp/tools/vault.rs +++ b/src/mcp/tools/vault.rs @@ -460,6 +460,55 @@ Returns: JSON with nodes (id, title) and edges (source, target)."#, } } +#[derive(Clone)] +pub struct DevkitVaultExportTool; + +impl McpTool for DevkitVaultExportTool { + fn name(&self) -> &'static str { + "devkit_vault_export" + } + + fn schema(&self) -> serde_json::Value { + serde_json::json!({ + "description": r#"Export the devbase Vault to a directory with integrity validation. + +Copies all Markdown notes preserving PARA directory structure, validates wikilink targets, and checks frontmatter YAML parseability. + +Use this when: +- Creating a backup of your knowledge base +- Migrating notes to Obsidian / Logseq / other Markdown tools +- Verifying vault integrity (broken links, malformed frontmatter) + +Parameters: +- output_dir: Destination directory for the export (created if missing) + +Returns: export statistics including file count, total bytes, broken links, and frontmatter errors."#, + "inputSchema": { + "type": "object", + "properties": { + "output_dir": { + "type": "string", + "description": "Destination directory for the exported vault" + } + }, + "required": ["output_dir"] + } + }) + } + + async fn invoke( + &self, + args: serde_json::Value, + ctx: &mut crate::storage::AppContext, + ) -> anyhow::Result { + let output_dir = args + .get("output_dir") + .and_then(|v| v.as_str()) + .context("Missing required argument: output_dir")?; + ctx.export_vault(output_dir) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/vault/export.rs b/src/vault/export.rs new file mode 100644 index 0000000..c6b0735 --- /dev/null +++ b/src/vault/export.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 juice094 +//! Vault export — data freedom and vendor lock-in elimination. + +use std::collections::HashSet; +use std::path::Path; + +/// Export vault notes to an output directory with integrity validation. +/// +/// - Copies all `.md` files preserving relative directory structure +/// - Validates wikilink targets exist (reports broken links) +/// - Validates frontmatter YAML is parseable +/// - Returns statistics and any integrity issues found +pub fn export_vault(vault_dir: &Path, output_dir: &Path) -> anyhow::Result { + std::fs::create_dir_all(output_dir)?; + + let mut exported = 0usize; + let mut bytes = 0usize; + let mut broken_links: Vec = Vec::new(); + let mut frontmatter_errors: Vec = Vec::new(); + + // First pass: collect all note IDs for broken link detection + let mut all_note_ids = HashSet::new(); + for entry in walkdir::WalkDir::new(vault_dir) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .filter(|e| e.path().extension().map(|ext| ext == "md").unwrap_or(false)) + { + let rel = entry.path().strip_prefix(vault_dir).unwrap_or(entry.path()); + let id = rel.to_string_lossy().replace('\\', "/"); + all_note_ids.insert(id.clone()); + // Also index by stem (without .md) for wikilink resolution + if let Some(stem) = id.strip_suffix(".md") { + all_note_ids.insert(stem.to_string()); + } + } + + // Second pass: copy and validate + for entry in walkdir::WalkDir::new(vault_dir) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + { + let src = entry.path(); + let rel = src.strip_prefix(vault_dir).unwrap_or(src); + let dst = output_dir.join(rel); + + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + + if src.extension().map(|e| e == "md").unwrap_or(false) { + let content = std::fs::read_to_string(src)?; + bytes += content.len(); + + // Validate frontmatter + if crate::vault::frontmatter::extract_frontmatter(&content).is_none() { + frontmatter_errors.push(serde_json::json!({ + "path": rel.to_string_lossy().replace('\\', "/"), + "error": "Failed to parse frontmatter", + })); + } + + // Validate wikilinks + for link in crate::vault::wikilink::extract_wikilinks(&content) { + let target_normalized = link.target.replace('\\', "/"); + if !all_note_ids.contains(&target_normalized) { + broken_links.push(serde_json::json!({ + "source": rel.to_string_lossy().replace('\\', "/"), + "target": link.target, + })); + } + } + + std::fs::write(&dst, content)?; + } else { + // Copy non-markdown assets as-is + std::fs::copy(src, dst)?; + } + exported += 1; + } + + Ok(serde_json::json!({ + "success": true, + "vault_dir": vault_dir.to_string_lossy(), + "output_dir": output_dir.to_string_lossy(), + "exported_files": exported, + "total_bytes": bytes, + "broken_links": { + "count": broken_links.len(), + "issues": broken_links, + }, + "frontmatter_errors": { + "count": frontmatter_errors.len(), + "issues": frontmatter_errors, + }, + })) +} diff --git a/src/vault/mod.rs b/src/vault/mod.rs index 91e130a..a09ebcf 100644 --- a/src/vault/mod.rs +++ b/src/vault/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 juice094 pub mod backlinks; +pub mod export; pub mod frontmatter; pub mod fs_io; pub mod indexer; @@ -150,4 +151,10 @@ impl crate::clients::VaultClient for AppContext { "edges": edges, })) } + + fn export_vault(&self, output_dir: &str) -> anyhow::Result { + let vault_dir = self.storage.workspace_dir()?.join("vault"); + let out = std::path::PathBuf::from(output_dir); + crate::vault::export::export_vault(&vault_dir, &out) + } }