Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/clients.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,5 @@ pub trait VaultClient: Send + Sync {
fn read_vault_note(&self, path: &str) -> Result<Value>;
fn get_backlinks(&self, note_id: &str) -> Result<Value>;
fn build_vault_graph(&self, repo_id: Option<&str>) -> Result<Value>;
fn export_vault(&self, output_dir: &str) -> Result<Value>;
}
14 changes: 14 additions & 0 deletions src/commands/knowledge.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2026 juice094
use devbase::clients::VaultClient;
use devbase::*;
use tracing::info;

Expand Down Expand Up @@ -134,6 +135,19 @@ pub async fn run_vault(
}
}
}
crate::VaultCommands::Export { output_dir } => {
let out = if output_dir.is_empty() {
format!("devbase-vault-export-{}", chrono::Local::now().format("%Y%m%d-%H%M%S"))
} else {
output_dir
};
let result = ctx.export_vault(&out)?;
println!("Vault exported to: {}", out);
println!(" Files: {}", result["exported_files"]);
println!(" Bytes: {}", result["total_bytes"]);
println!(" Broken links: {}", result["broken_links"]["count"]);
println!(" Frontmatter errors: {}", result["frontmatter_errors"]["count"]);
}
}
Ok(())
}
Expand Down
6 changes: 6 additions & 0 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,12 @@ pub(crate) enum VaultCommands {
#[arg(short, long, default_value_t = 20)]
limit: usize,
},
/// Export vault notes to a directory with integrity validation
Export {
/// Output directory for the exported vault
#[arg(default_value = "")]
output_dir: String,
},
}

#[derive(Subcommand)]
Expand Down
6 changes: 6 additions & 0 deletions src/mcp/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ pub enum McpToolEnum {
VaultBacklinks(DevkitVaultBacklinksTool),
VaultDaily(DevkitVaultDailyTool),
VaultGraph(DevkitVaultGraphTool),
VaultExport(DevkitVaultExportTool),
ProjectContext(DevkitProjectContextTool),
ProjectBrief(DevkitProjectBriefTool),
ImpactAnalysis(DevkitImpactAnalysisTool),
Expand Down Expand Up @@ -167,6 +168,7 @@ impl McpToolEnum {
McpToolEnum::VaultBacklinks(_) => ToolTier::Beta,
McpToolEnum::VaultDaily(_) => ToolTier::Beta,
McpToolEnum::VaultGraph(_) => ToolTier::Beta,
McpToolEnum::VaultExport(_) => ToolTier::Beta,
McpToolEnum::NaturalLanguageQuery(_) => ToolTier::Beta,
McpToolEnum::GithubInfo(_) => ToolTier::Beta,
// Experimental: new, behavior may change, pending prod validation
Expand Down Expand Up @@ -244,6 +246,7 @@ impl McpTool for McpToolEnum {
McpToolEnum::VaultBacklinks(t) => t.name(),
McpToolEnum::VaultDaily(t) => t.name(),
McpToolEnum::VaultGraph(t) => t.name(),
McpToolEnum::VaultExport(t) => t.name(),
McpToolEnum::ProjectContext(t) => t.name(),
McpToolEnum::ProjectBrief(t) => t.name(),
McpToolEnum::ImpactAnalysis(t) => t.name(),
Expand Down Expand Up @@ -314,6 +317,7 @@ impl McpTool for McpToolEnum {
McpToolEnum::VaultBacklinks(t) => t.schema(),
McpToolEnum::VaultDaily(t) => t.schema(),
McpToolEnum::VaultGraph(t) => t.schema(),
McpToolEnum::VaultExport(t) => t.schema(),
McpToolEnum::ProjectContext(t) => t.schema(),
McpToolEnum::ProjectBrief(t) => t.schema(),
McpToolEnum::ImpactAnalysis(t) => t.schema(),
Expand Down Expand Up @@ -388,6 +392,7 @@ impl McpTool for McpToolEnum {
McpToolEnum::VaultBacklinks(t) => t.invoke(args, ctx).await,
McpToolEnum::VaultDaily(t) => t.invoke(args, ctx).await,
McpToolEnum::VaultGraph(t) => t.invoke(args, ctx).await,
McpToolEnum::VaultExport(t) => t.invoke(args, ctx).await,
McpToolEnum::ProjectContext(t) => t.invoke(args, ctx).await,
McpToolEnum::ProjectBrief(t) => t.invoke(args, ctx).await,
McpToolEnum::ImpactAnalysis(t) => t.invoke(args, ctx).await,
Expand Down Expand Up @@ -652,6 +657,7 @@ pub fn build_server_with_tiers(tiers: Option<&HashSet<ToolTier>>) -> McpServer {
McpToolEnum::VaultBacklinks(DevkitVaultBacklinksTool),
McpToolEnum::VaultDaily(DevkitVaultDailyTool),
McpToolEnum::VaultGraph(DevkitVaultGraphTool),
McpToolEnum::VaultExport(DevkitVaultExportTool),
McpToolEnum::ProjectContext(DevkitProjectContextTool),
McpToolEnum::ProjectBrief(DevkitProjectBriefTool),
McpToolEnum::ImpactAnalysis(DevkitImpactAnalysisTool),
Expand Down
3 changes: 2 additions & 1 deletion src/mcp/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ async fn test_tools_list() {
let (mut ctx, _tmp) = test_ctx();
let resp = server.handle_request(req, &mut ctx).await.unwrap();
let tools = resp.get("result").unwrap().get("tools").unwrap().as_array().unwrap();
assert_eq!(tools.len(), 65);
assert_eq!(tools.len(), 66);
let names: Vec<&str> = tools.iter().map(|t| t.get("name").unwrap().as_str().unwrap()).collect();
assert!(names.contains(&"devkit_index_health"));
assert!(names.contains(&"devkit_vault_export"));
assert!(names.contains(&"devkit_session_save"));
assert!(names.contains(&"devkit_session_list"));
assert!(names.contains(&"devkit_session_resume"));
Expand Down
76 changes: 66 additions & 10 deletions src/mcp/tools/index_health.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
// Copyright (c) 2026 juice094
//! MCP tool: devkit_index_health — Tantivy + SQLite 索引健康度诊断。

use crate::clients::KnowledgeClient;
use crate::mcp::McpTool;
use crate::registry::ENTITY_TYPE_REPO;
use crate::search::list_indexed_repo_ids_at;
use crate::search::{list_indexed_repo_ids_at, sync_index_to_db_at};
use crate::storage::AppContext;
use std::collections::HashSet;
use tantivy::{Index, ReloadPolicy};
Expand All @@ -31,20 +32,28 @@ Use this when:
- Before/after running devkit_index to verify consistency
- Troubleshooting "missing repo" or "orphan document" issues

Parameters: none (inspects all registered indexes automatically)."#,
Parameters:
- repair: If true, automatically delete orphan documents and re-index missing repos. Default false (read-only diagnosis)."#,
"inputSchema": {
"type": "object",
"properties": {}
"properties": {
"repair": {
"type": "boolean",
"description": "If true, automatically repair detected inconsistencies (delete orphans, re-index missing repos)",
"default": false
}
}
}
})
}

async fn invoke(
&self,
_args: serde_json::Value,
args: serde_json::Value,
ctx: &mut AppContext,
) -> anyhow::Result<serde_json::Value> {
run_index_health(ctx)
let repair = args.get("repair").and_then(|v| v.as_bool()).unwrap_or(false);
run_index_health(ctx, repair)
}
}

Expand Down Expand Up @@ -85,7 +94,7 @@ fn check_index_at(
Ok((schema_valid, num_docs))
}

pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result<serde_json::Value> {
pub fn run_index_health(ctx: &mut AppContext, repair: bool) -> anyhow::Result<serde_json::Value> {
let index_path = ctx.storage.index_path()?;
let symbol_index_path = ctx.storage.symbol_index_path()?;

Expand Down Expand Up @@ -120,7 +129,7 @@ pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result<serde_json::Valu
let recorded_orphans = orphan_rows.len();

// 5. Live consistency: Tantivy IDs vs SQLite IDs
let (live_orphans, missing_from_index) = {
let (live_orphans, missing_from_index, missing_paths) = {
let tantivy_ids: HashSet<String> = match list_indexed_repo_ids_at(&index_path) {
Ok(ids) => ids.into_iter().collect(),
Err(_) => HashSet::new(),
Expand All @@ -132,12 +141,49 @@ pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result<serde_json::Valu
};
let orphans = tantivy_ids.difference(&sqlite_ids).count();
let missing = sqlite_ids.difference(&tantivy_ids).count();
(orphans, missing)

// Collect local paths for missing repos (used by repair)
let mut missing_paths = Vec::new();
if repair && missing > 0 {
for repo_id in sqlite_ids.difference(&tantivy_ids) {
if let Ok(path) = conn.query_row(
"SELECT local_path FROM entities WHERE id = ?1",
[repo_id],
|row| row.get::<_, String>(0),
) {
missing_paths.push(path);
}
}
}
(orphans, missing, missing_paths)
};

drop(conn);
// 6. Repair actions (if requested)
let mut repaired_orphans = 0usize;
let mut reindexed = 0usize;
let mut reindex_failed = 0usize;

if repair {
// 6a. Delete orphan documents from Tantivy
repaired_orphans = sync_index_to_db_at(&index_path, &conn).unwrap_or(0);

drop(conn); // Release pool connection before reindexing

// 6b. Reindex missing repos one by one
for path in missing_paths {
match ctx.run_index(&path) {
Ok(_) => reindexed += 1,
Err(e) => {
tracing::warn!("Failed to reindex {}: {}", path, e);
reindex_failed += 1;
}
}
}
} else {
drop(conn);
}

// 6. Health score calculation
// 7. Health score calculation
let mut score = 100i64;
if !repo_schema_valid {
score = 0;
Expand Down Expand Up @@ -171,6 +217,16 @@ pub fn run_index_health(ctx: &mut AppContext) -> anyhow::Result<serde_json::Valu
"live_orphans": live_orphans,
"missing_from_index": missing_from_index,
"orphan_repo_ids": orphan_rows,
},
"repair_actions": if repair {
serde_json::json!({
"requested": true,
"orphans_deleted": repaired_orphans,
"missing_reindexed": reindexed,
"missing_failed": reindex_failed,
})
} else {
serde_json::Value::Null
}
}))
}
49 changes: 49 additions & 0 deletions src/mcp/tools/vault.rs
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,55 @@ Returns: JSON with nodes (id, title) and edges (source, target)."#,
}
}

#[derive(Clone)]
pub struct DevkitVaultExportTool;

impl McpTool for DevkitVaultExportTool {
fn name(&self) -> &'static str {
"devkit_vault_export"
}

fn schema(&self) -> serde_json::Value {
serde_json::json!({
"description": r#"Export the devbase Vault to a directory with integrity validation.

Copies all Markdown notes preserving PARA directory structure, validates wikilink targets, and checks frontmatter YAML parseability.

Use this when:
- Creating a backup of your knowledge base
- Migrating notes to Obsidian / Logseq / other Markdown tools
- Verifying vault integrity (broken links, malformed frontmatter)

Parameters:
- output_dir: Destination directory for the export (created if missing)

Returns: export statistics including file count, total bytes, broken links, and frontmatter errors."#,
"inputSchema": {
"type": "object",
"properties": {
"output_dir": {
"type": "string",
"description": "Destination directory for the exported vault"
}
},
"required": ["output_dir"]
}
})
}

async fn invoke(
&self,
args: serde_json::Value,
ctx: &mut crate::storage::AppContext,
) -> anyhow::Result<serde_json::Value> {
let output_dir = args
.get("output_dir")
.and_then(|v| v.as_str())
.context("Missing required argument: output_dir")?;
ctx.export_vault(output_dir)
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
101 changes: 101 additions & 0 deletions src/vault/export.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2026 juice094
//! Vault export — data freedom and vendor lock-in elimination.

use std::collections::HashSet;
use std::path::Path;

/// Export vault notes to an output directory with integrity validation.
///
/// - Copies all `.md` files preserving relative directory structure
/// - Validates wikilink targets exist (reports broken links)
/// - Validates frontmatter YAML is parseable
/// - Returns statistics and any integrity issues found
pub fn export_vault(vault_dir: &Path, output_dir: &Path) -> anyhow::Result<serde_json::Value> {
std::fs::create_dir_all(output_dir)?;

let mut exported = 0usize;
let mut bytes = 0usize;
let mut broken_links: Vec<serde_json::Value> = Vec::new();
let mut frontmatter_errors: Vec<serde_json::Value> = Vec::new();

// First pass: collect all note IDs for broken link detection
let mut all_note_ids = HashSet::new();
for entry in walkdir::WalkDir::new(vault_dir)
.follow_links(false)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
.filter(|e| e.path().extension().map(|ext| ext == "md").unwrap_or(false))
{
let rel = entry.path().strip_prefix(vault_dir).unwrap_or(entry.path());
let id = rel.to_string_lossy().replace('\\', "/");
all_note_ids.insert(id.clone());
// Also index by stem (without .md) for wikilink resolution
if let Some(stem) = id.strip_suffix(".md") {
all_note_ids.insert(stem.to_string());
}
}

// Second pass: copy and validate
for entry in walkdir::WalkDir::new(vault_dir)
.follow_links(false)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
{
let src = entry.path();
let rel = src.strip_prefix(vault_dir).unwrap_or(src);
let dst = output_dir.join(rel);

if let Some(parent) = dst.parent() {
std::fs::create_dir_all(parent)?;
}

if src.extension().map(|e| e == "md").unwrap_or(false) {
let content = std::fs::read_to_string(src)?;
bytes += content.len();

// Validate frontmatter
if crate::vault::frontmatter::extract_frontmatter(&content).is_none() {
frontmatter_errors.push(serde_json::json!({
"path": rel.to_string_lossy().replace('\\', "/"),
"error": "Failed to parse frontmatter",
}));
}

// Validate wikilinks
for link in crate::vault::wikilink::extract_wikilinks(&content) {
let target_normalized = link.target.replace('\\', "/");
if !all_note_ids.contains(&target_normalized) {
broken_links.push(serde_json::json!({
"source": rel.to_string_lossy().replace('\\', "/"),
"target": link.target,
}));
}
}

std::fs::write(&dst, content)?;
} else {
// Copy non-markdown assets as-is
std::fs::copy(src, dst)?;
}
exported += 1;
}

Ok(serde_json::json!({
"success": true,
"vault_dir": vault_dir.to_string_lossy(),
"output_dir": output_dir.to_string_lossy(),
"exported_files": exported,
"total_bytes": bytes,
"broken_links": {
"count": broken_links.len(),
"issues": broken_links,
},
"frontmatter_errors": {
"count": frontmatter_errors.len(),
"issues": frontmatter_errors,
},
}))
}
Loading
Loading