Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions universal-refiner/.gemini-refiner.example.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"_comment_usage": "Legacy example only. Prefer .universal-refiner.example.json and copy that file to .universal-refiner.json. This file is safe to commit.",
"_comment_fields": "All fields are optional. Omit a section to use built-in defaults.",
"semantic": {
"_comment": "Configure the local OpenAI-compatible provider (Ollama, LM Studio, etc.)",
"localEnabled": true,
"baseUrl": "http://localhost:11434/v1",
"models": ["gemma3:12b", "gemma3"],
"mcpSamplingEnabled": true,
"timeoutMs": 120000,
"temperature": 0.2
}
}
429 changes: 421 additions & 8 deletions universal-refiner/package-lock.json

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion universal-refiner/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,10 @@
"dependencies": {
"@hono/node-server": "^1.19.13",
"@modelcontextprotocol/sdk": "^1.29.0",
"@lancedb/lancedb": "^0.30.0",
"better-sqlite3": "^12.8.0",
"chokidar": "^5.0.0",
"flexsearch": "^0.7.43",
"flexsearch": "^0.8.212",
"hono": "^4.12.25",
"typescript": "^5.9.3",
"zod": "^4.3.6"
Expand Down
77 changes: 77 additions & 0 deletions universal-refiner/scripts/obsidian-sync.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import { EventStore } from "../src/history/event-store.js";
import { ObsidianOrchestrator } from "../src/integrations/obsidian/obsidian-orchestrator.js";
import { ConfigManager } from "../src/core/config.js";
import * as path from "path";
import * as fs from "fs";

/**
* Historical Migration Script
* Sweeps the EventStore (events.db) and pushes all old session data to the Obsidian Vault.
*/
async function migrate() {
console.log("Starting Historical Migration to Obsidian...");

// Force the orchestrator to use the global obsidian vault
ConfigManager.getObsidianConfig = () => ({ vaultPath: "C:\\repo\\global.obsidian" });

const store = EventStore.getInstance();
const db = (store as any).db;

// 1. Get all unique projects (repo_ids)
const repos = db.prepare("SELECT DISTINCT repo_id FROM prompts WHERE repo_id IS NOT NULL").all() as { repo_id: string }[];
console.log(`Found ${repos.length} projects with history.`);

for (const { repo_id } of repos) {
console.log(`\nMigrating project: ${repo_id}...`);

// Simulate a rootPath for the orchestrator (it uses basename(rootPath) as repoId)
// We'll use a dummy path that ends with the repo_id
const dummyRootPath = `C:\\repo\\${repo_id}`;

// 2. Fetch all successful executions for this repo
const executions = db.prepare(`
SELECT p.raw_prompt, e.result_summary, e.ended_at, e.executor_name
FROM prompts p
JOIN executions e ON p.id = e.prompt_id
WHERE p.repo_id = ? AND e.status = 'completed'
ORDER BY e.ended_at ASC
`).all(repo_id) as any[];

console.log(`- Found ${executions.length} historical executions.`);

for (const exec of executions) {
const summary = `Historical: ${exec.result_summary || "Agent execution"}`;
const rationale = `Prompt: ${exec.raw_prompt}\n\nExecutor: ${exec.executor_name}\nDate: ${new Date(exec.ended_at).toLocaleString()}`;

await ObsidianOrchestrator.logActivity(dummyRootPath, summary, rationale);
}

// 3. Fetch all commits for this repo
const commits = db.prepare(`
SELECT message, committed_at, author, sha
FROM commits
WHERE repo_id = ?
ORDER BY committed_at ASC
`).all(repo_id) as any[];

console.log(`- Found ${commits.length} historical commits.`);

for (const commit of commits) {
const summary = `Historical Commit: ${commit.sha.substring(0, 7)} - ${commit.message}`;
const rationale = `Author: ${commit.author}\nDate: ${new Date(commit.committed_at).toLocaleString()}`;

await ObsidianOrchestrator.logActivity(dummyRootPath, summary, rationale);
}

// 4. Sync lessons (Engineering Mandates)
await ObsidianOrchestrator.syncToWiki(dummyRootPath);
}

console.log("\nMigration Complete!");
process.exit(0);
}

migrate().catch(err => {
console.error("Migration failed:", err);
process.exit(1);
});
44 changes: 43 additions & 1 deletion universal-refiner/src/core/background-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ import { AutoPilotStatus } from "./autopilot-status.js";
import { RuntimeLogger } from "./logger.js";
import { CommandCenterDashboard } from "./dashboard.js";
import { SerializedJobQueue } from "./job-queue.js";
import { ExecutionOrchestrator } from "./execution-orchestrator.js";
import { EventStore } from "../history/event-store.js";
import { ConfigManager } from "./config.js";

export class BackgroundAutonomyService {
private watcher: chokidar.FSWatcher | null = null;
Expand Down Expand Up @@ -100,14 +103,21 @@ export class BackgroundAutonomyService {
const engine = new CorrelationEngine();
const extractor = new LessonExtractor(this.requestModelText);
const lessonsBefore = AutoPilotStatus.getSnapshot().stats.lessonsExtracted;
const orchestrator = new ExecutionOrchestrator(EventStore.getInstance(), this.requestModelText);

const results = await Promise.allSettled([
engine.correlateAll(),
extractor.extractNewLessons(),
extractor.extractFailureLessons(),
]);

const rejected = results.find((result): result is PromiseRejectedResult => result.status === "rejected");
if (rejected) {
throw rejected.reason;
}

await this.attemptSelfHealing(orchestrator);

const lessonsAfter = AutoPilotStatus.getSnapshot().stats.lessonsExtracted;
if (lessonsAfter > lessonsBefore) {
AutoPilotStatus.record(`Extracted ${lessonsAfter - lessonsBefore} lesson(s)`, "lesson");
Expand All @@ -116,7 +126,9 @@ export class BackgroundAutonomyService {
AutoPilotStatus.incrementCycles();
AutoPilotStatus.setActive();
AutoPilotStatus.record("Cycle complete", "cycle_complete");
CommandCenterDashboard.log("Background Autonomy: Correlation and lesson extraction complete.");
CommandCenterDashboard.log("Background Autonomy: Correlation, lesson extraction, and self-healing complete.");

await this.syncObsidian();

} catch (error) {
AutoPilotStatus.setIdle();
Expand All @@ -127,6 +139,36 @@ export class BackgroundAutonomyService {
}
}

private async attemptSelfHealing(orchestrator: ExecutionOrchestrator) {
try {
const lessons = EventStore.getInstance().getApprovedLessonsWithExecutions(10);

for (const lesson of lessons) {
// Heal and retry
await orchestrator.healAndRetry(lesson.execution_id, lesson.id);
}
} catch (error) {
RuntimeLogger.error("Self-healing failed", error);
}
}

private async syncObsidian() {
const obsidianConfig = ConfigManager.getObsidianConfig(this.rootPath);
if (!obsidianConfig?.syncLessons) {
return;
}

try {
const { ObsidianOrchestrator } = await import("../integrations/obsidian/obsidian-orchestrator.js");
await ObsidianOrchestrator.syncToWiki(this.rootPath);
CommandCenterDashboard.log("Background Autonomy: Synced to Obsidian Vault.");
AutoPilotStatus.record("Synced to Obsidian Vault", "cycle_complete");
} catch (syncError) {
RuntimeLogger.error("Failed to sync to Obsidian", syncError);
CommandCenterDashboard.log("Background Autonomy: Failed to sync to Obsidian Vault.");
}
}

public stop() {
if (this.watcher) {
this.watcher.close();
Expand Down
10 changes: 10 additions & 0 deletions universal-refiner/src/core/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ export interface RefinerConfig {
mandates?: string[];
ignoredPaths?: string[];
semantic?: Partial<SemanticConfig>;
atlassian?: any;
obsidian?: any;
}

export interface SemanticConfig {
Expand Down Expand Up @@ -88,6 +90,14 @@ export class ConfigManager {
};
}

static getAtlassianConfig(rootPath: string = "."): any | null {
return this.loadConfig(rootPath).atlassian || null;
}

static getObsidianConfig(rootPath: string = "."): any | null {
return this.loadConfig(rootPath).obsidian || null;
}

static getPredictiveMandates(): string[] {
const logs = AgenticBlackboard.getLogs();
const recent = logs.slice(0, 10).map(l => l.message.toLowerCase());
Expand Down
58 changes: 49 additions & 9 deletions universal-refiner/src/core/dashboard.html
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,10 @@ <h2>Provider Metrics</h2>
const escapeHtml = (v) => String(v ?? '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
const encodeCandidateId = (v) => encodeURIComponent(String(v)).replace(/'/g, '%27');
const projectName = (p) => p.split(/[\\\/]/).filter(Boolean).pop() || 'ROOT';
const selectProject = (encodedProject) => {
currentProject = decodeURIComponent(encodedProject);
refreshData();
};

function switchView(viewId) {
document.querySelectorAll('.main-view').forEach(v => v.classList.add('hidden'));
Expand All @@ -199,13 +203,13 @@ <h2>Provider Metrics</h2>

document.getElementById('project-badge').textContent = projectName(currentProject).toUpperCase();
document.getElementById('project-dna').innerHTML = `
<div class="stat-row"><span class="stat-label">STACK</span><span>${state.stack}</span></div>
<div class="stat-row"><span class="stat-label">FRAMEWORK</span><span>${state.framework}</span></div>
<div class="stat-row"><span class="stat-label">PATTERN</span><span>${state.pattern}</span></div>
<div class="stat-row"><span class="stat-label">STACK</span><span>${escapeHtml(state.stack)}</span></div>
<div class="stat-row"><span class="stat-label">FRAMEWORK</span><span>${escapeHtml(state.framework)}</span></div>
<div class="stat-row"><span class="stat-label">PATTERN</span><span>${escapeHtml(state.pattern)}</span></div>
`;

document.getElementById('project-list').innerHTML = state.projects.map(p =>
`<div class="nav-item ${p === currentProject ? 'active' : ''}" onclick="currentProject='${p}';refreshData();" style="font-size: 0.7rem; margin-left: 0.5rem;">📂 ${projectName(p)}</div>`
`<div class="nav-item ${p === currentProject ? 'active' : ''}" onclick="selectProject('${encodeCandidateId(p)}')" style="font-size: 0.7rem; margin-left: 0.5rem;">📂 ${escapeHtml(projectName(p))}</div>`
).join('');

// 2. View Specific Refresh
Expand All @@ -214,14 +218,46 @@ <h2>Provider Metrics</h2>
const data = await res.json();
document.getElementById('timeline-terminal').innerHTML = data.map(e => {
let icon = 'EVT', color = 'var(--text)';
if (e.type === 'prompt') { icon = 'PRM'; color = 'var(--accent)'; }
let extraHtml = '';

if (e.type === 'prompt') {
icon = 'PRM'; color = 'var(--accent)';
if (e.details && e.details.intent === 'self-heal') {
icon = 'HEAL'; color = '#10b981';
extraHtml += `<div class="badge" style="background: rgba(16, 185, 129, 0.15); color: #6ee7b7; border-color: rgba(16, 185, 129, 0.3); margin-top: 8px; display: inline-block;">AUTONOMOUS SELF-HEALING RETRY</div>`;
}
if (e.details && e.details.normalized_prompt) {
extraHtml += `<div style="margin-top: 8px; padding: 10px; background: rgba(56, 189, 248, 0.05); border-left: 2px solid var(--accent); border-radius: 0 4px 4px 0; font-size: 0.75rem; white-space: pre-wrap; color: var(--dim);"><strong>IMPROVED PROMPT:</strong><br/>${escapeHtml(e.details.normalized_prompt)}</div>`;
}
}
else if (e.type === 'commit') { icon = 'GIT'; color = '#f472b6'; }
else if (e.type === 'log') { icon = 'LOG'; color = 'var(--dim)'; }
else if (e.type === 'execution') {
icon = 'EXEC';
color = e.event_type === 'failed' ? '#ef4444' : '#22c55e';
let execBadge = e.event_type === 'failed'
? `<span class="badge" style="background: rgba(239, 68, 68, 0.15); color: #fca5a5; border-color: rgba(239, 68, 68, 0.3);">AI ERROR</span>`
: `<span class="badge" style="background: rgba(34, 197, 94, 0.15); color: #86efac; border-color: rgba(34, 197, 94, 0.3);">SUCCESS</span>`;

extraHtml = `<div style="margin-top: 8px;">
${execBadge} <span style="color: var(--dim); font-size: 0.7rem; margin-left: 6px;">EXECUTOR: ${escapeHtml(e.author || 'unknown')}</span>
</div>`;

if (e.event_type === 'failed' && e.details && e.details.error) {
extraHtml += `<div style="margin-top: 8px; padding: 10px; background: rgba(239, 68, 68, 0.05); border-left: 2px solid #ef4444; border-radius: 0 4px 4px 0; font-size: 0.75rem; white-space: pre-wrap; color: #fca5a5; font-family: monospace;"><strong>RAW ERROR:</strong><br/>${escapeHtml(typeof e.details.error === 'object' ? JSON.stringify(e.details.error, null, 2) : String(e.details.error))}</div>`;
} else if (e.event_type === 'completed' && e.details && e.details.healedResponse) {
extraHtml += `<div style="margin-top: 8px; padding: 10px; background: rgba(34, 197, 94, 0.05); border-left: 2px solid #22c55e; border-radius: 0 4px 4px 0; font-size: 0.75rem; white-space: pre-wrap; color: #86efac;"><strong>HEALED RESPONSE:</strong><br/>${escapeHtml(e.details.healedResponse)}</div>`;
}
}

return `
<div class="log-line">
<span class="log-ts">${new Date(e.timestamp).toLocaleTimeString()}</span>
<span class="log-icon" style="color:${color}">${icon}</span>
<span>${escapeHtml(e.summary)}</span>
<div style="flex-grow: 1;">
<div style="white-space: pre-wrap; font-family: monospace; font-size: 0.85rem;">${escapeHtml(e.summary)}</div>
${extraHtml}
</div>
</div>
`;
}).join('');
Expand All @@ -235,14 +271,14 @@ <h2>Provider Metrics</h2>
<div style="display:flex; justify-content:space-between; align-items:flex-start;">
<div>
<code style="color:var(--accent)">${c.sha.substring(0, 7)}</code> <strong>${escapeHtml(c.message)}</strong>
<div style="font-size:0.7rem; color:var(--dim); margin-top:4px;">by ${c.author} • ${new Date(c.committed_at).toLocaleString()}</div>
<div style="font-size:0.7rem; color:var(--dim); margin-top:4px;">by ${escapeHtml(c.author)} • ${new Date(c.committed_at).toLocaleString()}</div>
</div>
<div style="text-align:right">
<span class="diff-add">+${JSON.parse(c.diff_stats_json).insertions || 0}</span>
<span class="diff-del">-${JSON.parse(c.diff_stats_json).deletions || 0}</span>
</div>
</div>
${c.prompt_id ? `<div class="badge" style="margin-top:10px; display:inline-block">Linked to Prompt: ${c.prompt_id}</div>` : ''}
${c.prompt_id ? `<div class="badge" style="margin-top:10px; display:inline-block">Linked to Prompt: ${escapeHtml(c.prompt_id)}</div>` : ''}
</div>
`).join('') || '<p style="color:var(--dim)">No commits ingested for this project.</p>';
}
Expand Down Expand Up @@ -328,9 +364,13 @@ <h2>Provider Metrics</h2>
}
}

// Auto-poll every 3 seconds so the user never has to click anything
setInterval(() => {
refreshData().catch(err => console.error("Auto-refresh failed:", err));
}, 3000);

// Initial Load
refreshData();
setInterval(refreshData, 10000);
</script>
</body>
</html>
34 changes: 29 additions & 5 deletions universal-refiner/src/core/dashboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@ import { AutoPilotStatus } from "./autopilot-status.js";
const __dirname = path.dirname(fileURLToPath(import.meta.url));

export function resolveDashboardHost(configuredHost = process.env.PROMPT_REFINER_DASHBOARD_HOST): string {
return configuredHost?.trim() || "127.0.0.1";
const host = configuredHost?.trim() || "127.0.0.1";
if (isLoopbackHost(host) || process.env.PROMPT_REFINER_DASHBOARD_ALLOW_REMOTE === "true") {
return host;
}
RuntimeLogger.warn("Ignoring non-loopback dashboard host without PROMPT_REFINER_DASHBOARD_ALLOW_REMOTE=true", { host });
return "127.0.0.1";
}

interface DashboardState {
Expand Down Expand Up @@ -60,7 +65,7 @@ function sanitizeEndpoint(rawUrl: string): string {

export function isSameOriginRequest(origin: string | undefined, requestUrl: string): boolean {
if (!origin) {
return true;
return false;
}

try {
Expand All @@ -70,6 +75,23 @@ export function isSameOriginRequest(origin: string | undefined, requestUrl: stri
}
}

export function isJsonContentType(contentType: string | undefined): boolean {
const mediaType = contentType?.split(";")[0]?.trim().toLowerCase();
return mediaType === "application/json" || Boolean(mediaType?.endsWith("+json"));
}

function isLoopbackHost(host: string): boolean {
const normalized = host.toLowerCase();
return normalized === "localhost" || normalized === "127.0.0.1" || normalized === "::1" || normalized === "[::1]";
}

function redactSensitive(value: string): string {
return value
.replace(/(ghp_|github_pat_|sk-|xox[baprs]-)[a-z0-9_\-]+/gi, "$1[REDACTED]")
.replace(/(token|api[_-]?key|password|secret|authorization)\s*[:=]\s*["']?[^"'\s,;]+/gi, "$1=[REDACTED]")
.slice(0, 2_000);
}

export class CommandCenterDashboard {
private static rootPath: string = ".";
private static server: { close: (callback?: (error?: Error) => void) => void } | null = null;
Expand All @@ -92,7 +114,7 @@ export class CommandCenterDashboard {
}

private static logRouteError(routeName: string, error: unknown, selectedPath?: string) {
const message = error instanceof Error ? error.stack || error.message : String(error);
const message = redactSensitive(error instanceof Error ? error.stack || error.message : String(error));
RuntimeLogger.error(`Dashboard route failed: ${routeName}`, {
selectedPath: selectedPath || this.rootPath,
error: message,
Expand Down Expand Up @@ -241,8 +263,10 @@ export class CommandCenterDashboard {

app.get("/api/timeline", async (c) => {
try {
const store = EventStore.getInstance();
const repoId = store.ensureRepository(this.resolveSelectedPath(c.req.query("project"))).id;
const provider = new TimelineProvider();
const timeline = provider.getUnifiedTimeline(50);
const timeline = provider.getUnifiedTimeline(50, repoId);
Comment on lines +267 to +269

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve the global timeline when no project is selected

When the dashboard calls this endpoint it still uses fetch('/api/timeline') with no project query, so resolveSelectedPath(...) falls back to this.rootPath and this new repoId is always passed to getUnifiedTimeline. In a dashboard with multiple visible projects, the “Global Intelligence Stream” now omits all non-root project activity instead of showing the unified timeline; only callers that add ?project= get scoped data.

Useful? React with 👍 / 👎.

return c.json(timeline);
} catch (error) {
this.logRouteError("api/timeline", error);
Expand Down Expand Up @@ -302,7 +326,7 @@ export class CommandCenterDashboard {
if (!isSameOriginRequest(c.req.header("origin"), c.req.url)) {
return c.json({ error: "Cross-origin review requests are not allowed" }, 403);
}
if (!c.req.header("content-type")?.toLowerCase().startsWith("application/json")) {
if (!isJsonContentType(c.req.header("content-type"))) {
return c.json({ error: "Review requests must use application/json" }, 415);
}

Expand Down
Loading