diff --git a/Makefile b/Makefile index 9d23d6f..28dd17e 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ DC := docker compose -f docker-compose.yml PYTEST_FOCUS ?= app PYTEST_APP_TESTS := archive/services/orchestrator_legacy_python/tests/test_orchestrator_retrieval.py archive/services/orchestrator_legacy_python/tests/test_migration_runtime.py -.PHONY: help launch all up up-core down status ps logs build rebuild pull clean prune mcp-proxy-up init qdrant-init mindsdb-seed letta-seed models-pull proxy-status doctor mem-ping monitor-open monitor-check dmg-build msi-build linux-bundle-build storage-audit qdrant-snapshot-prune qdrant-cutover cold-snapshot-pack cold-snapshot-tier cold-snapshot-restore telemetry-archive fanout-status fanout-deadletters fanout-rehydrate retention-install retention-uninstall retention-status retention-install-daily storage-ledger-capture storage-ledger-prune storage-ledger-install storage-ledger-uninstall storage-ledger-status memory-graph-quality memory-graph-quality-install memory-graph-quality-uninstall memory-graph-quality-status weekly-lineage-rollup weekly-lineage-install weekly-lineage-uninstall weekly-lineage-status docker-fs-watchdog-run docker-fs-watchdog-install docker-fs-watchdog-uninstall docker-fs-watchdog-status storage-migrate-hot-bindings disk-clean-safe mem-mode-show mem-mode-core mem-mode-balanced mem-mode-full mem-up-core mem-up-balanced mem-up-full observability-up observability-down launch-readiness-gate launch-readiness-gate-schedule launch-readiness-gate-schedule-status launch-readiness-gate-schedule-cancel paid-launch-checklist backup-restore-drill mem-up-release mem-up-lite-release release-lock-verify qdrant-cloud-check quickstart submission-preflight launch-lock launch-lock-public test-py bench-shortlist bench-qdrant-tuning bench-backend-lanes env-lock-check env-lock-apply sentrux-check sentrux-gate sentrux-gate-save agent-context-gate +.PHONY: help launch all up up-core down status ps logs build rebuild pull clean prune mcp-proxy-up init qdrant-init mindsdb-seed letta-seed models-pull proxy-status doctor mem-ping monitor-open monitor-check dmg-build msi-build linux-bundle-build storage-audit qdrant-snapshot-prune qdrant-cutover cold-snapshot-pack cold-snapshot-tier cold-snapshot-restore telemetry-archive fanout-status fanout-deadletters fanout-rehydrate retention-install retention-uninstall retention-status retention-install-daily storage-ledger-capture storage-ledger-prune storage-ledger-install storage-ledger-uninstall storage-ledger-status memory-graph-quality memory-graph-quality-install memory-graph-quality-uninstall memory-graph-quality-status recall-quality recall-quality-refresh recall-quality-tuning open-core-boundary-audit weekly-lineage-rollup weekly-lineage-install weekly-lineage-uninstall weekly-lineage-status docker-fs-watchdog-run docker-fs-watchdog-install docker-fs-watchdog-uninstall docker-fs-watchdog-status storage-migrate-hot-bindings disk-clean-safe mem-mode-show mem-mode-core mem-mode-balanced mem-mode-full mem-up-core mem-up-balanced mem-up-full observability-up observability-down launch-readiness-gate launch-readiness-gate-schedule launch-readiness-gate-schedule-status launch-readiness-gate-schedule-cancel paid-launch-checklist backup-restore-drill mem-up-release mem-up-lite-release release-lock-verify qdrant-cloud-check quickstart submission-preflight launch-lock launch-lock-public test-py bench-shortlist bench-qdrant-tuning bench-backend-lanes env-lock-check env-lock-apply sentrux-check sentrux-gate sentrux-gate-save agent-context-gate help: > echo "Targets:" @@ -54,6 +54,8 @@ help: > echo " storage-ledger-capture|storage-ledger-prune: append/prune metadata-only storage growth ledger" > echo " storage-ledger-install|storage-ledger-status: install hourly ledger runner (launchd)" > echo " memory-graph-quality*: score graph coverage and install bounded repair runner" +> echo " recall-quality*: run saved recall eval, terminal quality view, and tuning" +> echo " open-core-boundary-audit: verify lite/full/paid branch feature boundaries" > echo " weekly-lineage-rollup: generate weekly per-project lineage + global synergy rollups" > echo " weekly-lineage-install|weekly-lineage-status: install weekly lineage runner (launchd)" > echo " qdrant-cutover: set QDRANT_COLLECTION and rehydrate vectors" @@ -319,6 +321,21 @@ memory-graph-quality-uninstall: memory-graph-quality-status: > bash scripts/install_memory_graph_quality_runner.sh status +recall-quality: +> scripts/agent/recall-quality-eval --tuning --pretty + +recall-quality-refresh: +> scripts/agent/recall-quality-eval --refresh-cases --tuning --pretty + +recall-quality-tuning: +> if [ -f .env ]; then source .env >/dev/null 2>&1 || true; fi +> base="$${CONTEXTLATTICE_ORCHESTRATOR_URL:-http://127.0.0.1:8075}"; key="$${CONTEXTLATTICE_ORCHESTRATOR_API_KEY:-}"; \ +> if [ -n "$$key" ]; then curl -fsS -H "x-api-key: $$key" "$${base%/}/telemetry/recall/tuning?min_samples=1" | jq .; \ +> else curl -fsS "$${base%/}/telemetry/recall/tuning?min_samples=1" | jq .; fi + +open-core-boundary-audit: +> scripts/agent/audit-open-core-boundary --pretty + weekly-lineage-rollup: > scripts/context_storage_ops.sh weekly-lineage \ > --orchestrator-url "$${CONTEXTLATTICE_ORCHESTRATOR_URL:-http://127.0.0.1:8075}" \ diff --git a/contextlattice-dashboard/app/api/telemetry/recall/route.ts b/contextlattice-dashboard/app/api/telemetry/recall/route.ts new file mode 100644 index 0000000..2e17ff8 --- /dev/null +++ b/contextlattice-dashboard/app/api/telemetry/recall/route.ts @@ -0,0 +1,16 @@ +import { NextResponse } from "next/server"; +import { callOrchestrator } from "@/lib/orchestrator"; + +export async function GET(request: Request) { + const url = new URL(request.url); + const params = new URLSearchParams(); + for (const key of ["traffic_class"]) { + const value = url.searchParams.get(key); + if (value) { + params.set(key, value); + } + } + const suffix = params.toString() ? `?${params.toString()}` : ""; + const data = await callOrchestrator(`/telemetry/recall${suffix}`); + return NextResponse.json(data); +} diff --git a/contextlattice-dashboard/app/api/telemetry/recall/tuning/route.ts b/contextlattice-dashboard/app/api/telemetry/recall/tuning/route.ts new file mode 100644 index 0000000..522c2d8 --- /dev/null +++ b/contextlattice-dashboard/app/api/telemetry/recall/tuning/route.ts @@ -0,0 +1,16 @@ +import { NextResponse } from "next/server"; +import { callOrchestrator } from "@/lib/orchestrator"; + +export async function GET(request: Request) { + const url = new URL(request.url); + const params = new URLSearchParams(); + for (const key of ["lookback_hours", "min_samples", "max_samples"]) { + const value = url.searchParams.get(key); + if (value) { + params.set(key, value); + } + } + const suffix = params.toString() ? `?${params.toString()}` : ""; + const data = await callOrchestrator(`/telemetry/recall/tuning${suffix}`); + return NextResponse.json(data); +} diff --git a/contextlattice-dashboard/app/status/page.tsx b/contextlattice-dashboard/app/status/page.tsx index f438f83..84cdca3 100644 --- a/contextlattice-dashboard/app/status/page.tsx +++ b/contextlattice-dashboard/app/status/page.tsx @@ -3,6 +3,7 @@ import { useEffect, useState } from "react"; import { RetrievalPanel } from "@/components/RetrievalPanel"; import { MemoryGraphPanel, type MemoryGraphPayload } from "@/components/MemoryGraphPanel"; +import { RecallQualityPanel, type RecallQualityPayload, type RecallTuningPayload } from "@/components/RecallQualityPanel"; type Service = { name: string; @@ -63,18 +64,22 @@ export default function StatusPage() { const [topics, setTopics] = useState(null); const [memoryTelemetry, setMemoryTelemetry] = useState(null); const [memoryGraph, setMemoryGraph] = useState(null); + const [recallQuality, setRecallQuality] = useState(null); + const [recallTuning, setRecallTuning] = useState(null); const [error, setError] = useState(null); const [updatedAt, setUpdatedAt] = useState(null); async function loadStatus() { try { setError(null); - const [statusRes, prefRes, topicRes, memRes, graphRes] = await Promise.all([ + const [statusRes, prefRes, topicRes, memRes, graphRes, recallRes, tuningRes] = await Promise.all([ fetch("/api/memory/status", { cache: "no-store" }), fetch("/api/memory/preferences", { cache: "no-store" }), fetch("/api/memory/topics", { cache: "no-store" }), fetch("/api/telemetry/memory", { cache: "no-store" }), fetch("/api/telemetry/memory/graph", { cache: "no-store" }), + fetch("/api/telemetry/recall", { cache: "no-store" }), + fetch("/api/telemetry/recall/tuning", { cache: "no-store" }), ]); const statusData = await statusRes.json(); if (!statusRes.ok) { @@ -93,6 +98,12 @@ export default function StatusPage() { if (graphRes.ok) { setMemoryGraph(await graphRes.json()); } + if (recallRes.ok) { + setRecallQuality(await recallRes.json()); + } + if (tuningRes.ok) { + setRecallTuning(await tuningRes.json()); + } setUpdatedAt(new Date().toLocaleTimeString()); } catch (err: any) { setError(err?.message || "Status unavailable"); @@ -214,6 +225,8 @@ export default function StatusPage() { + + ); diff --git a/contextlattice-dashboard/components/RecallQualityPanel.tsx b/contextlattice-dashboard/components/RecallQualityPanel.tsx new file mode 100644 index 0000000..ba6d33d --- /dev/null +++ b/contextlattice-dashboard/components/RecallQualityPanel.tsx @@ -0,0 +1,195 @@ +"use client"; + +type QualityTotals = { + requests?: number; + timeouts?: number; + errors?: number; + sourceErrorRate?: number; + noHitRate?: number; + lowConfidenceRate?: number; + staleHitRate?: number; + recallAtK?: number | null; + mrr?: number | null; + numericExactness?: number | null; + citationCoverage?: number | null; + sourceDiversity?: number | null; + graphLift?: number | null; + evalP95Ms?: number | null; + lastEvalAt?: string | null; +}; + +export type RecallQualityPayload = { + updatedAt?: string; + trafficClass?: string; + quality?: { + status?: string; + totals?: QualityTotals; + sampleCount?: number; + recommendations?: string[]; + }; + alerts?: { + count?: number; + }; +}; + +export type RecallTuningPayload = { + window?: { + samples?: number; + minSamples?: number; + sufficient?: boolean; + }; + recommended?: { + quality?: { + graphExpansion?: { + enabled?: boolean; + depth?: number; + neighborLimit?: number; + }; + sourceOrder?: string[]; + recommendations?: string[]; + }; + }; +}; + +function numberValue(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function percentText(value: unknown): string { + if (typeof value !== "number" || !Number.isFinite(value)) { + return "-"; + } + return `${Math.round(value * 100)}%`; +} + +function statusTone(status: string) { + if (status === "repair_recommended" || status === "insufficient_cases") { + return "bg-amber-500 text-amber-950"; + } + if (status === "watch" || status === "unknown") { + return "bg-cyan-500 text-cyan-950"; + } + return "bg-emerald-500 text-emerald-950"; +} + +function QualityBar({ value, tone = "good" }: { value: number; tone?: "good" | "warn" | "neutral" }) { + const pct = Math.max(0, Math.min(100, value * 100)); + const color = tone === "warn" ? "bg-amber-300" : tone === "neutral" ? "bg-cyan-300" : "bg-emerald-300"; + return ( +