diff --git a/README.md b/README.md index 8f4aa843..5b344c25 100644 --- a/README.md +++ b/README.md @@ -702,8 +702,8 @@ one-line reminder in every session. The same `PYTHONPATH` limit as for the | `SessionStart` | `session_start` | Explains that the first Daem0n call establishes the session and brief; `session_brief` returns the full brief | `CLAUDE_PROJECT_DIR` env | Yes | | `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to call `daem0n_tool_call` targeting `memory_recall_file`; silent elsewhere | stdin event | Yes | | `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against active Daem0n rules (`governance_rules` / `rules`) in read-only SQLite mode and emits matching `additionalContext` reminders | stdin event (`TOOL_INPUT` fallback) | Yes (always exits 0) | -| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request when the project is paired with `--enable-edit-bridge` | stdin event | Yes | -| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired | stdin event | Yes | +| `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Retained edit-bridge plumbing: stages a receipt for an explicit host-created request. The current remind-only hooks create no requests, so pairing alone does not activate this flow | stdin event | Yes | +| `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Retained edit-bridge plumbing: reports an approved edit as a capture candidate. The current remind-only hooks create no requests, so ordinary client edits produce no bridge capture | stdin event | Yes | | `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, suggests direct `memory_store` and `memory_record_outcome` calls with counsel-challenge retry guidance as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | No hook blocks a tool call or keeps the agent running: every hook exits 0 and none @@ -739,6 +739,11 @@ A Claude Code skill is included at `.claude/skills/daem0nmcp-protocol/SKILL.md` OpenCode connects to Daem0n-MCP via the same MCP server. No server changes required -- the daemon serves all clients equally. +The V1 plugin provides Covenant guidance and edit/Bash reminders. Its installed +hooks do not create native edit-bridge requests or enforce a +deny/preflight/retry flow; bridge API acceptance is separate from this +remind-only client integration. + ### Configuration Create `opencode.json` at your project root: @@ -1003,9 +1008,9 @@ daem0nmcp/ ├── claude_hooks/ # Claude Code native hooks │ ├── install.py # install/uninstall-claude-hooks CLI │ ├── session_start.py # Suggest scoped v7 briefing -│ ├── pre_edit.py # Fail-closed v7 preflight guidance -│ ├── pre_bash.py # Rule enforcement on commands -│ ├── post_edit.py # Significance detection +│ ├── pre_edit.py # Remind-only file recall guidance +│ ├── pre_bash.py # Read-only command-rule reminders +│ ├── post_edit.py # Retained edit-bridge capture reporting │ ├── stop.py # Read-only outcome/memory suggestions │ └── _client.py # Hook helper utilities ├── compression/ # LLMLingua-2 context compression diff --git a/daem0nmcp/api/v7/launcher.py b/daem0nmcp/api/v7/launcher.py index 8e4b86dc..46502d1d 100644 --- a/daem0nmcp/api/v7/launcher.py +++ b/daem0nmcp/api/v7/launcher.py @@ -92,21 +92,28 @@ def run_server( if not isinstance(options, ServerOptions): raise ValueError("server options are required") - if options.transport == "stdio": - run_strict_stdio(server) - return - validate_security(options.host, getattr(server, "auth", None)) - origin_middleware = build_origin_middleware(options.host, options.port) - from ...transport_security import build_uvicorn_security_config - - server.run( - transport="streamable-http", - host=options.host, - port=options.port, - middleware=origin_middleware, - stateless_http=False, - uvicorn_config=build_uvicorn_security_config(), - ) + if options.transport != "stdio": + validate_security(options.host, getattr(server, "auth", None)) + from ...tracing import configure_tracing, shutdown_tracing + + tracing_provider = configure_tracing() + try: + if options.transport == "stdio": + run_strict_stdio(server) + return + origin_middleware = build_origin_middleware(options.host, options.port) + from ...transport_security import build_uvicorn_security_config + + server.run( + transport="streamable-http", + host=options.host, + port=options.port, + middleware=origin_middleware, + stateless_http=False, + uvicorn_config=build_uvicorn_security_config(), + ) + finally: + shutdown_tracing(tracing_provider) __all__ = [ diff --git a/daem0nmcp/api/v7/production.py b/daem0nmcp/api/v7/production.py index 79bee444..8c3482fb 100644 --- a/daem0nmcp/api/v7/production.py +++ b/daem0nmcp/api/v7/production.py @@ -242,8 +242,9 @@ async def aclose(self) -> None: # Only entry points the profile's own extra installs; a module from another # profile would fail here and degrade a working one. _NATIVE_PROFILE_MODULES = { - "local": ("qdrant_client",), - "models-local": ("sentence_transformers", "onnx", "onnxruntime"), + "local": ("numpy", "qdrant_client"), + "models-local": ("numpy", "sentence_transformers", "onnx", "onnxruntime"), + "late-interaction": ("numpy",), "graph": ("networkx", "igraph", "leidenalg"), } @@ -278,7 +279,10 @@ def start(self) -> None: # broken native DLLs degrade upfront; auto-enabled profiles use # lightweight spec checks to avoid multi-second startup imports. try: - if eager: + # Windows can deadlock when a worker loads NumPy's DLLs + # while the main thread starts another worker. Initialize + # that shared prerequisite before any provider runs. + if eager or (os.name == "nt" and module == "numpy"): importlib.import_module(module) elif importlib.util.find_spec(module) is None: raise ModuleNotFoundError(module) diff --git a/daem0nmcp/claude_hooks/pre_bash.py b/daem0nmcp/claude_hooks/pre_bash.py index 6b348d9f..7d5deddb 100644 --- a/daem0nmcp/claude_hooks/pre_bash.py +++ b/daem0nmcp/claude_hooks/pre_bash.py @@ -15,6 +15,7 @@ from pathlib import Path from ..similarity import TFIDFIndex +from ..storage_activation import PointerValidationError, resolve_active_database from ._client import ( block, find_project_root, @@ -63,26 +64,10 @@ def _resolve_db_file(project_path: str) -> Path | None: storage_dir = Path(project_path) / ".daem0nmcp" / "storage" if not storage_dir.is_dir(): return None - pointer_path = storage_dir / "active.json" - if pointer_path.is_file(): - try: - pointer = json.loads(pointer_path.read_text(encoding="utf-8")) - candidate = ( - pointer.get("active_generation") if isinstance(pointer, dict) else None - ) - if ( - isinstance(candidate, str) - and candidate - and "/" not in candidate - and "\\" not in candidate - ): - candidate_path = storage_dir / candidate - if candidate_path.is_file(): - return candidate_path - except (OSError, ValueError): - pass - default_db = storage_dir / "daem0nmcp.db" - return default_db if default_db.is_file() else None + try: + return resolve_active_database(storage_dir).path + except (OSError, PointerValidationError): + return None def _load_active_rules(project_path: str) -> list[_RuleCandidate]: diff --git a/daem0nmcp/tracing.py b/daem0nmcp/tracing.py index 37d8326f..7c514ba1 100644 --- a/daem0nmcp/tracing.py +++ b/daem0nmcp/tracing.py @@ -4,13 +4,23 @@ FastMCP 3.0 has native OpenTelemetry support. This module provides configuration and helper utilities for Daem0n-specific tracing. -Enable by setting: +Install the observability extra and enable by setting: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 OTEL_SERVICE_NAME=daem0nmcp + +The default OTLP protocol is grpc; set OTEL_EXPORTER_OTLP_PROTOCOL to +http/protobuf for an HTTP collector. Configuration failures warn on stderr +and leave the core server available. """ +from __future__ import annotations + import logging import os +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from opentelemetry.sdk.trace import TracerProvider logger = logging.getLogger(__name__) @@ -18,22 +28,119 @@ def is_tracing_enabled() -> bool: - """Check if tracing is configured.""" + """Return whether startup installed or found an active tracing provider.""" + return _TRACING_ENABLED is True + + +def get_service_name() -> str: + """Get the service name for tracing.""" + return os.environ.get("OTEL_SERVICE_NAME", "daem0nmcp") + + +def configure_tracing() -> TracerProvider | None: + """Configure requested OTLP export and return only a provider we own.""" global _TRACING_ENABLED + _TRACING_ENABLED = False + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or os.environ.get( + "OTEL_EXPORTER_OTLP_ENDPOINT" + ) + if not endpoint or os.environ.get("OTEL_SDK_DISABLED", "").lower() == "true": + return None - if _TRACING_ENABLED is not None: - return _TRACING_ENABLED + from .capabilities import CapabilityRegistry - # Check for OTLP endpoint configuration - endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") - _TRACING_ENABLED = endpoint is not None and len(endpoint) > 0 + capability = CapabilityRegistry().get("observability") + if ( + capability["status"] == "disabled" + and capability["remediation"]["action"] == "enable_configuration" + ): + return None + try: + from opentelemetry import trace + except ImportError: + logger.warning( + 'OpenTelemetry tracing unavailable; install "daem0nmcp[observability]".' + ) + return None + try: + existing_provider = trace.get_tracer_provider() + except Exception: + logger.warning( + "OpenTelemetry tracing unavailable; check the configured tracing provider." + ) + return None + if not isinstance(existing_provider, trace.ProxyTracerProvider): + _TRACING_ENABLED = True + return None + protocol = ( + ( + os.environ.get("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") + or os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") + or "grpc" + ) + .strip() + .lower() + ) + if protocol not in {"grpc", "http/protobuf"}: + logger.warning( + "OpenTelemetry tracing unavailable; set OTEL_EXPORTER_OTLP_PROTOCOL " + "or OTEL_EXPORTER_OTLP_TRACES_PROTOCOL to grpc or http/protobuf." + ) + return None + if capability["status"] != "ready": + logger.warning( + "OpenTelemetry tracing unavailable; enable the observability profile " + 'and install "daem0nmcp[observability]".' + ) + return None - if _TRACING_ENABLED: - logger.info(f"OpenTelemetry tracing enabled: {endpoint}") + provider = None + try: + from opentelemetry.sdk.resources import OTELResourceDetector, Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor - return _TRACING_ENABLED + if protocol == "http/protobuf": + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + else: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + resource = Resource.create({"service.name": "daem0nmcp"}).merge( + OTELResourceDetector().detect() + ) + provider = TracerProvider(resource=resource, shutdown_on_exit=False) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(provider) + if trace.get_tracer_provider() is not provider: + shutdown_tracing(provider) + return None + except Exception: + shutdown_tracing(provider) + logger.warning( + "OpenTelemetry tracing unavailable; check the observability " + "installation and OTLP exporter configuration." + ) + return None + _TRACING_ENABLED = True + logger.info("OpenTelemetry tracing enabled.") + return provider -def get_service_name() -> str: - """Get the service name for tracing.""" - return os.environ.get("OTEL_SERVICE_NAME", "daem0nmcp") +def shutdown_tracing(provider: TracerProvider | None) -> None: + """Flush owned spans without allowing telemetry errors to mask shutdown.""" + global _TRACING_ENABLED + if provider is None: + return + _TRACING_ENABLED = False + try: + if not provider.force_flush(timeout_millis=5000): + logger.warning("OpenTelemetry trace flush timed out.") + except Exception: + logger.warning("OpenTelemetry trace flush failed.") + try: + provider.shutdown() + except Exception: + logger.warning("OpenTelemetry tracing shutdown failed.") diff --git a/daem0nmcp/ui/static/renderers/community.js b/daem0nmcp/ui/static/renderers/community.js index 4313277c..6d539626 100644 --- a/daem0nmcp/ui/static/renderers/community.js +++ b/daem0nmcp/ui/static/renderers/community.js @@ -85,9 +85,9 @@ svg.setAttribute("aria-label", "Memory communities"); const details = ui.element("aside", "community-details", "Select a community for details."); details.hidden = true; - if (data.communities.length && globalThis.D3) { - const root = globalThis.D3.hierarchy(hierarchy(data)).sum(value => value.value || 0); - globalThis.D3.treemap().size([960, 540]).padding(3)(root); + if (data.communities.length && globalThis.d3) { + const root = globalThis.d3.hierarchy(hierarchy(data)).sum(value => value.value || 0); + globalThis.d3.treemap().size([960, 540]).padding(3)(root); for (const leaf of root.leaves()) { const item = leaf.data.item; if (!item) continue; @@ -121,7 +121,9 @@ message.setAttribute("x", "480"); message.setAttribute("y", "270"); message.setAttribute("text-anchor", "middle"); - message.textContent = "No communities found."; + message.textContent = data.communities.length + ? "Community map is unavailable." + : "No communities found."; svg.append(message); } frame.append(svg, details); diff --git a/daem0nmcp/ui/static/renderers/graph.js b/daem0nmcp/ui/static/renderers/graph.js index 2bfd714c..2ec7c9ba 100644 --- a/daem0nmcp/ui/static/renderers/graph.js +++ b/daem0nmcp/ui/static/renderers/graph.js @@ -104,7 +104,7 @@ mount.replaceChildren(header, controls, frame); const context = canvas.getContext("2d"); - if (!context || !globalThis.D3) { + if (!context || !globalThis.d3) { loading.hidden = true; return; } @@ -252,7 +252,7 @@ canvas.height = Math.round(height * pixelRatio); clampPan(); if (simulation) { - simulation.force("center", globalThis.D3.forceCenter(width / 2, height / 2)); + simulation.force("center", globalThis.d3.forceCenter(width / 2, height / 2)); simulation.alpha(0.25).restart(); } draw(); @@ -260,12 +260,12 @@ const progressive = nodes.length > progressiveThreshold; activeNodes = progressive ? [] : nodes; - const linkForce = globalThis.D3.forceLink([]).id(node => ui.idKey(node.id)).distance(80); - simulation = globalThis.D3.forceSimulation(activeNodes) + const linkForce = globalThis.d3.forceLink([]).id(node => ui.idKey(node.id)).distance(80); + simulation = globalThis.d3.forceSimulation(activeNodes) .force("link", linkForce) - .force("charge", globalThis.D3.forceManyBody().strength(-120)) - .force("center", globalThis.D3.forceCenter(width / 2, height / 2)) - .force("collision", globalThis.D3.forceCollide(14)) + .force("charge", globalThis.d3.forceManyBody().strength(-120)) + .force("center", globalThis.d3.forceCenter(width / 2, height / 2)) + .force("collision", globalThis.d3.forceCollide(14)) .on("tick", draw); function rebuildLinks() { diff --git a/docs/release/v7/acceptance-2026-11-11.md b/docs/release/v7/acceptance-2026-11-11.md new file mode 100644 index 00000000..9e97b946 --- /dev/null +++ b/docs/release/v7/acceptance-2026-11-11.md @@ -0,0 +1,245 @@ +# Disposable end-to-end acceptance, 2026-11-11 + +This pass exercises production entry points and installed packages using +synthetic data in disposable workspaces. It is functional development evidence, +not final-release certification of every requirement in `requirements.json`. +Existing user databases and global client configurations were not changed. + +Base: `1fb98bb087e94fad949942b9a7d3227c62d55bd3` (merged PR #74). +Branch: `codex/v7-end-to-end-acceptance`. The base's +[merged-main CI run](https://github.com/9thLevelSoftware/Daem0n-MCP/actions/runs/42962203152) +passed all 12 jobs, including Python 3.10–3.12 on Linux, Windows and macOS. +That run predates the fixes below; it is not CI evidence for this branch. + +## Defects found and repaired + +1. **Command reminders read an obsolete database generation.** A real + CLI-installed Bash hook continued reading the legacy database after + `active-db.json` selected another generation. The hook now uses the shared + canonical pointer resolver. Invalid pointers and missing selected databases + produce no stale-rule fallback. The actual hook scenario and nine focused + tests passed; the canonical missing-target fixture also passed separately. +2. **Configured telemetry exported no spans.** A real loopback collector + received positive-control SDK spans but none from the production server. + The shared launcher now initializes the requested OTLP provider and flushes + only the provider it owns. HTTP/protobuf and default gRPC collectors receive + actual MCP spans. Disabled profiles stay disabled; missing dependencies and + invalid provider/exporter configuration warn without breaking core startup + or printing endpoint/header secrets. Provider ownership and cleanup have + focused regression coverage: 29 tests and five subtests passed. +3. **Windows restart could stall loading NumPy from a worker.** The real + five-source durable-consolidation scenario intermittently timed out at + `session_brief` after restart. A stack capture taken after the stall showed + the main thread starting a pool worker while the Qdrant worker loaded + NumPy's native core. Windows now initializes NumPy on the main thread for + ready dependent profiles before provider workers start. Other heavy auto + imports remain lazy. The ordering regression failed before the fix; 22 + production-composition tests plus five subtests, the original process case, + and three further restart runs passed with the original 30-second deadline. + The exact Windows loader-lock mechanism is an inference from those stacks. +4. **Community and graph visualizations stayed empty with valid data.** Real + Chromium rendering showed a community count of one alongside an empty-map + fallback. The shipped D3 bundle exports `d3`, while both renderers looked + for `D3`. Their references now use the actual bundle's global. Two new Node + regressions failed before the fix and passed afterward; they load the + shipped D3 bundle and check actual SVG cells/canvas drawing. Source browser + checks now show the populated treemap and linked graph and exercise their + controls through actual MCP calls. A review follow-up also distinguishes + unavailable visualization from an actually empty community payload. + +All four production changes received independent correctness and simplicity +review. The default-core gateway also gained real stdio/HTTP regression tests +for discovery, writes, idempotency, exact counsel remedies, spent-token refusal +and persisted state after restart. No gateway production defect was found. + +## Package and production-path evidence + +Artifacts and raw logs remain under ignored `.test_tmp/acceptance-20261111-*` +directories. They are local evidence, not published packages. The runtime wheel +below includes all three Python fixes; the sdist was built in the same invocation. +Subsequent test-only corrections do not change packaged runtime code. +No final commit identity is implied by these development builds. + +| Artifact | SHA-256 | +| --- | --- | +| `dist-final/daem0nmcp-7.0.0.dev0-py3-none-any.whl` | `351e6f42a9c1f65ef068a95f88564695b9b85a5c7e23b7ce72a8cf75f5cf43ed` | +| `dist-final/daem0nmcp-7.0.0.dev0.tar.gz` | `98fca909f369d6554ff45c4f6bd636d3f8246539ba9edccc85fdba373f6dbc28` | +| `dist-ui-fixed/daem0nmcp-7.0.0.dev0-py3-none-any.whl` | `375f83b1fe50f078bba43fa9dd87151425ba54985084aca03fd41303282254c7` | +| `dist-ui-fixed/daem0nmcp-7.0.0.dev0.tar.gz` | `a7ff7106798416619ec381ad667a7ed448e52eb8d1585f5b6185e71cb5f667e5` | +| `dist-reviewed/daem0nmcp-7.0.0.dev0-py3-none-any.whl` | `9fb3e25946fe17851a05559793af5f8d34588cc6a8a56c0b1f1174ca848c1e46` | +| `dist-reviewed/daem0nmcp-7.0.0.dev0.tar.gz` | `2d4d569f4912ef5c14a32308524556f7c5ee52f51f12ac0f6ab2a54957967887` | + +The artifact paths above abbreviate `.test_tmp/acceptance-20261111-core/`. +The rebuilt `dist-ui-fixed` wheel contains the renderer repair. Comparing every +ZIP member with the earlier wheel found changes only in the two renderer files +and `RECORD`; all Python runtime bytes are identical. Its private installation +was verified with `python -I`. +The final `dist-reviewed` wheel changes only `community.js` and `RECORD` from +that build, adding the distinct unavailable-map message. Its installed renderer +bytes match the tested source; every Python runtime file remains identical. + +| Check | Observed result | +| --- | --- | +| Baseline wheel and sdist | Fresh private installs, dependency checks and isolated package-import provenance passed; packaged dashboard and OpenCode assets present. | +| Installed-wheel process/boundary batch | 100 passed, three Windows platform skips. Covers memory rituals, stateful tools, history, workspace identity/relocation/isolation, HTTP/auth boundaries, code/discovery/dreaming, migration/recovery and portability. Some cases are component/race checks rather than independent process scenarios. | +| Baseline sdist ritual | Both stdio and HTTP passed, including outcome recording and persistence after restart. | +| Rebuilt sdist | Python 3.12.14 private installation and dependency check passed; isolated imports came from site-packages. Both stdio/HTTP memory and restart rituals passed against the final `dist-ui-fixed` sdist. | +| New core gateway | Two source and two installed-wheel transport cases passed. | +| Python 3.10.21 wheel install | Fresh private `dev,observability` install and dependency check passed; isolated imports resolve to its site-packages. Both real memory/restart rituals and all three real OTLP collector cases passed. | +| E2B SDK without live credentials | Fresh Python 3.12.14 `dev,agency-e2b` wheel install passed dependency/provenance checks. All 18 controlled SDK transport/provider tests and three subtests passed with `e2b-code-interpreter` 2.10.0 and `e2b` 2.63.0. These do not establish live sandbox isolation. | +| POSIX-only safeguards | Four skipped Windows cases passed on Linux. Three required native `/tmp` fixtures because Windows DrvFS does not implement the tested POSIX permissions; those initial DrvFS failures were not repaired in product code. | +| Real models and retrieval | ModernBERT ONNX returned finite normalized 256-dimensional vectors. Local Qdrant projection, dense retrieval, actual ColBERT reranking and restart passed on stdio and HTTP. | +| Compression and token counting | Actual LLMLingua CPU compression reduced 5,045 tokens to 1,750 while retaining the forced marker; local tiktoken passed. The `models-hosted` profile advertises tiktoken, so this check does not require a hosted-model API key. | +| Public HTTPS ingestion | Both transports fetched the actual public HTTPS fixture and verified canonical provenance and idempotency. | +| Final-wheel tasks | Four authenticated Redis lifecycle/restart and five-source consolidation cases passed on stdio/HTTP in 50.43 seconds. Unauthorized Redis access was rejected. | +| Remaining Redis scenarios | All 14 dispatcher cases and five subtests passed in 29.17 seconds against the accepted private wheel: cancellation/restart, admission/claim recovery, quotas/revocation, lost wakeups, outage recovery, shutdown, database faults and authority isolation. Combined with the four process cases, all 18 Redis-gated cases are covered. | +| Final-wheel models | Actual HTTP ModernBERT → local Qdrant → ColBERT rerank and restart passed in 26.26 seconds. Isolated imports and installed `production.py` bytes were verified against the wheel. | +| Service cleanup | The owned Redis process was stopped only after verifying its PID, executable and data directory; its loopback port was closed. | + +## Actual clients, hooks and dashboards + +Five real CLI/install/emitted-hook scenarios passed. The hook/install/bridge +batch passed 67 tests, with its Windows symlink skip separately covered on +Linux. Documentation checks passed 13 tests and 41 subtests; 23 JavaScript +protocol/UI tests passed. + +**Claude Code 2.1.319:** an authenticated installed CLI used an isolated MCP +configuration and disposable project to brief, inspect health, store, recall, +read/edit a file, obtain file guidance through the gateway and record an +outcome. Nine tool calls and four hook event pairs completed. Independent +SQLite inspection found exactly one synthetic record and one outcome. + +**OpenCode 1.18.21:** isolated plugin/MCP installation passed. Its existing +OpenAI provider OAuth refresh returned HTTP 401 before any MCP call. An actual +assistant workflow therefore remains unverified until local provider login is +restored (`opencode providers login`). No credential was copied into the report. + +Installed Claude/OpenCode hooks are intentionally remind-only. They do not +create native edit-bridge requests. Backend bridge subprocess acceptance does +not establish a native-client deny/preflight/retry or automatic capture flow; +the README and operations guide now state that boundary accurately. + +**Dashboard browser:** Chromium 153.0.8010.12 loaded all six unmodified +production `ui://` shells inside a disposable local Apps test host. That host +forwarded data/actions to an actual stdio MCP session using synthetic records, +links and a rebuilt community. Infrastructure, search/filter/refresh, +briefing/context, Covenant refresh, community drill-down and graph controls +passed. The final community/graph source runs had no page or console errors; +screenshots show the populated visualizations. This is local-host acceptance, +not compatibility certification inside Claude Desktop or ChatGPT. + +The rebuilt wheel passed the two affected browser scenarios using its private +Python installation for the actual MCP server. Five backend calls completed, +the treemap and nontransparent graph pixels were asserted, and page/console +error lists were empty. Source renderer checks passed 18 cases, including the +two actual-bundle regressions. The wheel result and screenshots are retained in +`acceptance-20261111-dashboard/packaged/`. + +## Regression and bounded stability + +The full Python 3.12 suite passed **3,957 tests and 2,097 subtests**, with 76 +skips and 510 warnings, in 1,517.79 seconds. Command: +`python -u -X utf8 -m pytest tests/ -q --asyncio-mode=auto +--junitxml=.test_tmp/acceptance-20261111-core/final-suite.xml`. +The skip inventory is retained in `core/skip-summary.json`: 25 platform-only +cases, 18 optional SDK cases, 18 Redis cases, ten live E2B cases, three OTLP +collector cases and two public-ingestion cases. Explicit provider runs above +closed all 18 SDK cases, all 18 Redis cases, all three OTLP cases and both +public-ingestion cases. Four of the platform-only cases also passed in local +Linux fixtures. These separate runs do not change the full suite's skip count; +the live E2B and remaining platform cases were not accepted from skipped runs. + +Full Ruff lint/format checks passed +(505 files), v7 mypy passed (48 source files with `--platform linux`), and +release inventory passed (77 tools, 10 resources, 587 requirements). + +The first [PR #76 CI run](https://github.com/9thLevelSoftware/Daem0n-MCP/actions/runs/43138245898) +passed eight of nine platform jobs and all three static gates. Windows/Python +3.11 exposed a test-only floating-point boundary: subtracting monotonic clock +values yielded `299.9999999999998` for a correctly scheduled 300-second wait. +The backoff test now compares absolute deadlines, retaining the strict minimum +wait and existing exponential/reset checks. The affected test and file-level +Ruff checks passed; independent review approved the assertion repair. Runtime +code and the accepted packages are unchanged. New-head CI is tracked on the PR. +The completed Ubuntu 3.11 verbose log also verified that all 25 platform-only +tests skipped locally on Windows actually ran and passed. + +The [second CI run](https://github.com/9thLevelSoftware/Daem0n-MCP/actions/runs/43142862389), +for `3fc261870c1ec7495496c71e2b544c6982d8f3af`, passed all 12 CI jobs, +including all nine OS/Python combinations. GitGuardian and Kilo also completed +successfully. Kilo's comment still contained a warning and suggestions, which +were checked separately from its green conclusion: + +- The proposed offline SQLite URI substitution is unsafe for this live hook. + Actual concurrent writers produced 65 readable rules with the current normal + SQLite locking, but zero rules and a false `SQLITE_CORRUPT` read with the + proposed immutable helper on both Windows and Linux. The Windows database + remained healthy (`quick_check=ok`, 65 rows). Normal read-only SQLite may + create temporary WAL coordination files; physically read-only storage with + absent sidecars still cannot provide those files. This configuration is not + claimed as supported by this acceptance pass. No unsafe fallback was added. +- Qdrant already declares NumPy as a dependency. Replacing the telemetry resource + merge as suggested would lose supported service-name environment precedence. + Neither proposal required a runtime change. +- The community view's missing-library fallback wording was addressed after + explicit review feedback: populated data without D3 reports "Community map + is unavailable."; genuinely empty data retains "No communities found.". + The new regression failed before the fix; both affected renderer checks + passed afterward, covering empty data with/without the real bundle and + populated data with/without it. Independent review approved the change. +- The new OTLP test's parent control did inherit ambient SDK/exporter settings. + `OTEL_SDK_DISABLED=true` reproduced a failed control span. The test now clears + inherited `OTEL_*` values through `monkeypatch`, restoring them afterward. + All three actual collector cases passed in 13.06 seconds with host SDK + disablement and gzip compression both set. Explicit server overrides remain + covered. This final test-only improvement passed file-level Ruff checks and + independent review; new-head CI remains linked from PR #76. + +The bounded HTTP mixed-workload run completed in 600.125 seconds with 1,604 +recalls, ten writes, ten outcomes, ten resource reads, ten health checks and two +briefings. It used concurrency four and retained ten resource samples. Python +source stayed unchanged throughout. Warm sampled RSS was roughly 0.99–1.03 GB; +after shutdown, owned child-process, thread, handle and RSS counts were all zero. +This is a successful ten-minute functional/stability smoke, not evidence of +eight-hour leak freedom or reference-hardware performance certification. +Command: `python -m benchmarks.v7_soak --hours 0.1667 +--transport streamable-http --interval 1`. + +## Remaining external and release boundaries + +- Live E2B sandbox isolation/network behavior remains unverified: no configured + `E2B_API_KEY` was available for staging acceptance. The SDK was installed and + its controlled transport/provider tests passed separately. +- OpenCode needs provider reauthentication before its real assistant workflow + can run. Installation success is not that workflow's acceptance. +- A local browser host can prove rendering and forwarding to a real MCP + server; it does not prove compatibility with a third-party MCP Apps host. +- Local Qdrant and loopback Redis/OTLP checks do not establish remote-service + deployment, internet TLS/proxy configuration or a production operating history. +- A bounded mixed-workload check is not the planned eight-hour soak, 100,000 + memory/one-million-event scale benchmark, relevance evaluation or RC release. + This report does not mark the release requirement ledger accepted. + +## Reproduction and retained evidence + +Build with `uv build --python .venv/Scripts/python.exe --out-dir `. +Install a built wheel/sdist in a fresh environment; verify imports with +`python -I` before running process tests with that interpreter. The production +process harness uses the same interpreter for its server subprocesses. + +Relevant tests include `test_process_client.py`, `test_process_gateway.py`, +`test_process_external_certification.py`, `test_process_observability.py`, +`test_production.py`, `test_claude_hooks_pre_bash.py` and `test_tracing.py`. +Real external cases require their explicit opt-in environment and provider +configuration; skipped default runs do not certify those integrations. + +| Local evidence prefix | Contents | +| --- | --- | +| `acceptance-20261111-core` | Package builds, clean installs, process JUnit/logs, full regression, static gates and bounded soak. | +| `acceptance-20261111-hooks` | Five-scenario JSON report, actual Claude/OpenCode results, hook checks and redacted client event summaries. | +| `acceptance-20261111-profiles` | Real-provider report, stalled-process stack capture, deterministic ordering regression, final-wheel model/task proof and owned Redis cleanup. | +| `acceptance-20261111-observability` | Before/after collector probes and focused telemetry results. | +| `acceptance-20261111-dashboard` | Local browser host, actual MCP-backed rendering/action checks and browser evidence. | + +Disposable service configuration may contain a fixture password. It is excluded +from Git and must not be copied into PR descriptions or published evidence. diff --git a/docs/release/v7/progress.md b/docs/release/v7/progress.md index 133acc0d..8a108963 100644 --- a/docs/release/v7/progress.md +++ b/docs/release/v7/progress.md @@ -35,12 +35,22 @@ P10 wheel/sdist core and profile installations, accurate complete docs/examples, Real MCP calls for every tool/argument/profile; ritual through restart; local/remote native edit and capture workflows; dashboards/dreaming/tasks/previews. Persistence interruption, disk/lock/permission/corruption failures. Python 3.10-3.12 Windows/macOS/Linux. Actual local/remote Qdrant, embeddings/ONNX, Redis/Valkey, E2B and both clients. Full Python/JS/Ruff/packaging/blocking changed-surface type checks. Reference 8-core/32GB/SSD benchmark: 100k memories/1m events; warm 4-way lexical p95 <=500ms, hybrid <=2s; write visibility lexical <=5s, specialized <=60s, dense <=5min; migration <=30min excluding dense; Recall@10/nDCG@10 regression <=0.02; 8h mixed soak without resource accumulation. Publish cold start, initial indexing, disk and stress evidence. ## Progress -Accepted final-release requirements: **0 / 571**. The current inventory contains -75 tools and 10 resources. No package has final-commit acceptance. Counts include +Accepted final-release requirements: **0 / 587**. The current inventory contains +77 tools and 10 resources. No package has final-commit acceptance. Counts include surface requirements and explicit package scenarios; passing component tests alone is not release acceptance. The dated entries below preserve development history and can describe findings repaired by later entries. +## Functional acceptance checkpoint - 2026-11-11 + +The [disposable end-to-end pass](acceptance-2026-11-11.md) records current +package, real-provider, client, dashboard and transport evidence. It found and +repaired stale-generation command reminders, missing OTLP initialization, +Windows NumPy startup ordering and the dashboard D3 global mismatch. The report +distinguishes verified local functionality from OpenCode provider login, live +E2B, third-party Apps hosts and the broader release/performance gates. It does +not convert those outstanding requirements into accepted release evidence. + ## Current local certification checkpoint - 2026-09-17 - This is evidence from the uncommitted `v7/completion` tree based on diff --git a/docs/release/v7/tool-process-coverage.md b/docs/release/v7/tool-process-coverage.md index ffc0446a..7f6fd259 100644 --- a/docs/release/v7/tool-process-coverage.md +++ b/docs/release/v7/tool-process-coverage.md @@ -13,6 +13,10 @@ protected mutation, and asserts stored state after each transition. The apps TODO scenario uses only `DAEM0NMCP_APPS_ENABLED=true`; it is skipped when `tree_sitter_language_pack` is unavailable. +`test_process_gateway.py` exercises the default core surface over both +transports: hidden-tool discovery, gateway writes and reads, idempotent replay, +exact counsel remedies, spent-token refusal, and persisted state after restart. + | Tool | Process evidence | Status | | --- | --- | --- | | `active_context_add` | `test_process_surface.py::test_production_stateful_core_surface` (stdio, HTTP) | real process | @@ -34,6 +38,8 @@ TODO scenario uses only `DAEM0NMCP_APPS_ENABLED=true`; it is skipped when | `context_trigger_list` | same; create/delete state assertions | real process | | `context_triggers_match` | same; matched trigger ID assertion | real process | | `covenant_status` | `test_process_surface.py::test_production_core_read_batch_and_projection_surface` (stdio, HTTP) | real process | +| `daem0n_tool_call` | `test_process_gateway.py::test_production_gateway_mutation_counsel_replay_and_restart` (stdio, HTTP) | real process | +| `daem0n_tools_search` | same; hidden-tool descriptors, schema and successful dispatch | real process | | `decision_debate` | `test_process_discovery_surface.py::test_production_debate_and_dream_cleanup_surface` (stdio, HTTP) | real process | | `decision_simulate` | `test_process_surface.py::test_production_intelligence_time_filter_surface` (stdio, HTTP; RFC3339 transaction time) | real process | | `document_ingest_url` | `test_process_external_certification.py::test_actual_public_document_ingestion` (apps, stdio, HTTP; actual HTTPS, provenance and replay); `.tmp/public-ingest-process-final.log` | real process / explicit network run | @@ -91,11 +97,11 @@ TODO scenario uses only `DAEM0NMCP_APPS_ENABLED=true`; it is skipped when | `workspace_links_list` | `test_process_surface.py::test_production_workspace_link_lifecycle_surface` (stdio, HTTP) | real process | | `workspace_unlink` | `test_process_surface.py::test_production_workspace_link_lifecycle_surface` (stdio, HTTP) | real process | -There is no registered `rule_delete` tool in the current 75-tool schema. +There is no registered `rule_delete` tool in the current 77-tool schema. Deleting a context trigger is covered above; rule lifecycle has create, list, check, and update (including `enabled: false`). -Current successful real-process coverage is **74 of 75 tools**. Public URL +Current successful real-process coverage is **76 of 77 tools**. Public URL ingestion passed with `DAEM0NMCP_CERTIFY_PUBLIC_INGEST=1` on both transports; without that opt-in its skip is not certification evidence. Real E2B sandbox execution remains open. These scenarios establish successful production paths, diff --git a/docs/v7-operations.md b/docs/v7-operations.md index 41028841..14299663 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -40,6 +40,16 @@ has no CPython 3.10 distribution. On Python 3.10, the resolver omits this profile's dependencies and the runtime reports the Python-version remediation; core installation remains supported. +For telemetry, install `observability` and set `OTEL_EXPORTER_OTLP_ENDPOINT` +and optionally `OTEL_SERVICE_NAME` before launching either server entry point. +OTLP uses `grpc` by default; set `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` +for an HTTP collector (normally port 4318). The standard `OTEL_*_TRACES_*` +overrides and exporter authentication/TLS settings are also honored. +`OTEL_SDK_DISABLED=true` or `DAEM0NMCP_OBSERVABILITY_ENABLED=false` disables +setup. Missing dependencies or invalid exporter configuration produce a stderr +warning while core MCP operations remain available. A provider already installed +by the embedding application is preserved and remains its responsibility. + The server, not an MCP caller, establishes the workspace set. Configure the default root and any allowed additional roots before launch: @@ -535,7 +545,7 @@ remind only: |-----------------|------|--------------|-------|--------------| | `SessionStart` | `session_start` | Explains that the first gated call briefs automatically and `session_brief` returns the full brief | `CLAUDE_PROJECT_DIR` env | Yes | | `PreToolUse` (`Edit\|Write\|NotebookEdit`) | `pre_edit` | Inside a Daem0n project, adds a one-line `additionalContext` reminder to use `daem0n_tool_call` for `memory_recall_file`; silent elsewhere | stdin event | Yes | -| `PreToolUse` (`Bash`) | `pre_bash` | Checks the command against Daem0n rules. Currently inert: it reads a `TOOL_INPUT` env var that Claude Code does not set | `TOOL_INPUT` env | Yes (always exits 0) | +| `PreToolUse` (`Bash`) | `pre_bash` | Reads active-generation Daem0n rules in read-only SQLite mode and emits matching `additionalContext` reminders | stdin event (`TOOL_INPUT` fallback) | Yes (always exits 0) | | `PostToolUse` (`mcp__.*__edit_preflight`) | `post_edit_preflight` | Edit-bridge plumbing: stages an `edit_preflight` receipt for a bridge edit request. Nothing in Claude Code creates those requests any more, so it is a no-op | stdin event | Yes | | `PostToolUse` (`Edit\|Write\|NotebookEdit`) | `post_edit` | Edit-bridge plumbing: reports a bridge-approved edit as a capture candidate. Only loads the bridge when the project is paired; a no-op in Claude Code today | stdin event | Yes | | `Stop`, `SubagentStop` | `stop` | When the transcript shows a finished task with no recorded outcome, suggests direct replay-safe `memory_store`/`memory_record_outcome` calls and counsel-challenge retry guidance as a `systemMessage`. Never writes memory | stdin event (`transcript_path`) | Yes | @@ -543,7 +553,7 @@ remind only: No hook blocks a tool call or keeps the agent running: every hook exits 0 and none returns a `permissionDecision` or `decision`. -Pairing writes an owner-only credential to +With `--enable-edit-bridge`, optional pairing writes an owner-only credential to `~/.daem0nmcp/edit-bridges//credential.json`, uses `~/.daem0nmcp/edit-bridges/run//` for the bridge socket, and adds `DAEM0NMCP_EDIT_BRIDGE_CREDENTIAL_FILE`, `DAEM0NMCP_EDIT_BRIDGE_RUNTIME_DIR`, @@ -562,11 +572,18 @@ python -m daem0nmcp.cli --project-path /srv/project-a install-opencode --interfa ``` The installer copies the packaged `daem0n.ts` asset and provisions host-side -credentials. Native edit flow is exact: the host creates an edit request, -`edit_preflight` returns a one-use receipt for that native edit, the identical -edit is retried, capture is listed with `memory_capture_list`, and promotion -uses `memory_capture_promote` after a matching `memory_preflight`. Do not add -forgeable `_client_meta` fields to a plugin or tool call. +credentials. The V1 plugin injects the Covenant guidance, runs the same +remind-only edit and Bash hooks, and appends their reminders to tool results. +Use the MCP memory tools to store, recall, and record outcomes; the plugin +does not require approval before a native edit. + +The retained edit-bridge API supports explicit host-created edit requests, +one-use `edit_preflight` receipts, capture listing, and preflight-protected +promotion. The current Claude Code and OpenCode hooks do not create those +requests, so pairing alone does not enable native-edit capture or a +deny/preflight/retry workflow in either client. Bridge subprocess tests cover +that API separately from the installed host integration. Do not add forgeable +`_client_meta` fields to a plugin or tool call. Protected remote pairing uses the installer’s explicit remote credential, workspace, URL, CA-file, and origin options through `install_opencode`; it is diff --git a/tests/api_v7/test_process_gateway.py b/tests/api_v7/test_process_gateway.py new file mode 100644 index 00000000..e4c42d3f --- /dev/null +++ b/tests/api_v7/test_process_gateway.py @@ -0,0 +1,119 @@ +"""Successful discovery and gateway calls through the production transports.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from tests.api_v7.process_client import ( + call, + initialize_workspaces, + process_client, + succeed, +) + + +async def _gateway(session, scope, tool, arguments): + result = await succeed( + session, + "daem0n_tool_call", + {**scope, "tool": tool, "arguments": arguments}, + ) + assert result["tool"] == tool + return result["data"] + + +@pytest.mark.parametrize("transport", ["stdio", "streamable-http"]) +async def test_production_gateway_mutation_counsel_replay_and_restart( + tmp_path, transport +): + [workspace] = await initialize_workspaces((tmp_path,)) + scope = {"workspace_id": workspace.workspace_id} + content = "Gateway persistence acceptance keeps this decision across restart." + store_arguments = { + "record_type": "decision", + "content": content, + "idempotency_key": "process-gateway-store-0001", + } + + async with process_client(workspace.root, transport) as session: + listed = {tool.name for tool in (await session.list_tools()).tools} + assert "daem0n_tools_search" in listed + assert "daem0n_tool_call" in listed + for tool in ("memory_archive_set", "memory_search_text"): + assert tool not in listed + found = await succeed(session, "daem0n_tools_search", {"query": tool}) + descriptor = next(item for item in found["tools"] if item["name"] == tool) + assert descriptor["listed"] is False + assert "workspace_id" in descriptor["input_schema"]["required"] + + stored = await _gateway(session, scope, "memory_store", store_arguments) + record_id = stored["record"]["record_id"] + assert stored["idempotent_replay"] is False + replayed = await _gateway(session, scope, "memory_store", store_arguments) + assert replayed["idempotent_replay"] is True + assert replayed["record"]["record_id"] == record_id + assert replayed["event_id"] == stored["event_id"] + + deadline = asyncio.get_running_loop().time() + 5 + while True: + searched = await _gateway( + session, + scope, + "memory_search_text", + {"query": "gateway persistence acceptance"}, + ) + if searched["items"]: + break + assert asyncio.get_running_loop().time() < deadline, searched + await asyncio.sleep(0.1) + assert len(searched["items"]) == 1 + assert searched["items"][0]["record"]["record_id"] == record_id + assert searched["items"][0]["bounded_excerpt"] == content + + archive_arguments = {"record_id": record_id, "archived": True} + challenged = await call( + session, + "daem0n_tool_call", + {**scope, "tool": "memory_archive_set", "arguments": archive_arguments}, + ) + assert challenged["ok"] is False + error = challenged["error"] + assert error["code"] == "COUNSEL_REQUIRED" + assert "DESTRUCTIVE_OPERATION" in error["counsel"]["reasons"] + remedy = error["remedy"] + assert remedy == { + "tool": "daem0n_tool_call", + "arguments": { + **scope, + "tool": "memory_archive_set", + "arguments": { + **archive_arguments, + "preflight_token": error["counsel"]["preflight_token"], + }, + }, + } + archived = await succeed(session, remedy["tool"], remedy["arguments"]) + assert archived["tool"] == "memory_archive_set" + assert archived["data"]["affected_ids"] == [record_id] + reused = await call(session, remedy["tool"], remedy["arguments"]) + assert reused["ok"] is False + assert reused["error"]["code"] == "COUNSEL_REQUIRED" + assert ( + reused["error"]["counsel"]["preflight_token"] + != error["counsel"]["preflight_token"] + ) + + async with process_client(workspace.root, transport) as restarted: + history = await _gateway( + restarted, + scope, + "memory_versions_list", + {"record_id": record_id}, + ) + assert len(history["items"]) == 2 + latest = history["items"][-1]["record"] + assert latest["record_id"] == record_id + assert latest["excerpt"] == content + assert latest["current_status"] == "archived" diff --git a/tests/api_v7/test_process_observability.py b/tests/api_v7/test_process_observability.py new file mode 100644 index 00000000..c06886e7 --- /dev/null +++ b/tests/api_v7/test_process_observability.py @@ -0,0 +1,168 @@ +"""Real MCP requests export spans to a disposable local OTLP collector.""" + +from __future__ import annotations + +import asyncio +import os +import threading +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from daem0nmcp.workspace import WorkspaceRegistry +from tests.api_v7.process_client import process_client, succeed + + +@contextmanager +def _collector(protocol): + messages = [] + trace_pb = pytest.importorskip( + "opentelemetry.proto.collector.trace.v1.trace_service_pb2" + ) + if protocol == "grpc": + grpc = pytest.importorskip("grpc") + service_pb = pytest.importorskip( + "opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc" + ) + + class Service(service_pb.TraceServiceServicer): + def Export(self, request, _context): # noqa: N802 - generated gRPC API + messages.append(request) + return trace_pb.ExportTraceServiceResponse() + + with ThreadPoolExecutor(max_workers=1) as executor: + server = grpc.server(executor) + service_pb.add_TraceServiceServicer_to_server(Service(), server) + port = server.add_insecure_port("127.0.0.1:0") + server.start() + try: + yield f"http://127.0.0.1:{port}", messages + finally: + server.stop(0).wait(5) + return + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + raw = self.rfile.read(int(self.headers["Content-Length"])) + messages.append(trace_pb.ExportTraceServiceRequest.FromString(raw)) + self.send_response(200) + self.send_header("Content-Type", "application/x-protobuf") + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", messages + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + assert not thread.is_alive() + + +def _service_spans(messages, service): + for message in tuple(messages): + for resource in message.resource_spans: + attributes = { + item.key: item.value.string_value + for item in resource.resource.attributes + } + if attributes.get("service.name") == service: + for scope in resource.scope_spans: + for span in scope.spans: + yield attributes, span.name + + +@pytest.mark.parametrize( + ("transport", "protocol"), + [ + ("stdio", "http/protobuf"), + ("streamable-http", "http/protobuf"), + ("stdio", "grpc"), + ], +) +async def test_production_otlp_export(transport, protocol, tmp_path, monkeypatch): + # The parent collector control must not inherit host SDK/exporter settings. + for name in tuple(os.environ): + if name.startswith("OTEL_"): + monkeypatch.delenv(name) + exporter_module = pytest.importorskip( + "opentelemetry.exporter.otlp.proto." + + ("http" if protocol == "http/protobuf" else "grpc") + + ".trace_exporter" + ) + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + workspace = WorkspaceRegistry([tmp_path], default_root=tmp_path).default + scope = {"workspace_id": workspace.workspace_id} + service = f"daem0n-observability-{transport}" + with _collector(protocol) as (endpoint, messages): + control = TracerProvider(resource=Resource.create({"service.name": "control"})) + control.add_span_processor( + SimpleSpanProcessor( + exporter_module.OTLPSpanExporter( + endpoint=endpoint + "/v1/traces" + if protocol == "http/protobuf" + else endpoint + ) + ) + ) + try: + with control.get_tracer("acceptance-control").start_as_current_span( + "collector-control" + ): + pass + finally: + control.shutdown() + assert list(_service_spans(messages, "control")) + environment = { + "OTEL_SERVICE_NAME": service, + "OTEL_RESOURCE_ATTRIBUTES": "acceptance.run=isolated,service.name=ignored", + "OTEL_BSP_SCHEDULE_DELAY": "100", + } + if protocol == "grpc": + environment["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint + elif transport == "stdio": + environment.update( + OTEL_EXPORTER_OTLP_ENDPOINT=endpoint, + OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf", + ) + else: + environment.update( + OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:1", + OTEL_EXPORTER_OTLP_PROTOCOL="grpc", + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=endpoint + "/v1/traces", + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf", + ) + async with process_client( + tmp_path, transport, environment_overrides=environment + ) as session: + await succeed(session, "session_brief", scope) + await succeed(session, "system_health", scope) + await succeed( + session, + "memory_recall", + {**scope, "query": "private-observability-query"}, + ) + deadline = asyncio.get_running_loop().time() + 5 + while True: + spans = list(_service_spans(messages, service)) + if "tools/call memory_recall" in {name for _, name in spans}: + break + assert asyncio.get_running_loop().time() < deadline, spans + await asyncio.sleep(0.05) + names = {name for _, name in spans} + assert {"tools/call session_brief", "tools/call system_health"} <= names + assert all( + attributes["acceptance.run"] == "isolated" for attributes, _ in spans + ) + assert all("private-observability-query" not in str(item) for item in messages) diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index 142ab15d..4055e819 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -86,7 +86,7 @@ def test_native_runtime_preloads_only_each_enabled_profile( production._OptionalNativeRuntimeLifecycle( {"local": "ready", "models-local": "disabled"} ).start() - self.assertEqual(["qdrant_client"], imported) + self.assertEqual(["numpy", "qdrant_client"], imported) production._OptionalNativeRuntimeLifecycle( {"local": "disabled", "models-local": "ready"} ).start() @@ -94,7 +94,9 @@ def test_native_runtime_preloads_only_each_enabled_profile( self.assertEqual( [ + "numpy", "qdrant_client", + "numpy", "sentence_transformers", "onnx", "onnxruntime", @@ -133,6 +135,106 @@ def import_module(name: str) -> object: self.assertEqual("degraded", states["graph"].status) self.assertIn("igraph", str(states["graph"].remediation)) + def test_windows_auto_native_profiles_initialize_numpy_before_workers(self) -> None: + from daem0nmcp.api.v7 import production + + for profile in ("local", "models-local", "late-interaction"): + with self.subTest(profile=profile): + calls: list[tuple[str, str]] = [] + with ( + patch.object(production.os, "name", "nt"), + patch.object( + production.importlib, + "import_module", + side_effect=lambda name, calls=calls: calls.append( + ("import", name) + ), + ), + patch.object( + production.importlib.util, + "find_spec", + side_effect=lambda name, calls=calls: ( + calls.append(("spec", name)) or object() + ), + ), + ): + production._OptionalNativeRuntimeLifecycle( + {profile: "ready"}, environ={} + ).start() + + self.assertEqual(("import", "numpy"), calls[0]) + self.assertEqual( + [("import", "numpy")], + [call for call in calls if call[0] == "import"], + ) + + def test_windows_broken_numpy_degrades_only_its_ready_dependents(self) -> None: + from daem0nmcp.api.v7 import production + + statuses = { + "local": "ready", + "models-local": "ready", + "late-interaction": "ready", + "graph": "ready", + "apps": "ready", + } + + def import_module(name): + self.assertEqual("numpy", name) + raise OSError("native array runtime unavailable") + + with ( + patch.object(production.os, "name", "nt"), + patch.object(production.importlib, "import_module", import_module), + patch.object(production.importlib.util, "find_spec", return_value=object()), + ): + lifecycle = production._OptionalNativeRuntimeLifecycle(statuses, environ={}) + lifecycle.start() + + for profile in ("local", "models-local", "late-interaction"): + self.assertEqual("degraded", statuses[profile]) + self.assertIn("numpy", lifecycle.failures[profile]) + self.assertEqual("ready", statuses["graph"]) + self.assertEqual("ready", statuses["apps"]) + + def test_posix_auto_native_profiles_keep_imports_lazy(self) -> None: + from daem0nmcp.api.v7 import production + + with ( + patch.object(production.os, "name", "posix"), + patch.object(production.importlib, "import_module") as imported, + patch.object(production.importlib.util, "find_spec", return_value=object()), + ): + production._OptionalNativeRuntimeLifecycle( + { + "local": "ready", + "models-local": "ready", + "late-interaction": "ready", + "graph": "ready", + }, + environ={}, + ).start() + imported.assert_not_called() + + def test_disabled_native_profiles_initialize_no_prerequisites(self) -> None: + from daem0nmcp.api.v7 import production + + with ( + patch.object(production.os, "name", "nt"), + patch.object(production.importlib, "import_module") as imported, + patch.object(production.importlib.util, "find_spec") as inspected, + ): + production._OptionalNativeRuntimeLifecycle( + { + "local": "disabled", + "models-local": "disabled", + "late-interaction": "disabled", + }, + environ={}, + ).start() + imported.assert_not_called() + inspected.assert_not_called() + def test_capability_states_carry_the_registry_remediation(self) -> None: """Health must say how to enable a profile, not 'review' it.""" from daem0nmcp.api.v7 import production diff --git a/tests/js/test_ui_renderers.mjs b/tests/js/test_ui_renderers.mjs index d4389b01..04cc0f99 100644 --- a/tests/js/test_ui_renderers.mjs +++ b/tests/js/test_ui_renderers.mjs @@ -230,7 +230,7 @@ function renderApp(appId, payload, options = {}) { const trace = { linkCounts: [], nodeCounts: [], simulation: null, tick: null }; context.document = document; context.devicePixelRatio = options.devicePixelRatio ?? 1; - context.D3 = options.d3 === false ? undefined : fakeD3(trace); + context.d3 = options.d3 === false ? undefined : fakeD3(trace); context.requestAnimationFrame = options.animationFrames ? callback => { nextAnimationFrame += 1; animationFrames.set(nextAnimationFrame, callback); @@ -244,6 +244,11 @@ function renderApp(appId, payload, options = {}) { context.cancelIdleCallback = () => {}; context.setTimeout = () => 1; context.clearTimeout = () => {}; + context.setInterval = () => 1; + context.clearInterval = () => {}; + if (options.realD3) { + vm.runInContext(fs.readFileSync(path.join(root, "daem0nmcp/ui/static/d3.bundle.js"), "utf8"), context); + } context.addEventListener = () => {}; const messenger = { on(name, handler) { handlers.set(name, handler); }, @@ -280,6 +285,38 @@ test("all six fixed renderers register without a DOM", () => { assert.deepEqual(Array.from(ui.appIds()), ["test", "search", "briefing", "covenant", "community", "graph"]); }); +test("community draws populated cells with the shipped D3 bundle", () => { + const rendered = renderApp("community", { + communities: [{ id: "community_1", name: "Actual layout", member_count: 2, level: 0 }], + }, { realD3: true }); + assert.equal(findAll(rendered.document.mount, node => node.getAttribute("class") === "community-cell__rect").length, 1); + assert.ok(findByText(rendered.document.mount, "Actual layout")); + assert.equal(findByText(rendered.document.mount, "No communities found."), null); +}); + +test("community distinguishes empty data from a missing visualization library", () => { + for (const options of [{ realD3: true }, { d3: false }]) { + const empty = renderApp("community", { communities: [] }, options); + assert.ok(findByText(empty.document.mount, "No communities found.")); + assert.equal(findByText(empty.document.mount, "Community map is unavailable."), null); + } + + const unavailable = renderApp("community", { + communities: [{ id: "community_1", name: "Retained community", member_count: 2 }], + }, { d3: false }); + assert.ok(findByText(unavailable.document.mount, "Community map is unavailable.")); + assert.equal(findByText(unavailable.document.mount, "No communities found."), null); + assert.equal(findAll(unavailable.document.mount, node => node.getAttribute("class") === "community-cell__rect").length, 0); +}); + +test("graph draws populated nodes with the shipped D3 bundle", () => { + const rendered = renderApp("graph", { + nodes: [{ id: "record_1", category: "decision", content: "Actual node" }], + }, { realD3: true }); + assert.ok(rendered.document.canvasContext.operations.some(operation => operation[0] === "arc")); + assert.equal(findByClass(rendered.document.mount, "graph-loading").hidden, true); +}); + test("search normalizer clamps numeric values and rejects boolean IDs", () => { const result = ui.normalize("search", { decisions: [{ id: true, content: "ok", relevance: 95, semantic_match: -1, recency_weight: Infinity }], diff --git a/tests/test_claude_hooks_pre_bash.py b/tests/test_claude_hooks_pre_bash.py index 74bad53b..eb1d5dab 100644 --- a/tests/test_claude_hooks_pre_bash.py +++ b/tests/test_claude_hooks_pre_bash.py @@ -96,11 +96,17 @@ def test_no_project_exits_clean(tmp_path, monkeypatch): assert exc_info.value.code == 0 -def test_v7_governance_rules_stdin_reminder(tmp_path): +@pytest.mark.parametrize("pointer_state", ["legacy", "active", "malformed", "missing"]) +def test_v7_governance_rules_stdin_reminder(tmp_path, pointer_state): import json import sqlite3 from daem0nmcp.claude_hooks.pre_bash import reminder + from daem0nmcp.event_store import canonical_json_bytes + from daem0nmcp.storage_activation import ( + ActiveDatabasePointer, + write_active_pointer, + ) storage = tmp_path / ".daem0nmcp" / "storage" storage.mkdir(parents=True) @@ -132,6 +138,35 @@ def test_v7_governance_rules_stdin_reminder(tmp_path): finally: conn.close() + if pointer_state == "active": + active_file = storage / "current-v7.db" + with sqlite3.connect(db_file) as source, sqlite3.connect(active_file) as active: + source.backup(active) + active.execute( + "UPDATE governance_rules SET must_not_json = ?", + (json.dumps(["Never force push from the active generation"]),), + ) + write_active_pointer( + storage, + ActiveDatabasePointer( + 7, 2, active_file.name, db_file.name, "mig_" + "a" * 64 + ), + ) + elif pointer_state == "malformed": + (storage / "active-db.json").write_text("invalid JSON", encoding="utf-8") + elif pointer_state == "missing": + (storage / "active-db.json").write_bytes( + canonical_json_bytes( + { + "format_version": 7, + "generation": 2, + "active_db": "missing.db", + "previous_db": db_file.name, + "migration_run_id": "mig_" + "a" * 64, + } + ), + ) + output_raw = reminder( { "cwd": str(tmp_path), @@ -139,8 +174,16 @@ def test_v7_governance_rules_stdin_reminder(tmp_path): "tool_input": {"command": "git push --force origin main"}, } ) + if pointer_state in {"malformed", "missing"}: + assert output_raw is None + return assert output_raw is not None output = json.loads(output_raw)["hookSpecificOutput"] assert output["hookEventName"] == "PreToolUse" - assert "Never force push to main" in output["additionalContext"] + expected = ( + "Never force push from the active generation" + if pointer_state == "active" + else "Never force push to main" + ) + assert expected in output["additionalContext"] assert "permissionDecision" not in output diff --git a/tests/test_tracing.py b/tests/test_tracing.py index c6263cc4..a054ba88 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -1,6 +1,10 @@ """Test OpenTelemetry tracing integration.""" import os +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest def test_tracing_module_exists(): @@ -36,3 +40,152 @@ def test_get_service_name_default(): finally: if old_val: os.environ["OTEL_SERVICE_NAME"] = old_val + + +@pytest.fixture +def tracing_environment(monkeypatch): + from daem0nmcp import tracing + + for name in tuple(os.environ): + if name.startswith("OTEL_") or name == "DAEM0NMCP_OBSERVABILITY_ENABLED": + monkeypatch.delenv(name) + monkeypatch.setattr(tracing, "_TRACING_ENABLED", None) + return tracing + + +@pytest.mark.parametrize("disabled", ["no-endpoint", "sdk", "profile"]) +def test_disabled_tracing_does_not_import_optional_sdk( + tracing_environment, monkeypatch, disabled +): + import builtins + + if disabled != "no-endpoint": + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318") + if disabled == "sdk": + monkeypatch.setenv("OTEL_SDK_DISABLED", "TRUE") + if disabled == "profile": + monkeypatch.setenv("DAEM0NMCP_OBSERVABILITY_ENABLED", "false") + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name.startswith(("opentelemetry.sdk", "opentelemetry.exporter")): + raise AssertionError("disabled tracing imported an optional dependency") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + assert tracing_environment.configure_tracing() is None + assert not tracing_environment.is_tracing_enabled() + + +def test_preconfigured_tracing_provider_is_not_replaced_or_closed( + tracing_environment, monkeypatch +): + from opentelemetry import trace + + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318") + provider = SimpleNamespace(shutdown=Mock()) + monkeypatch.setattr(trace, "get_tracer_provider", lambda: provider) + install = Mock() + monkeypatch.setattr(trace, "set_tracer_provider", install) + owned = tracing_environment.configure_tracing() + tracing_environment.shutdown_tracing(owned) + assert owned is None + assert tracing_environment.is_tracing_enabled() + install.assert_not_called() + provider.shutdown.assert_not_called() + + +@pytest.mark.parametrize( + "failure", + [ + "missing-dependencies", + "auto-missing-dependencies", + "invalid-protocol", + "invalid-provider", + ], +) +def test_tracing_configuration_failure_keeps_protocol_clean_and_core_available( + tracing_environment, monkeypatch, caplog, capsys, failure +): + from opentelemetry import trace + + from daem0nmcp.api.v7 import launcher + from daem0nmcp.capabilities import CapabilityRegistry + + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_ENDPOINT", "http://private-token@example.invalid:4318" + ) + if failure == "invalid-provider": + monkeypatch.setattr( + trace, + "get_tracer_provider", + Mock(side_effect=RuntimeError("private-provider-secret")), + ) + elif failure == "invalid-protocol": + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "invalid-secret-protocol") + else: + monkeypatch.setattr( + CapabilityRegistry, + "get", + lambda _self, _name: { + "status": "disabled" + if failure == "auto-missing-dependencies" + else "degraded", + "remediation": {"action": "install_extra"}, + }, + ) + calls = [] + server = SimpleNamespace(run=lambda **kwargs: calls.append(kwargs)) + launcher.run_server(server, launcher.ServerOptions("stdio", "127.0.0.1", 8765)) + assert len(calls) == 1 + assert calls[0]["transport"] == "stdio" + assert not tracing_environment.is_tracing_enabled() + assert "OpenTelemetry tracing unavailable" in caplog.text + assert "private-token" not in caplog.text + assert "invalid-secret-protocol" not in caplog.text + assert "private-provider-secret" not in caplog.text + assert capsys.readouterr().out == "" + + +def test_owned_tracing_cleanup_does_not_mask_launch_failure( + tracing_environment, monkeypatch, caplog +): + from daem0nmcp.api.v7 import launcher + + provider = SimpleNamespace( + force_flush=Mock(side_effect=RuntimeError("private-exporter-secret")), + shutdown=Mock(side_effect=RuntimeError("private-exporter-secret")), + ) + monkeypatch.setattr(tracing_environment, "configure_tracing", lambda: provider) + + def fail(**_kwargs): + raise ValueError("original launch failure") + + with pytest.raises(ValueError, match="original launch failure"): + launcher.run_server( + SimpleNamespace(run=fail), + launcher.ServerOptions("stdio", "127.0.0.1", 8765), + ) + provider.force_flush.assert_called_once_with(timeout_millis=5000) + provider.shutdown.assert_called_once_with() + assert "private-exporter-secret" not in caplog.text + + +def test_http_security_failure_does_not_initialize_tracing( + tracing_environment, monkeypatch +): + from daem0nmcp.api.v7 import launcher + + initialize = Mock() + monkeypatch.setattr(tracing_environment, "configure_tracing", initialize) + + def reject(_host, _auth): + raise ValueError("remote server rejected") + + with pytest.raises(ValueError, match="remote server rejected"): + launcher.run_server( + SimpleNamespace(auth=None), + launcher.ServerOptions("streamable-http", "0.0.0.0", 8765), + validate_security=reject, + ) + initialize.assert_not_called() diff --git a/tests/test_v7_dreaming_runtime.py b/tests/test_v7_dreaming_runtime.py index d0ef465a..2d4a9b0b 100644 --- a/tests/test_v7_dreaming_runtime.py +++ b/tests/test_v7_dreaming_runtime.py @@ -1427,7 +1427,11 @@ def over_deadline(self, workspace_id, **_kwargs): False, ) first_wait = state.community_retry_at - passes[-1] - assert first_wait >= dreaming_runtime._COMMUNITY_BACKOFF_SECONDS + # Subtracting clock values can round a full wait just below 300 seconds. + assert ( + state.community_retry_at + >= passes[-1] + dreaming_runtime._COMMUNITY_BACKOFF_SECONDS + ) # The same inputs are retried once the wait has elapsed, and a second # consecutive deadline waits longer.