[codex] unify trace queries and outcome recording

.github/workflows/ci.yml

@@ -18,10 +18,10 @@ jobs:
         run: |
           python -m pip install uv
           uv python install 3.11
-          uv sync --extra dev --extra trace-query
+          uv sync --extra dev --extra plugin
       - name: Ruff
-        run: uv run ruff check src tests plugins/openflywheel-trace-query/scripts/mcp_server.py
+        run: uv run ruff check src tests plugins/openflywheel/scripts/mcp_server.py
       - name: mypy
-        run: uv run mypy src tests plugins/openflywheel-trace-query/scripts/mcp_server.py
+        run: uv run mypy src tests plugins/openflywheel/scripts/mcp_server.py
       - name: Tests and coverage
         run: uv run pytest --cov=ofw --cov-report=term-missing --cov-fail-under=90 -q

AGENTS.md

@@ -64,13 +64,14 @@ These instructions apply to the entire repository.
 ## Verification commands
 
 ```bash
-uv sync --extra dev --extra trace-query
-uv run ruff check src tests plugins/openflywheel-trace-query/scripts/mcp_server.py
-uv run mypy src tests plugins/openflywheel-trace-query/scripts/mcp_server.py
+uv sync --extra dev --extra plugin
+uv run ruff check src tests plugins/openflywheel/scripts/mcp_server.py
+uv run mypy src tests plugins/openflywheel/scripts/mcp_server.py
 uv run pytest --cov=ofw --cov-report=term-missing --cov-fail-under=90 -q
-uvx --from radon radon cc -s -a src tests plugins/openflywheel-trace-query/scripts/mcp_server.py
-python3 ~/.codex/skills/.system/skill-creator/scripts/quick_validate.py plugins/openflywheel-trace-query/skills/trace-query-planner
-python3 ~/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py plugins/openflywheel-trace-query
+uvx --from radon radon cc -s -a src tests plugins/openflywheel/scripts/mcp_server.py
+python3 ~/.codex/skills/.system/skill-creator/scripts/quick_validate.py plugins/openflywheel/skills/trace-query-planner
+python3 ~/.codex/skills/.system/skill-creator/scripts/quick_validate.py plugins/openflywheel/skills/outcome-recorder
+python3 ~/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py plugins/openflywheel
 ```
 
 ## Repository hygiene

plugins/openflywheel/.codex-plugin/plugin.json

@@ -0,0 +1,24 @@
+{
+  "name": "openflywheel",
+  "version": "0.2.0",
+  "description": "Query Langfuse trajectories and record authoritative verifier outcomes.",
+  "author": {
+    "name": "OpenFlyWheel"
+  },
+  "license": "MIT",
+  "keywords": ["agents", "evaluation", "langfuse", "traces"],
+  "skills": "./skills/",
+  "interface": {
+    "displayName": "OpenFlyWheel",
+    "shortDescription": "Query traces and record verifier outcomes",
+    "longDescription": "Select and inspect bounded Langfuse trajectory evidence, then record authoritative external-verifier outcomes on exact traces.",
+    "developerName": "OpenFlyWheel",
+    "category": "Productivity",
+    "capabilities": ["Read", "Write"],
+    "defaultPrompt": [
+      "Inspect this Langfuse trace with the fewest bounded queries.",
+      "Record this completed verifier outcome on its exact trace."
+    ]
+  },
+  "mcpServers": "./.mcp.json"
+}

plugins/openflywheel-trace-query/.mcp.json → plugins/openflywheel/.mcp.json

@@ -1,10 +1,10 @@
 {
   "mcpServers": {
-    "openflywheel-trace-query": {
+    "openflywheel": {
       "command": "sh",
       "args": [
         "-c",
-        "exec uv run --project \"${OPENFLYWHEEL_ROOT:-$PWD}\" --extra trace-query python \"$PLUGIN_ROOT/scripts/mcp_server.py\""
+        "exec uv run --project \"${OPENFLYWHEEL_ROOT:-$PWD}\" --extra plugin python \"$PLUGIN_ROOT/scripts/mcp_server.py\""
       ]
     }
   }

plugins/openflywheel-trace-query/scripts/mcp_server.py → plugins/openflywheel/scripts/mcp_server.py

@@ -1,17 +1,26 @@
 #!/usr/bin/env python3
-"""Official typed MCP surface for read-only OpenFlyWheel trace queries."""
+"""Typed OpenFlyWheel MCP surface for trace queries and outcome recording."""
 
 from __future__ import annotations
 
 import os
 from collections.abc import Callable
+from datetime import datetime
+from enum import StrEnum
 from typing import Annotated, TypeVar
 
 from mcp.server.fastmcp import FastMCP
 from mcp.types import ToolAnnotations
 from pydantic import BaseModel, Field
 
+from ofw.evaluation.langfuse import (
+    LangfuseOutcomeStore,
+    OutcomeStoreObservation,
+    OutcomeStoreStatus,
+)
+from ofw.evaluation.outcome import OutcomeEvaluation, TaskId, VerifierId
 from ofw.observability.langfuse.contracts import LangfuseProject
+from ofw.observability.langfuse.domain import TraceId
 from ofw.observability.langfuse.trace_query import (
     GetSpanContextInput,
     GetTraceSchemaInput,
@@ -25,6 +34,7 @@
     TraceTimeRange,
 )
 from ofw.observability.langfuse.transport import LangfuseHttpClient
+from ofw.runtime import EvidenceReference, VerifierResult, VerifierVerdict
 
 QueryInput = TypeVar("QueryInput")
 QueryOutput = TypeVar("QueryOutput", bound=BaseModel)
@@ -33,10 +43,18 @@
 SpanIdentifier = Annotated[str, Field(min_length=1, max_length=256)]
 CursorIdentifier = Annotated[str, Field(min_length=1, max_length=4096)]
 TracePageLimit = Annotated[int, Field(strict=True, ge=1, le=50)]
+TaskIdentifier = Annotated[str, Field(min_length=1, max_length=256)]
+VerifierIdentifier = Annotated[str, Field(min_length=1, max_length=256)]
+OutcomeScore = Annotated[float, Field(strict=True, ge=0.0, le=1.0)]
+EvidenceIdentifier = Annotated[str, Field(min_length=1, max_length=1024)]
+OutcomeEvidence = Annotated[tuple[EvidenceIdentifier, ...], Field(min_length=1, max_length=10)]
 
 server = FastMCP[None](  # type: ignore[misc]  # MCP auth generics are untyped upstream.
-    name="openflywheel-trace-query",
-    instructions="Read-only structural ITSMBench trace queries. Never judges or mutates data.",
+    name="openflywheel",
+    instructions=(
+        "Read bounded Langfuse trace evidence and record only authoritative external-verifier "
+        "outcomes. Never infer outcomes or mutate traces."
+    ),
     log_level="DEBUG",
 )
 read_only = ToolAnnotations(
@@ -45,14 +63,42 @@
     idempotentHint=True,
     openWorldHint=True,
 )
+record_write = ToolAnnotations(
+    readOnlyHint=False,
+    destructiveHint=False,
+    idempotentHint=True,
+    openWorldHint=True,
+)
 
 
-def _client() -> LangfuseHttpClient:
-    project = LangfuseProject.from_env(
+class OutcomeToolErrorCode(StrEnum):
+    STORE_FAILED = "outcome_store_failed"
+
+
+class OutcomeToolError(Exception):
+    """Sanitized outcome-recording failure returned to the MCP client."""
+
+    __slots__ = ("code", "trace_id")
+
+    def __init__(self, code: OutcomeToolErrorCode, trace_id: str) -> None:
+        self.code = code
+        self.trace_id = trace_id
+        super().__init__(f"{code.value}: {trace_id}")
+
+
+def _project() -> LangfuseProject:
+    return LangfuseProject.from_env(
         environment=os.environ.get("LANGFUSE_ENVIRONMENT", "ofw-local"),
         allow_private_network=os.environ.get("LANGFUSE_ALLOW_PRIVATE_NETWORK") == "1",
     )
-    return LangfuseHttpClient(project, timeout_seconds=_QUERY_TIMEOUT_SECONDS)
+
+
+def _client() -> LangfuseHttpClient:
+    return LangfuseHttpClient(_project(), timeout_seconds=_QUERY_TIMEOUT_SECONDS)
+
+
+def _outcome_store() -> LangfuseOutcomeStore:
+    return LangfuseOutcomeStore.from_project(_project())
 
 
 def _execute(
@@ -123,5 +169,47 @@ def get_span_context(
     return _execute(query, TraceQueryService.get_span_context)
 
 
+@server.tool(annotations=record_write, structured_output=True)
+def record_outcome(
+    trace_id: TraceIdentifier,
+    task_id: TaskIdentifier,
+    verifier_id: VerifierIdentifier,
+    evaluated_at: datetime,
+    verdict: VerifierVerdict,
+    evidence: OutcomeEvidence,
+    score: OutcomeScore | None = None,
+) -> OutcomeStoreObservation:
+    """Record one authoritative external-verifier outcome on its exact trace."""
+    result = VerifierResult(
+        verdict=verdict,
+        score=score,
+        feedback="Recorded by the OpenFlywheel outcome tool.",
+        evidence=tuple(EvidenceReference(reference) for reference in evidence),
+    )
+    outcome = OutcomeEvaluation.from_verifier_result(
+        trace_id=TraceId(trace_id),
+        task_id=TaskId(task_id),
+        verifier_id=VerifierId(verifier_id),
+        evaluated_at=evaluated_at,
+        result=result,
+    )
+    try:
+        store = _outcome_store()
+        try:
+            submission = store.store(outcome)
+        finally:
+            store.close()
+    except Exception:
+        raise OutcomeToolError(OutcomeToolErrorCode.STORE_FAILED, trace_id) from None
+    return OutcomeStoreObservation(
+        status=OutcomeStoreStatus.SUCCESS,
+        summary=f"Stored authoritative {verdict.value} outcome on the trace.",
+        next_actions=("Continue only after retaining this score receipt.",),
+        artifacts=(trace_id, submission.score_id.value),
+        trace_id=trace_id,
+        score_id=submission.score_id.value,
+    )
+
+
 if __name__ == "__main__":
     server.run(transport="stdio")

plugins/openflywheel/skills/outcome-recorder/SKILL.md

@@ -0,0 +1,17 @@
+---
+name: outcome-recorder
+description: Record a completed authoritative verifier result on its exact Langfuse trace. Use after an external verifier returns a final outcome with trace, task, verifier, UTC timestamp, score, and evidence; do not use to infer outcomes or classify implicit failures.
+---
+
+# Outcome Recorder
+
+After an authoritative verifier finishes, call `record_outcome` before continuing to failure mining or harness optimization.
+
+- Use the exact `trace_id` emitted for the task run. Never select a trace by guesswork; use the read tools to resolve it when necessary.
+- Use a stable `task_id` and versioned `verifier_id`. Use the verifier completion time as UTC `evaluated_at`.
+- Map the verifier result directly: `pass` and `fail` require a normalized score from 0 to 1; `abstain` and `error` require no score.
+- Include one to ten stable evidence references such as verifier reports, audit artifacts, or environment checks. Do not copy trace blobs or credentials into evidence.
+- Treat the returned `score_id` as the receipt. Retrying the same outcome is safe and must return the same logical score.
+- Stop and report the missing field when the verifier result cannot satisfy the contract. Never invent a verdict, score, timestamp, evidence reference, or verifier version.
+
+This skill records only authoritative outcomes. It does not judge trajectory quality, mine failure types, promote dataset cases, or alter trace data.

plugins/openflywheel/skills/outcome-recorder/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Outcome Recorder"
+  short_description: "Record verified outcomes on Langfuse traces"
+  default_prompt: "Record this completed verifier result on its exact Langfuse trace."

plugins/openflywheel-trace-query/skills/trace-query-planner/SKILL.md → plugins/openflywheel/skills/trace-query-planner/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: trace-query-planner
-description: Plan minimal read-only structural queries for an ITSMBench Langfuse session or trace. Use for selecting traces and spans by IDs, tools, types, UTC ranges, or error flags; do not use for judging, summarizing, or semantic search.
+description: Plan minimal read-only structural queries for a Langfuse session or trace. Use for selecting traces and spans by IDs, tools, types, UTC ranges, or error flags; do not use for judging, recording outcomes, summarizing, or semantic search.
 ---
 
 # TraceQueryPlanner
@@ -17,6 +17,6 @@ Accept either a session query or a `trace_id` with optional deterministic filter
 - If the request is ambiguous, ask for exactly the single field named in `missing_fields`.
 - Stop when the returned spans answer the structural request. Do not fetch a full trace blob or expand every match.
 
-Only use `list_traces`, `get_trace_schema`, `query_spans`, and `get_span_context`. They are read-only. Never call write or update APIs, judge correctness, infer failure types, summarize beyond returned span content, or add semantic/vector search.
+For trace querying, only use `list_traces`, `get_trace_schema`, `query_spans`, and `get_span_context`. They are read-only. Never use `record_outcome` while following this skill, judge correctness, infer failure types, summarize beyond returned span content, or add semantic/vector search.
 
 Preserve every tool observation as returned. All tools return `status`, `summary`, `next_actions`, `artifacts`, `ordering`, `filters_applied`, `next_cursor`, and `truncated`. `list_traces` additionally returns `traces_found`; the trace-span tools additionally return `trace_id`, `spans_found`, `span_types`, and `missing_fields`.

plugins/openflywheel/skills/trace-query-planner/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Trace Query Planner"
+  short_description: "Plan minimal read-only Langfuse trace queries"
+  default_prompt: "Find the smallest bounded query for this Langfuse trace."

pyproject.toml

@@ -21,7 +21,7 @@ dev = [
     "pytest-cov>=5,<6",
     "ruff>=0.5,<1",
 ]
-trace-query = ["mcp>=1.13,<2"]
+plugin = ["mcp>=1.13,<2"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/ofw"]

src/ofw/__init__.py

@@ -26,19 +26,32 @@
     Sha256Digest,
     WorkspaceFile,
 )
+from ofw.evaluation import (
+    LangfuseOutcomeStore,
+    OutcomeErrorCode,
+    OutcomeEvaluation,
+    OutcomeEvaluationError,
+    OutcomeScoreSubmission,
+    OutcomeStoreObservation,
+    OutcomeStoreStatus,
+    TaskId,
+    VerifierId,
+)
 from ofw.harness import EditableFile, Harness, Subagent, Tool, editable
 from ofw.observability.langfuse import (
     CollectionError,
     CollectionErrorCode,
     LangfuseProject,
     TraceWindow,
 )
+from ofw.observability.langfuse.domain import TraceId
 from ofw.runtime import (
     CanaryCase,
     CaseId,
     CommandLoop,
     CommandVerifier,
     E2BSandbox,
+    EvidenceReference,
     ModelFingerprint,
     ProcessCommand,
     ProcessLimits,
@@ -65,6 +78,7 @@ class _OfwNamespace:
     def editable(self, path: Path) -> EditableFile:
         return editable(path)
 
+
 ofw = _OfwNamespace()
 
 __all__ = [
@@ -78,6 +92,7 @@ def editable(self, path: Path) -> EditableFile:
     "CommandVerifier",
     "E2BSandbox",
     "EditableFile",
+    "EvidenceReference",
     "GitCommit",
     "Harness",
     "HarnessAsset",
@@ -87,10 +102,17 @@ def editable(self, path: Path) -> EditableFile:
     "HarnessRevisionId",
     "HarnessValidationError",
     "Langfuse",
+    "LangfuseOutcomeStore",
     "LangfuseOtelSpanAttributes",
     "LangfuseProject",
     "LangfuseSpan",
     "ModelFingerprint",
+    "OutcomeErrorCode",
+    "OutcomeEvaluation",
+    "OutcomeEvaluationError",
+    "OutcomeScoreSubmission",
+    "OutcomeStoreObservation",
+    "OutcomeStoreStatus",
     "RepositorySnapshot",
     "ProcessCommand",
     "ProcessLimits",
@@ -99,10 +121,13 @@ def editable(self, path: Path) -> EditableFile:
     "RunStatus",
     "Sha256Digest",
     "Subagent",
+    "TaskId",
     "Tool",
+    "TraceId",
     "TraceWindow",
     "VerifierResult",
     "VerifierVerdict",
+    "VerifierId",
     "WorkspaceFile",
     "editable",
     "get_client",