diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 72e2957..1ed1b09 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", - "version": "0.2.0", - "description": "Query Langfuse trajectories and record authoritative verifier outcomes.", + "version": "0.3.0", + "description": "Initialize ITSM-bench harness workspaces, query Langfuse trajectories, and record authoritative verifier outcomes.", "author": { "name": "OpenFlyWheel" }, @@ -10,14 +10,14 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Query traces and record verifier outcomes", - "longDescription": "Select and inspect bounded Langfuse trajectory evidence, then record authoritative external-verifier outcomes on exact traces.", + "shortDescription": "Initialize ITSM workspaces and inspect traces", + "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, select and inspect bounded Langfuse trajectory evidence, and record authoritative external-verifier outcomes on exact traces.", "developerName": "OpenFlyWheel", "category": "Productivity", "capabilities": ["Read", "Write"], "defaultPrompt": [ - "Inspect this Langfuse trace with the fewest bounded queries.", - "Record this completed verifier outcome on its exact trace." + "Read this repository, identify the primary agent harness the user wants to evaluate with ITSM-bench, and use $workspace-init to collect its experiment configuration and perform the initial setup.", + "When PROGRAM.md is ready, read it and continue the optimization loop until its declared goal or stop condition is met." ] }, "mcpServers": "./.mcp.json" diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md new file mode 100644 index 0000000..b1873c6 --- /dev/null +++ b/plugins/openflywheel/program_templates/base.md @@ -0,0 +1,73 @@ +# OpenFlywheel Agent Program + +This file is generated by `workspace_prepare`. Do not edit it directly. + +## Mission + +Improve the connected agent harness until the goal in `experiment_config.yaml` is met, +while respecting its quality, cost, latency, budget, and stopping constraints. + +The baseline has already been recorded. Begin at step 2; do not rerun the unchanged +baseline. + +## Authority + +- `experiment_config.yaml` defines the harness, editable surface, frozen controls, goal, + benchmark, verifier, budget, and stopping conditions. +- The external verifier is authoritative for task outcome. The agent's completion claim + is not proof of success. +- Langfuse is the source of truth for trajectories, usage, cost, and latency. +- Missing verifier evidence is `unverified`, not failure and not success. + +## Editable and frozen surfaces + +Edit only paths explicitly allowed by `experiment_config.yaml`. Never change the benchmark, +held-out tasks, verifier, model, reasoning budget, observability identity, or this program +to improve a score. + +Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control +flow changes unless the evidence requires the combination. + +## Optimization loop + +### 2. Analyze failures + +Start from verifier-backed failed outcomes. Use bounded trace queries to locate relevant +evidence, then inspect only the spans needed to explain the observed behavior. Do not load +or copy complete traces when filters answer the question, and do not inspect held-out +trajectory content. + +### 3. Form one hypothesis + +State the failure pattern, supporting trace and verifier evidence, proposed harness change, +expected improvement, and possible regressions. Stop if the evidence cannot distinguish +between materially different changes. + +### 4. Improve the harness + +Make the smallest change within the declared editable surface. Preserve frozen controls and +unrelated user changes. + +### 5. Gate the change + +Run only the prepared experiment command and gates declared by the workspace. Compare +task-level verifier outcomes and report quality, cost, and latency separately. Missing or +errored trials remain visible and cannot disappear from the denominator. + +### 6. Keep or revert + +Keep the change only when the configured gate admits it. Otherwise revert only the current +iteration's harness edit, retain the evidence, and try a different hypothesis. Never weaken +the gate to admit a candidate. + +### 7. Repeat + +Return to step 2 with the newly recorded run. Stop when the configured goal is met, the +budget or iteration limit is exhausted, the no-improvement condition is reached, or required +authoritative evidence is unavailable. + +## Final report + +Report the accepted harness revision, verifier-backed quality, cost, latency, remaining +unverified trials, iteration count, and exact stopping reason. Do not claim improvement from +an agent-authored summary alone. diff --git a/plugins/openflywheel/program_templates/itsm.md b/plugins/openflywheel/program_templates/itsm.md new file mode 100644 index 0000000..c43a30d --- /dev/null +++ b/plugins/openflywheel/program_templates/itsm.md @@ -0,0 +1,58 @@ +## ITSM-bench Instructions + +This program supports only `benchmark: itsm-bench`. + +## Record the prepared baseline outcomes + +Before diagnosing failures, process every terminal Harbor trial from the prepared baseline: + +1. Read the trial's `result.json`, `exception_info`, verifier status, and verifier artifacts. +2. Map a reward only when the verifier completed and the trial has no execution or verifier + error: + - Exact `1.0` -> `pass` with score `1.0`. + - Exact `0.0` -> `fail` with score `0.0`. + - Any other present reward -> record nothing and report an unsupported-reward mapping + blocker. + - An explicit authoritative `abstain` or `error` verdict -> preserve that verdict without + a score. + - An exception, verifier error, or missing verifier result without an explicit authoritative + verdict -> record nothing and report the trial as unverified. +3. Use the task directory name as `task_id`, `itsm-bench@` as + `verifier_id`, and the verifier completion time as `evaluated_at`. +4. Resolve exactly one Langfuse trace using the prepared session, environment, release, + and the trial's agent-execution time window. +5. Call `record_outcome` with stable verifier evidence references and retain its score + receipt. + +If the verifier result is absent, record nothing. If trace selection is empty or ambiguous, +record nothing and report the mapping blocker. `record_outcome` is the only permitted +Langfuse write. + +## Analyze failed ITSM trajectories + +Use the trace tools in the smallest sufficient sequence: + +1. `list_traces` selects candidate traces for the prepared session. +2. `get_trace_schema` skims structure without loading input or output. +3. `query_spans` selects exact observations by ID, tool, type, UTC range, error flag, or + deterministic text filter. +4. `get_span_context` retrieves bounded raw context only for a selected span. + +An intermediate tool error is evidence, not an outcome failure, when the agent recovered +and the verifier passed. A technically clean trajectory is still a failure when the ITSM +verifier shows that the required environment state was not achieved. + +## ITSM optimization constraints + +- Treat the ITSM verifier score as the authoritative quality metric. +- Read cost and latency from Langfuse; do not write separate cost or latency scores. +- Preserve least-privilege behavior and verify environment state before declaring success. +- Do not expose held-out ITSM tasks or verifier internals to the harness being optimized. +- Run trials sequentially when deterministic trace-to-trial mapping depends on execution + windows. + +## ITSM iteration report + +For every candidate run, report verifier passes, verifier failures, unverified trials, +outcome receipts, trace-mapping blockers, the count and values of unsupported-reward mapping +blockers, total Langfuse cost, latency, and the gate decision. diff --git a/plugins/openflywheel/skills/workspace-init/SKILL.md b/plugins/openflywheel/skills/workspace-init/SKILL.md new file mode 100644 index 0000000..87aa146 --- /dev/null +++ b/plugins/openflywheel/skills/workspace-init/SKILL.md @@ -0,0 +1,74 @@ +--- +name: workspace-init +description: Initialize an OpenFlywheel ITSM-bench optimization workspace by inspecting an agent-harness repository, collecting its experiment configuration one field at a time, creating the managed PROGRAM.md placeholder, and handing preparation to workspace_prepare. Use when onboarding a primary harness for ITSM-bench; do not use for other benchmarks, an already-prepared workspace, ordinary trace queries, or outcome recording. +--- + +# Workspace Init + +Initialize one primary agent harness for ITSM-bench without starting optimization +prematurely. This skill supports only `benchmark: itsm-bench`. Stop and report the +unsupported benchmark if the user requests anything else. + +## 1. Discover the harness + +Read the repository before asking questions. Identify likely prompts, tools, agent loops, +benchmark runners, verifiers, observability wiring, and editable harness files. Present the +most likely primary harness and ask the user to confirm it. Do not choose among materially +different harnesses without confirmation. + +## 2. Collect the experiment configuration + +Ask one focused question at a time. Infer repository facts first and recommend a default +when the evidence supports one. Collect, in order: + +1. Harness root and explicitly editable files or directories. +2. Optimization goal, primary metric, target, and stopping condition. Keep quality, cost, + and latency constraints separate rather than hiding them in one average. +3. ITSM-bench root, Harbor task manifest or selection, and expected task count. +4. Authoritative verifier, reward interpretation, and pass threshold. +5. Frozen model, reasoning effort, concurrency, per-task timeout, retry policy, and budget. +6. Langfuse environment, release, and session naming rule. + +Never request secret values in chat or write them into configuration. Check only whether +the required environment-variable names are present. + +Summarize the complete proposed experiment and obtain confirmation before writing files or +starting a potentially costly baseline. Then create `/experiment_config.yaml` +using only the confirmed values. + +## 3. Create the managed program placeholder + +Copy [assets/PROGRAM.md](assets/PROGRAM.md) byte-for-byte to +`/PROGRAM.md`. Do not overwrite an existing different `PROGRAM.md`; stop and +ask whether the existing program should be preserved or replaced. + +Do not compose the final program yourself. Call `workspace_prepare` with the confirmed +experiment configuration. That tool owns validation, baseline execution, result parsing, +and deterministic composition from `program_templates/base.md` and +`program_templates/itsm.md`. + +`workspace_prepare` is long-running and re-entrant: + +- On `running`, retain the preparation ID and poll the same request after the returned + interval. Never start a second baseline. +- On `failed`, report its typed recovery instruction and stop at its declared stop + condition. +- On `ready`, retain the baseline artifacts and confirm that `PROGRAM.md` is no longer the + placeholder. + +If `workspace_prepare` is unavailable, stop after the confirmed configuration and +placeholder. Report that workspace preparation is not installed. Do not replace the tool +with an improvised shell command. + +## 4. Hand off to the optimization program + +When preparation is `ready`, start a fresh Codex session with exactly this task: + +```text +Read PROGRAM.md and start the optimization loop. +The baseline is already recorded. Start from step 2 (analyze failures). +``` + +The generated program is authoritative for editable files, gates, metrics, budgets, and +stopping conditions. Do not rerun the baseline, weaken the verifier, expose held-out traces, +or continue past the declared goal or stop condition. diff --git a/plugins/openflywheel/skills/workspace-init/agents/openai.yaml b/plugins/openflywheel/skills/workspace-init/agents/openai.yaml new file mode 100644 index 0000000..237e08d --- /dev/null +++ b/plugins/openflywheel/skills/workspace-init/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Initialize ITSM Harness Workspace" + short_description: "Configure an ITSM optimization workspace" + default_prompt: "Use $workspace-init to inspect this repository, collect the ITSM-bench experiment configuration one question at a time, and initialize its primary agent harness." +policy: + allow_implicit_invocation: true diff --git a/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md new file mode 100644 index 0000000..90ed88a --- /dev/null +++ b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md @@ -0,0 +1,8 @@ +# Placeholder - do not edit this file directly. + +`workspace_prepare()` populates this file from: + +- `program_templates/base.md` (shared sections) +- `program_templates/itsm.md` (ITSM-bench sections) + +`experiment_config.yaml` must select `itsm-bench`.