From 0b4e15b43c3f63227a554cc78e1a751b8a62f475 Mon Sep 17 00:00:00 2001 From: divo12 Date: Fri, 28 Aug 2026 03:17:05 +0530 Subject: [PATCH 1/5] add workspace initialization skill --- .../openflywheel/.codex-plugin/plugin.json | 12 ++-- .../skills/workspace-init/SKILL.md | 71 +++++++++++++++++++ .../skills/workspace-init/agents/openai.yaml | 6 ++ .../skills/workspace-init/assets/PROGRAM.md | 9 +++ 4 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 plugins/openflywheel/skills/workspace-init/SKILL.md create mode 100644 plugins/openflywheel/skills/workspace-init/agents/openai.yaml create mode 100644 plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 72e2957..2d72c16 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", - "version": "0.2.0", - "description": "Query Langfuse trajectories and record authoritative verifier outcomes.", + "version": "0.3.0", + "description": "Initialize harness workspaces, query Langfuse trajectories, and record authoritative verifier outcomes.", "author": { "name": "OpenFlyWheel" }, @@ -10,14 +10,14 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Query traces and record verifier outcomes", - "longDescription": "Select and inspect bounded Langfuse trajectory evidence, then record authoritative external-verifier outcomes on exact traces.", + "shortDescription": "Initialize workspaces and inspect traces", + "longDescription": "Initialize an agent-harness optimization workspace, select and inspect bounded Langfuse trajectory evidence, and record authoritative external-verifier outcomes on exact traces.", "developerName": "OpenFlyWheel", "category": "Productivity", "capabilities": ["Read", "Write"], "defaultPrompt": [ - "Inspect this Langfuse trace with the fewest bounded queries.", - "Record this completed verifier outcome on its exact trace." + "Read this repository, identify the primary agent harness the user wants to improve, and use $workspace-init to collect its experiment configuration and perform the initial setup.", + "When PROGRAM.md is ready, read it and continue the optimization loop until its declared goal or stop condition is met." ] }, "mcpServers": "./.mcp.json" diff --git a/plugins/openflywheel/skills/workspace-init/SKILL.md b/plugins/openflywheel/skills/workspace-init/SKILL.md new file mode 100644 index 0000000..e8fd83c --- /dev/null +++ b/plugins/openflywheel/skills/workspace-init/SKILL.md @@ -0,0 +1,71 @@ +--- +name: workspace-init +description: Initialize an OpenFlywheel optimization workspace by inspecting an agent-harness repository, collecting its experiment configuration one field at a time, creating the managed PROGRAM.md placeholder, and handing preparation to workspace_prepare. Use when onboarding a primary harness; do not use for an already-prepared workspace, ordinary trace queries, or outcome recording. +--- + +# Workspace Init + +Initialize one primary agent harness without starting optimization prematurely. + +## 1. Discover the harness + +Read the repository before asking questions. Identify likely prompts, tools, agent loops, +benchmark runners, verifiers, observability wiring, and editable harness files. Present the +most likely primary harness and ask the user to confirm it. Do not choose among materially +different harnesses without confirmation. + +## 2. Collect the experiment configuration + +Ask one focused question at a time. Infer repository facts first and recommend a default +when the evidence supports one. Collect, in order: + +1. Harness root and explicitly editable files or directories. +2. Optimization goal, primary metric, target, and stopping condition. Keep quality, cost, + and latency constraints separate rather than hiding them in one average. +3. Benchmark kind, root, task selection or split, and expected task count. +4. Authoritative verifier, reward interpretation, and pass threshold. +5. Frozen model, reasoning effort, concurrency, per-task timeout, retry policy, and budget. +6. Langfuse environment, release, and session naming rule. + +Never request secret values in chat or write them into configuration. Check only whether +the required environment-variable names are present. + +Summarize the complete proposed experiment and obtain confirmation before writing files or +starting a potentially costly baseline. Then create `/experiment_config.yaml` +using only the confirmed values. + +## 3. Create the managed program placeholder + +Copy [assets/PROGRAM.md](assets/PROGRAM.md) byte-for-byte to +`/PROGRAM.md`. Do not overwrite an existing different `PROGRAM.md`; stop and +ask whether the existing program should be preserved or replaced. + +Do not compose the final program yourself. Call `workspace_prepare` with the confirmed +experiment configuration. That tool owns validation, baseline execution, result parsing, +and deterministic composition from the shared and benchmark-specific program templates. + +`workspace_prepare` is long-running and re-entrant: + +- On `running`, retain the preparation ID and poll the same request after the returned + interval. Never start a second baseline. +- On `failed`, report its typed recovery instruction and stop at its declared stop + condition. +- On `ready`, retain the baseline artifacts and confirm that `PROGRAM.md` is no longer the + placeholder. + +If `workspace_prepare` is unavailable, stop after the confirmed configuration and +placeholder. Report that workspace preparation is not installed. Do not replace the tool +with an improvised shell command. + +## 4. Hand off to the optimization program + +When preparation is `ready`, start a fresh Codex session with exactly this task: + +```text +Read PROGRAM.md and start the optimization loop. +The baseline is already recorded. Start from step 2 (analyze failures). +``` + +The generated program is authoritative for editable files, gates, metrics, budgets, and +stopping conditions. Do not rerun the baseline, weaken the verifier, expose held-out traces, +or continue past the declared goal or stop condition. diff --git a/plugins/openflywheel/skills/workspace-init/agents/openai.yaml b/plugins/openflywheel/skills/workspace-init/agents/openai.yaml new file mode 100644 index 0000000..5ba70c6 --- /dev/null +++ b/plugins/openflywheel/skills/workspace-init/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Initialize Harness Workspace" + short_description: "Configure an optimization workspace" + default_prompt: "Use $workspace-init to inspect this repository, collect the experiment configuration one question at a time, and initialize its primary agent harness." +policy: + allow_implicit_invocation: true diff --git a/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md new file mode 100644 index 0000000..ddf4407 --- /dev/null +++ b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md @@ -0,0 +1,9 @@ +# Placeholder - do not edit this file directly. + +`workspace_prepare()` populates this file from: + +- `program_templates/base.md` (shared sections) +- `program_templates/.md` (benchmark-specific sections, such as + `tau_bench.md`, `terminal_bench.md`, `bird_interact.md`, or `itsm_bench.md`) + +The benchmark is selected by `experiment_config.yaml`. From 3b2ba74cd6c0448bdf47047949f8f0f3dadd94d2 Mon Sep 17 00:00:00 2001 From: divo12 Date: Fri, 28 Aug 2026 03:21:11 +0530 Subject: [PATCH 2/5] scope workspace init to ITSM bench --- plugins/openflywheel/.codex-plugin/plugin.json | 8 ++++---- plugins/openflywheel/skills/workspace-init/SKILL.md | 11 +++++++---- .../skills/workspace-init/agents/openai.yaml | 6 +++--- .../skills/workspace-init/assets/PROGRAM.md | 5 ++--- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 2d72c16..1ed1b09 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", "version": "0.3.0", - "description": "Initialize harness workspaces, query Langfuse trajectories, and record authoritative verifier outcomes.", + "description": "Initialize ITSM-bench harness workspaces, query Langfuse trajectories, and record authoritative verifier outcomes.", "author": { "name": "OpenFlyWheel" }, @@ -10,13 +10,13 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Initialize workspaces and inspect traces", - "longDescription": "Initialize an agent-harness optimization workspace, select and inspect bounded Langfuse trajectory evidence, and record authoritative external-verifier outcomes on exact traces.", + "shortDescription": "Initialize ITSM workspaces and inspect traces", + "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, select and inspect bounded Langfuse trajectory evidence, and record authoritative external-verifier outcomes on exact traces.", "developerName": "OpenFlyWheel", "category": "Productivity", "capabilities": ["Read", "Write"], "defaultPrompt": [ - "Read this repository, identify the primary agent harness the user wants to improve, and use $workspace-init to collect its experiment configuration and perform the initial setup.", + "Read this repository, identify the primary agent harness the user wants to evaluate with ITSM-bench, and use $workspace-init to collect its experiment configuration and perform the initial setup.", "When PROGRAM.md is ready, read it and continue the optimization loop until its declared goal or stop condition is met." ] }, diff --git a/plugins/openflywheel/skills/workspace-init/SKILL.md b/plugins/openflywheel/skills/workspace-init/SKILL.md index e8fd83c..4f781dd 100644 --- a/plugins/openflywheel/skills/workspace-init/SKILL.md +++ b/plugins/openflywheel/skills/workspace-init/SKILL.md @@ -1,11 +1,13 @@ --- name: workspace-init -description: Initialize an OpenFlywheel optimization workspace by inspecting an agent-harness repository, collecting its experiment configuration one field at a time, creating the managed PROGRAM.md placeholder, and handing preparation to workspace_prepare. Use when onboarding a primary harness; do not use for an already-prepared workspace, ordinary trace queries, or outcome recording. +description: Initialize an OpenFlywheel ITSM-bench optimization workspace by inspecting an agent-harness repository, collecting its experiment configuration one field at a time, creating the managed PROGRAM.md placeholder, and handing preparation to workspace_prepare. Use when onboarding a primary harness for ITSM-bench; do not use for other benchmarks, an already-prepared workspace, ordinary trace queries, or outcome recording. --- # Workspace Init -Initialize one primary agent harness without starting optimization prematurely. +Initialize one primary agent harness for ITSM-bench without starting optimization +prematurely. This skill supports only `benchmark: itsm-bench`. Stop and report the +unsupported benchmark if the user requests anything else. ## 1. Discover the harness @@ -22,7 +24,7 @@ when the evidence supports one. Collect, in order: 1. Harness root and explicitly editable files or directories. 2. Optimization goal, primary metric, target, and stopping condition. Keep quality, cost, and latency constraints separate rather than hiding them in one average. -3. Benchmark kind, root, task selection or split, and expected task count. +3. ITSM-bench root, Harbor task manifest or selection, and expected task count. 4. Authoritative verifier, reward interpretation, and pass threshold. 5. Frozen model, reasoning effort, concurrency, per-task timeout, retry policy, and budget. 6. Langfuse environment, release, and session naming rule. @@ -42,7 +44,8 @@ ask whether the existing program should be preserved or replaced. Do not compose the final program yourself. Call `workspace_prepare` with the confirmed experiment configuration. That tool owns validation, baseline execution, result parsing, -and deterministic composition from the shared and benchmark-specific program templates. +and deterministic composition from `program_templates/base.md` and +`program_templates/itsm_bench.md`. `workspace_prepare` is long-running and re-entrant: diff --git a/plugins/openflywheel/skills/workspace-init/agents/openai.yaml b/plugins/openflywheel/skills/workspace-init/agents/openai.yaml index 5ba70c6..237e08d 100644 --- a/plugins/openflywheel/skills/workspace-init/agents/openai.yaml +++ b/plugins/openflywheel/skills/workspace-init/agents/openai.yaml @@ -1,6 +1,6 @@ interface: - display_name: "Initialize Harness Workspace" - short_description: "Configure an optimization workspace" - default_prompt: "Use $workspace-init to inspect this repository, collect the experiment configuration one question at a time, and initialize its primary agent harness." + display_name: "Initialize ITSM Harness Workspace" + short_description: "Configure an ITSM optimization workspace" + default_prompt: "Use $workspace-init to inspect this repository, collect the ITSM-bench experiment configuration one question at a time, and initialize its primary agent harness." policy: allow_implicit_invocation: true diff --git a/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md index ddf4407..b259cb7 100644 --- a/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md +++ b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md @@ -3,7 +3,6 @@ `workspace_prepare()` populates this file from: - `program_templates/base.md` (shared sections) -- `program_templates/.md` (benchmark-specific sections, such as - `tau_bench.md`, `terminal_bench.md`, `bird_interact.md`, or `itsm_bench.md`) +- `program_templates/itsm_bench.md` (ITSM-bench sections) -The benchmark is selected by `experiment_config.yaml`. +`experiment_config.yaml` must select `itsm-bench`. From dd5c7f6458710c0091df98000c0a6b073303c574 Mon Sep 17 00:00:00 2001 From: divo12 Date: Fri, 28 Aug 2026 03:23:31 +0530 Subject: [PATCH 3/5] add ITSM program templates --- .../openflywheel/program_templates/base.md | 73 +++++++++++++++++++ .../openflywheel/program_templates/itsm.md | 50 +++++++++++++ .../skills/workspace-init/SKILL.md | 2 +- .../skills/workspace-init/assets/PROGRAM.md | 2 +- 4 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 plugins/openflywheel/program_templates/base.md create mode 100644 plugins/openflywheel/program_templates/itsm.md diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md new file mode 100644 index 0000000..b1873c6 --- /dev/null +++ b/plugins/openflywheel/program_templates/base.md @@ -0,0 +1,73 @@ +# OpenFlywheel Agent Program + +This file is generated by `workspace_prepare`. Do not edit it directly. + +## Mission + +Improve the connected agent harness until the goal in `experiment_config.yaml` is met, +while respecting its quality, cost, latency, budget, and stopping constraints. + +The baseline has already been recorded. Begin at step 2; do not rerun the unchanged +baseline. + +## Authority + +- `experiment_config.yaml` defines the harness, editable surface, frozen controls, goal, + benchmark, verifier, budget, and stopping conditions. +- The external verifier is authoritative for task outcome. The agent's completion claim + is not proof of success. +- Langfuse is the source of truth for trajectories, usage, cost, and latency. +- Missing verifier evidence is `unverified`, not failure and not success. + +## Editable and frozen surfaces + +Edit only paths explicitly allowed by `experiment_config.yaml`. Never change the benchmark, +held-out tasks, verifier, model, reasoning budget, observability identity, or this program +to improve a score. + +Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control +flow changes unless the evidence requires the combination. + +## Optimization loop + +### 2. Analyze failures + +Start from verifier-backed failed outcomes. Use bounded trace queries to locate relevant +evidence, then inspect only the spans needed to explain the observed behavior. Do not load +or copy complete traces when filters answer the question, and do not inspect held-out +trajectory content. + +### 3. Form one hypothesis + +State the failure pattern, supporting trace and verifier evidence, proposed harness change, +expected improvement, and possible regressions. Stop if the evidence cannot distinguish +between materially different changes. + +### 4. Improve the harness + +Make the smallest change within the declared editable surface. Preserve frozen controls and +unrelated user changes. + +### 5. Gate the change + +Run only the prepared experiment command and gates declared by the workspace. Compare +task-level verifier outcomes and report quality, cost, and latency separately. Missing or +errored trials remain visible and cannot disappear from the denominator. + +### 6. Keep or revert + +Keep the change only when the configured gate admits it. Otherwise revert only the current +iteration's harness edit, retain the evidence, and try a different hypothesis. Never weaken +the gate to admit a candidate. + +### 7. Repeat + +Return to step 2 with the newly recorded run. Stop when the configured goal is met, the +budget or iteration limit is exhausted, the no-improvement condition is reached, or required +authoritative evidence is unavailable. + +## Final report + +Report the accepted harness revision, verifier-backed quality, cost, latency, remaining +unverified trials, iteration count, and exact stopping reason. Do not claim improvement from +an agent-authored summary alone. diff --git a/plugins/openflywheel/program_templates/itsm.md b/plugins/openflywheel/program_templates/itsm.md new file mode 100644 index 0000000..a75935d --- /dev/null +++ b/plugins/openflywheel/program_templates/itsm.md @@ -0,0 +1,50 @@ +## ITSM-bench Instructions + +This program supports only `benchmark: itsm-bench`. + +## Record the prepared baseline outcomes + +Before diagnosing failures, process every terminal Harbor trial from the prepared baseline: + +1. Read the trial's `result.json` and verifier artifacts. +2. When `verifier_result.rewards.reward` exists, map the result directly: + - `1.0` -> `pass` with score `1.0`. + - `0.0` -> `fail` with score `0.0`. +3. Use the task directory name as `task_id`, `itsm-bench@` as + `verifier_id`, and the verifier completion time as `evaluated_at`. +4. Resolve exactly one Langfuse trace using the prepared session, environment, release, + and the trial's agent-execution time window. +5. Call `record_outcome` with stable verifier evidence references and retain its score + receipt. + +If the verifier result is absent, record nothing. If trace selection is empty or ambiguous, +record nothing and report the mapping blocker. `record_outcome` is the only permitted +Langfuse write. + +## Analyze failed ITSM trajectories + +Use the trace tools in the smallest sufficient sequence: + +1. `list_traces` selects candidate traces for the prepared session. +2. `get_trace_schema` skims structure without loading input or output. +3. `query_spans` selects exact observations by ID, tool, type, UTC range, error flag, or + deterministic text filter. +4. `get_span_context` retrieves bounded raw context only for a selected span. + +An intermediate tool error is evidence, not an outcome failure, when the agent recovered +and the verifier passed. A technically clean trajectory is still a failure when the ITSM +verifier shows that the required environment state was not achieved. + +## ITSM optimization constraints + +- Treat the ITSM verifier score as the authoritative quality metric. +- Read cost and latency from Langfuse; do not write separate cost or latency scores. +- Preserve least-privilege behavior and verify environment state before declaring success. +- Do not expose held-out ITSM tasks or verifier internals to the harness being optimized. +- Run trials sequentially when deterministic trace-to-trial mapping depends on execution + windows. + +## ITSM iteration report + +For every candidate run, report verifier passes, verifier failures, unverified trials, +outcome receipts, trace-mapping blockers, total Langfuse cost, latency, and the gate decision. diff --git a/plugins/openflywheel/skills/workspace-init/SKILL.md b/plugins/openflywheel/skills/workspace-init/SKILL.md index 4f781dd..87aa146 100644 --- a/plugins/openflywheel/skills/workspace-init/SKILL.md +++ b/plugins/openflywheel/skills/workspace-init/SKILL.md @@ -45,7 +45,7 @@ ask whether the existing program should be preserved or replaced. Do not compose the final program yourself. Call `workspace_prepare` with the confirmed experiment configuration. That tool owns validation, baseline execution, result parsing, and deterministic composition from `program_templates/base.md` and -`program_templates/itsm_bench.md`. +`program_templates/itsm.md`. `workspace_prepare` is long-running and re-entrant: diff --git a/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md index b259cb7..90ed88a 100644 --- a/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md +++ b/plugins/openflywheel/skills/workspace-init/assets/PROGRAM.md @@ -3,6 +3,6 @@ `workspace_prepare()` populates this file from: - `program_templates/base.md` (shared sections) -- `program_templates/itsm_bench.md` (ITSM-bench sections) +- `program_templates/itsm.md` (ITSM-bench sections) `experiment_config.yaml` must select `itsm-bench`. From d5fa007119977612824d50bac9185c05e19ca45f Mon Sep 17 00:00:00 2001 From: divo12 Date: Fri, 28 Aug 2026 03:34:00 +0530 Subject: [PATCH 4/5] handle ambiguous ITSM verifier rewards --- plugins/openflywheel/program_templates/itsm.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/openflywheel/program_templates/itsm.md b/plugins/openflywheel/program_templates/itsm.md index a75935d..5a4e957 100644 --- a/plugins/openflywheel/program_templates/itsm.md +++ b/plugins/openflywheel/program_templates/itsm.md @@ -6,10 +6,17 @@ This program supports only `benchmark: itsm-bench`. Before diagnosing failures, process every terminal Harbor trial from the prepared baseline: -1. Read the trial's `result.json` and verifier artifacts. -2. When `verifier_result.rewards.reward` exists, map the result directly: - - `1.0` -> `pass` with score `1.0`. - - `0.0` -> `fail` with score `0.0`. +1. Read the trial's `result.json`, `exception_info`, verifier status, and verifier artifacts. +2. Map a reward only when the verifier completed and the trial has no execution or verifier + error: + - Exact `1.0` -> `pass` with score `1.0`. + - Exact `0.0` -> `fail` with score `0.0`. + - Any other present reward -> record nothing and report an unsupported-reward mapping + blocker. + - An explicit authoritative `abstain` or `error` verdict -> preserve that verdict without + a score. + - An exception, verifier error, or missing verifier result without an explicit authoritative + verdict -> record nothing and report the trial as unverified. 3. Use the task directory name as `task_id`, `itsm-bench@` as `verifier_id`, and the verifier completion time as `evaluated_at`. 4. Resolve exactly one Langfuse trace using the prepared session, environment, release, From bb399b2614b2e206f31a37c75618fcf0227593e3 Mon Sep 17 00:00:00 2001 From: divo12 Date: Fri, 28 Aug 2026 03:39:06 +0530 Subject: [PATCH 5/5] report unsupported ITSM rewards --- plugins/openflywheel/program_templates/itsm.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/openflywheel/program_templates/itsm.md b/plugins/openflywheel/program_templates/itsm.md index 5a4e957..c43a30d 100644 --- a/plugins/openflywheel/program_templates/itsm.md +++ b/plugins/openflywheel/program_templates/itsm.md @@ -54,4 +54,5 @@ verifier shows that the required environment state was not achieved. ## ITSM iteration report For every candidate run, report verifier passes, verifier failures, unverified trials, -outcome receipts, trace-mapping blockers, total Langfuse cost, latency, and the gate decision. +outcome receipts, trace-mapping blockers, the count and values of unsupported-reward mapping +blockers, total Langfuse cost, latency, and the gate decision.