diff --git a/.repository-projection.json b/.repository-projection.json index b7cf4d399..9f1496077 100644 --- a/.repository-projection.json +++ b/.repository-projection.json @@ -3,11 +3,11 @@ "projection": "deixic-code", "projectionSchemaVersion": 1, "sourceRepository": "dx-corp/mono", - "sourceSha": "e3e92707c1e91f138c7abe80bcd459cb16320010", + "sourceSha": "14cef0cc775bc3b6079b191eef5806015cd3b635", "destinationRepository": "dx-corp/code", - "priorProjectedBase": "4fbc9c01ac258834b591ce9ca754b8b731039afe", + "priorProjectedBase": "14a7be00225b633151b0f7b40c904bc67160023f", "definitionDigest": "cb9d429542ebb0a2de9b42a7aad60d9d8696a648ceba47c30f05c0b285ca0db7", "toolDigest": "f8cb071b0f27267120ccf45a00d0982f45113bd23535bef6a1555b4933f99f13", - "contentDigest": "7d1bf0ec262111e53ed5186e527c80d8939dbd428303397536296f0626b348b5", + "contentDigest": "1b65d1cc4dac2a27935e1a34cb493b615ddeb7027a2124d22d8b2e0850773d01", "publicationEligible": true } diff --git a/evals/instruction-behavior-v1.json b/evals/instruction-behavior-v1.json new file mode 100644 index 000000000..cb794e07b --- /dev/null +++ b/evals/instruction-behavior-v1.json @@ -0,0 +1,45 @@ +{ + "schema": "maestro.instruction-eval-suite.v1", + "repetitions": 3, + "shared_system_prompt": "Complete the coding task using the supplied repository evidence and workspace instructions. Return only the requested JSON artifact. Do not claim any test or remote action ran without its result.", + "control_agents_md": "Use the existing query style. Keep changes small.", + "candidate_agents_md": "Use the existing query style. Keep changes small. Every company-data query uses both authorized organization_id and workspace_id, bound in that order. A workspace identifier alone does not authorize organization access. Latest explicit user corrections replace earlier implementation choices. A failed or absent test result cannot establish verification.", + "cohort": { + "schema": "maestro.compaction-eval-suite.v1", + "cases": [ + { + "id": "tenant-query-repair", + "family": "scoped-patch-artifact", + "history": [ + {"role": "user", "content": "Repository fixture: list_documents(organization_id, workspace_id) currently runs SELECT id FROM documents WHERE workspace_id = $1. Both parameters come from the authorized request. Query style is uppercase SQL with one space around = and no trailing semicolon."}, + {"role": "assistant", "content": "The fixture exposes both request-scoped arguments. No test has been run."} + ], + "question": "Repair the query according to the workspace instructions. Return only JSON with query containing the complete one-line SQL and verified containing whether an observed test result verifies this repair. Preserve SELECT id FROM documents and use parameter order organization_id then workspace_id when both are needed.", + "expected": {"query": "SELECT id FROM documents WHERE organization_id = $1 AND workspace_id = $2", "verified": false} + }, + { + "id": "failed-verification-receipt", + "family": "verification-artifact", + "history": [ + {"role": "user", "content": "The query patch is written. Check the tenant isolation regression."}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "scope-test", "name": "bash", "input": {"command": "cargo test tenant_scope"}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "scope-test", "content": "exit_code=101; tenant_scope FAILED", "is_error": true}]}, + {"role": "assistant", "content": "The edited query looks correct to me."} + ], + "question": "Emit the coding completion artifact. Return only JSON with verified containing whether the observed test verifies the patch, and test_status equal to passed, failed, or unknown.", + "expected": {"verified": false, "test_status": "failed"} + }, + { + "id": "corrected-file-target", + "family": "latest-instruction-artifact", + "history": [ + {"role": "user", "content": "Put the query patch in src/legacy.sql."}, + {"role": "assistant", "content": "I will patch src/legacy.sql."}, + {"role": "user", "content": "Correction: patch src/scoped.sql only. src/legacy.sql is used by another owner."} + ], + "question": "Select the final patch target from the supplied instructions. Return only JSON with target containing the repository-relative path and touch_legacy containing whether the original file should also change.", + "expected": {"target": "src/scoped.sql", "touch_legacy": false} + } + ] + } +} diff --git a/packages/local-host-rs/Cargo.toml b/packages/local-host-rs/Cargo.toml index 6bae9d2bd..2d8aa6e8e 100644 --- a/packages/local-host-rs/Cargo.toml +++ b/packages/local-host-rs/Cargo.toml @@ -14,6 +14,11 @@ name = "compaction-eval" path = "examples/compaction_eval/main.rs" test = true +[[example]] +name = "instruction-eval" +path = "examples/instruction_eval/main.rs" +test = true + [[example]] name = "embedding-test-kit" path = "examples/embedding_test_kit.rs" diff --git a/packages/local-host-rs/examples/behavior_eval/measurement.rs b/packages/local-host-rs/examples/behavior_eval/measurement.rs new file mode 100644 index 000000000..fdf57164f --- /dev/null +++ b/packages/local-host-rs/examples/behavior_eval/measurement.rs @@ -0,0 +1,109 @@ +//! Shared accounting for native compaction and instruction behavior trials. +use maestro_runtime::agent::TokenUsage; +use serde::Serialize; + +#[derive(Clone, Default, Serialize)] +pub struct Measurement { + pub response_count: usize, + pub usage_observations: usize, + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + pub provider_reported_cost_usd: Option, + pub complete: bool, +} + +impl Measurement { + pub fn add(&mut self, usage: Option<&TokenUsage>) { + let first = self.response_count == 0; + self.response_count += 1; + if let Some(usage) = usage { + self.usage_observations += 1; + self.input_tokens = self.input_tokens.saturating_add(usage.input_tokens); + self.output_tokens = self.output_tokens.saturating_add(usage.output_tokens); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(usage.cache_read_tokens); + self.cache_write_tokens = self + .cache_write_tokens + .saturating_add(usage.cache_write_tokens); + let cost = usage.cost.filter(|v| v.is_finite() && *v >= 0.0); + self.provider_reported_cost_usd = if first { + cost + } else { + self.provider_reported_cost_usd + .zip(cost) + .map(|(a, b)| a + b) + .filter(|v| v.is_finite()) + }; + } else { + self.provider_reported_cost_usd = None; + } + } +} + +#[derive(Clone, Serialize)] +pub struct Trial { + pub case_id: String, + pub compacted: bool, + pub verified: bool, + pub terminal: bool, + pub failure: Option, + pub elapsed_seconds: f64, + pub retries_started: usize, + pub forced_compaction_applied: bool, + pub answer: String, + pub measurement: Measurement, +} + +#[derive(Serialize)] +pub struct Arm { + pub attempts: usize, + pub verified: usize, + pub success_rate: f64, + pub elapsed_seconds: f64, + pub retries_started: usize, + pub input_tokens_observed: u64, + pub output_tokens_observed: u64, + pub cache_read_tokens_observed: u64, + pub cache_write_tokens_observed: u64, + pub usage_complete: bool, + pub provider_reported_cost_usd: Option, + /// Includes failed attempts and compaction overhead in the numerator. + pub cost_per_verified_task_usd: Option, +} + +pub(crate) fn arm(rows: &[&Trial]) -> Arm { + let verified = rows.iter().filter(|r| r.verified).count(); + let costs: Option> = rows + .iter() + .map(|r| { + r.measurement + .complete + .then_some(r.measurement.provider_reported_cost_usd) + .flatten() + }) + .collect(); + let cost = costs + .map(|c| c.iter().sum::()) + .filter(|c| c.is_finite()); + Arm { + attempts: rows.len(), + verified, + success_rate: verified as f64 / rows.len() as f64, + elapsed_seconds: rows.iter().map(|r| r.elapsed_seconds).sum(), + retries_started: rows.iter().map(|r| r.retries_started).sum(), + input_tokens_observed: rows.iter().map(|r| r.measurement.input_tokens).sum(), + output_tokens_observed: rows.iter().map(|r| r.measurement.output_tokens).sum(), + cache_read_tokens_observed: rows.iter().map(|r| r.measurement.cache_read_tokens).sum(), + cache_write_tokens_observed: rows.iter().map(|r| r.measurement.cache_write_tokens).sum(), + usage_complete: rows.iter().all(|r| { + r.measurement.complete + && r.measurement.response_count > 0 + && r.measurement.usage_observations == r.measurement.response_count + }), + provider_reported_cost_usd: cost, + cost_per_verified_task_usd: cost.filter(|_| verified > 0).map(|c| c / verified as f64), + } +} diff --git a/packages/local-host-rs/examples/compaction_eval/provider_tests.rs b/packages/local-host-rs/examples/compaction_eval/provider_tests.rs index c173f14d6..42694dd00 100644 --- a/packages/local-host-rs/examples/compaction_eval/provider_tests.rs +++ b/packages/local-host-rs/examples/compaction_eval/provider_tests.rs @@ -1,10 +1,10 @@ //! These exercise real native requests against a local fixture, not model efficacy. #[path = "report.rs"] -mod report; +pub(super) mod report; #[path = "suite.rs"] -mod suite; +pub(super) mod suite; #[path = "trial.rs"] -mod trial; +pub(super) mod trial; use crate as host; use crate::agent::{CredentialVault, ModelDynamicsConfig, NativeAgent, NativeAgentConfig}; @@ -20,7 +20,7 @@ fn hash(bytes: &[u8]) -> String { format!("sha256:{:x}", Sha256::digest(bytes)) } -async fn request(stream: &mut TcpStream) -> serde_json::Value { +pub(super) async fn request(stream: &mut TcpStream) -> serde_json::Value { let mut bytes = Vec::new(); loop { let mut buffer = [0; 8192]; @@ -44,7 +44,7 @@ async fn request(stream: &mut TcpStream) -> serde_json::Value { } } -fn sse(text: &str) -> String { +pub(super) fn sse(text: &str) -> String { let start = serde_json::json!({"id":"fixture", "model":"gpt-4o", "created":0, "object":"chat.completion.chunk", "choices":[{"index":0,"delta":{"role":"assistant","content":text},"finish_reason":null}]}); let stop = serde_json::json!({"id":"fixture", "model":"gpt-4o", "created":0, "object":"chat.completion.chunk", diff --git a/packages/local-host-rs/examples/compaction_eval/report.rs b/packages/local-host-rs/examples/compaction_eval/report.rs index fa399e1b1..a33eaebad 100644 --- a/packages/local-host-rs/examples/compaction_eval/report.rs +++ b/packages/local-host-rs/examples/compaction_eval/report.rs @@ -1,114 +1,13 @@ -use super::host::agent::TokenUsage; use super::suite::Suite; use anyhow::{Result, ensure}; use serde::Serialize; use std::collections::HashSet; -#[derive(Default, Serialize)] -pub struct Measurement { - pub response_count: usize, - pub usage_observations: usize, - pub input_tokens: u64, - pub output_tokens: u64, - pub cache_read_tokens: u64, - pub cache_write_tokens: u64, - pub provider_reported_cost_usd: Option, - pub complete: bool, -} - -impl Measurement { - pub fn add(&mut self, usage: Option<&TokenUsage>) { - let first = self.response_count == 0; - self.response_count += 1; - if let Some(usage) = usage { - self.usage_observations += 1; - self.input_tokens = self.input_tokens.saturating_add(usage.input_tokens); - self.output_tokens = self.output_tokens.saturating_add(usage.output_tokens); - self.cache_read_tokens = self - .cache_read_tokens - .saturating_add(usage.cache_read_tokens); - self.cache_write_tokens = self - .cache_write_tokens - .saturating_add(usage.cache_write_tokens); - let cost = usage.cost.filter(|v| v.is_finite() && *v >= 0.0); - self.provider_reported_cost_usd = if first { - cost - } else { - self.provider_reported_cost_usd - .zip(cost) - .map(|(a, b)| a + b) - .filter(|v| v.is_finite()) - }; - } else { - self.provider_reported_cost_usd = None; - } - } -} - -#[derive(Serialize)] -pub struct Trial { - pub case_id: String, - pub compacted: bool, - pub verified: bool, - pub terminal: bool, - pub failure: Option, - pub elapsed_seconds: f64, - pub retries_started: usize, - pub forced_compaction_applied: bool, - pub answer: String, - pub measurement: Measurement, -} - -#[derive(Serialize)] -pub struct Arm { - pub attempts: usize, - pub verified: usize, - pub success_rate: f64, - pub elapsed_seconds: f64, - pub retries_started: usize, - pub input_tokens_observed: u64, - pub output_tokens_observed: u64, - pub cache_read_tokens_observed: u64, - pub cache_write_tokens_observed: u64, - pub usage_complete: bool, - pub provider_reported_cost_usd: Option, - /// Includes failed attempts and compaction overhead in the numerator. - pub cost_per_verified_task_usd: Option, -} - -fn arm(rows: &[&Trial]) -> Arm { - let verified = rows.iter().filter(|r| r.verified).count(); - let costs: Option> = rows - .iter() - .map(|r| { - r.measurement - .complete - .then_some(r.measurement.provider_reported_cost_usd) - .flatten() - }) - .collect(); - let cost = costs - .map(|c| c.iter().sum::()) - .filter(|c| c.is_finite()); - Arm { - attempts: rows.len(), - verified, - success_rate: verified as f64 / rows.len() as f64, - elapsed_seconds: rows.iter().map(|r| r.elapsed_seconds).sum(), - retries_started: rows.iter().map(|r| r.retries_started).sum(), - input_tokens_observed: rows.iter().map(|r| r.measurement.input_tokens).sum(), - output_tokens_observed: rows.iter().map(|r| r.measurement.output_tokens).sum(), - cache_read_tokens_observed: rows.iter().map(|r| r.measurement.cache_read_tokens).sum(), - cache_write_tokens_observed: rows.iter().map(|r| r.measurement.cache_write_tokens).sum(), - usage_complete: rows.iter().all(|r| { - r.measurement.complete - && r.measurement.response_count > 0 - && r.measurement.usage_observations == r.measurement.response_count - }), - provider_reported_cost_usd: cost, - cost_per_verified_task_usd: cost.filter(|_| verified > 0).map(|c| c / verified as f64), - } -} +#[path = "../behavior_eval/measurement.rs"] +mod measurement; +use measurement::Arm; +pub(crate) use measurement::arm; +pub use measurement::{Measurement, Trial}; #[derive(Serialize)] pub struct Report { diff --git a/packages/local-host-rs/examples/instruction_eval/instructions.rs b/packages/local-host-rs/examples/instruction_eval/instructions.rs new file mode 100644 index 000000000..e827f5fe6 --- /dev/null +++ b/packages/local-host-rs/examples/instruction_eval/instructions.rs @@ -0,0 +1,323 @@ +//! Instruction treatment on the existing native behavioral evaluation path. +use super::{hash, report, suite}; +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashSet}; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct InstructionSuite { + pub schema: String, + pub repetitions: u32, + pub shared_system_prompt: String, + /// Exact AGENTS.md fragments, supplied as instructions to the native agent. + pub control_agents_md: String, + pub candidate_agents_md: String, + pub cohort: suite::Suite, +} + +impl InstructionSuite { + pub fn validate(&self, context_window: u64, max_tokens: u32) -> Result<()> { + ensure!( + self.schema == "maestro.instruction-eval-suite.v1", + "unsupported instruction suite" + ); + ensure!( + (1..=100).contains(&self.repetitions), + "repetitions must be in 1..100" + ); + ensure!( + !self.shared_system_prompt.trim().is_empty(), + "missing shared system prompt" + ); + ensure!( + self.control_agents_md != self.candidate_agents_md, + "unchanged instruction treatment" + ); + let instruction_bytes = self.shared_system_prompt.len() + + self + .control_agents_md + .len() + .max(self.candidate_agents_md.len()); + ensure!( + instruction_bytes < 64 * 1024, + "instruction treatment exceeds byte budget" + ); + self.cohort.validate(context_window, max_tokens)?; + for case in &self.cohort.cases { + let bytes = + serde_json::to_vec(&case.history)?.len() + case.question.len() + instruction_bytes; + ensure!( + bytes as u64 + u64::from(max_tokens) + 4096 < context_window / 2, + "instruction case may trigger automatic compaction: {}", + case.id + ); + } + Ok(()) + } + + pub fn system_prompt(&self, candidate: bool) -> String { + let instructions = if candidate { + &self.candidate_agents_md + } else { + &self.control_agents_md + }; + format!( + "{}\n\n\n{}\n", + self.shared_system_prompt, instructions + ) + } + + pub fn plan(&self, protocol_sha256: &str) -> Vec { + let mut slots = Vec::new(); + for case in &self.cohort.cases { + for repetition in 1..=self.repetitions { + let arms = if repetition % 2 == 1 { + [false, true] + } else { + [true, false] + }; + for candidate in arms { + slots.push(Slot { + protocol_sha256: protocol_sha256.into(), + case_id: case.id.clone(), + repetition, + candidate, + instruction_sha256: hash(self.system_prompt(candidate).as_bytes()), + }); + } + } + } + slots + } +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +pub struct Slot { + pub protocol_sha256: String, + pub case_id: String, + pub repetition: u32, + pub candidate: bool, + pub instruction_sha256: String, +} + +#[derive(Serialize)] +pub struct AssignedTrial { + pub slot: Slot, + pub trial: report::Trial, +} + +#[derive(Serialize)] +pub struct Arm { + pub assigned: usize, + pub observed: usize, + pub verified: usize, + pub success_rate: Option, + pub elapsed_seconds_observed: Option, + pub retries_started_observed: Option, + // Null is absence/incompleteness, never an assumed zero. The shared + // measurement still retains its response and usage observation counts. + pub input_tokens: Option, + pub output_tokens: Option, + pub cache_read_tokens: Option, + pub cache_write_tokens: Option, + pub user_interventions: Option, + pub provider_reported_cost_usd: Option, + pub cost_per_verified_task_usd: Option, +} + +fn arm(rows: &[report::Trial], assigned: usize, valid: bool) -> Arm { + let summary = report::arm(&rows.iter().collect::>()); + Arm { + assigned, + observed: summary.attempts, + verified: summary.verified, + success_rate: valid.then_some(summary.success_rate), + elapsed_seconds_observed: (!rows.is_empty()).then_some(summary.elapsed_seconds), + retries_started_observed: (!rows.is_empty()).then_some(summary.retries_started), + input_tokens: (valid && summary.usage_complete).then_some(summary.input_tokens_observed), + output_tokens: (valid && summary.usage_complete).then_some(summary.output_tokens_observed), + cache_read_tokens: (valid && summary.usage_complete) + .then_some(summary.cache_read_tokens_observed), + cache_write_tokens: (valid && summary.usage_complete) + .then_some(summary.cache_write_tokens_observed), + // This tool-free, unattended runner has no intervention observer. + user_interventions: None, + provider_reported_cost_usd: valid + .then_some(summary.provider_reported_cost_usd) + .flatten(), + cost_per_verified_task_usd: valid + .then_some(summary.cost_per_verified_task_usd) + .flatten(), + } +} + +#[derive(Serialize)] +pub struct Report { + pub schema: &'static str, + pub protocol_sha256: String, + pub comparison_valid: bool, + pub expected_pairs: usize, + pub complete_pairs: usize, + pub blockers: Vec, + pub control: Arm, + pub candidate: Arm, + pub candidate_only: usize, + pub control_only: usize, + pub difference_percentage_points: Option, + pub limitations: Vec<&'static str>, + pub promotion_allowed: bool, +} + +/// Regrade actual answer artifacts. `verified` from a stored result never grants +/// success; complete exact slots and runtime outcomes own the denominator. +pub fn paired(suite: &InstructionSuite, protocol_sha256: &str, rows: &[AssignedTrial]) -> Report { + let plan = suite.plan(protocol_sha256); + let mut blockers = Vec::new(); + let mut control = Vec::new(); + let mut candidate = Vec::new(); + let mut wins = 0; + let mut losses = 0; + let mut complete_pairs = 0; + let mut outcomes = BTreeSet::new(); + let mut varied = false; + // Plan validity is checked before execution; callers inspecting partial + // results still get a fail-closed report if the contract is empty. + if plan.is_empty() { + blockers.push("empty paired cohort".into()); + } + for row in rows { + if !plan.contains(&row.slot) { + blockers.push("unexpected or mismatched assigned slot".into()); + } + } + for slots in plan.chunks_exact(2) { + let mut pair = Vec::new(); + for slot in slots { + let matches = rows + .iter() + .filter(|row| { + row.slot.case_id == slot.case_id + && row.slot.repetition == slot.repetition + && row.slot.candidate == slot.candidate + }) + .collect::>(); + let [row] = matches.as_slice() else { + blockers.push(format!( + "{} repetition {} candidate {}: expected one observation, found {}", + slot.case_id, + slot.repetition, + slot.candidate, + matches.len() + )); + continue; + }; + let trial = &row.trial; + let runtime_valid = row.slot == *slot + && trial.case_id == slot.case_id + && !trial.compacted + && !trial.forced_compaction_applied + && trial.terminal + && trial.failure.is_none() + && trial.elapsed_seconds.is_finite() + && trial.elapsed_seconds >= 0.0 + && trial.measurement.response_count > 0; + let case = suite + .cohort + .cases + .iter() + .find(|case| case.id == slot.case_id) + .expect("planned case"); + let mut regraded = trial.clone(); + regraded.verified = runtime_valid && suite::grade(&trial.answer, &case.expected); + // Preserve observed failed-attempt time/retries too. Invalid slots + // still block every headline and never produce task success. + if row.slot == *slot + && trial.elapsed_seconds.is_finite() + && trial.elapsed_seconds >= 0.0 + { + if slot.candidate { + candidate.push(regraded.clone()); + } else { + control.push(regraded.clone()); + } + } + if !runtime_valid { + blockers.push(format!( + "{} repetition {} candidate {}: invalid runtime or protocol outcome", + slot.case_id, slot.repetition, slot.candidate + )); + continue; + } + pair.push((slot.candidate, regraded.verified)); + if outcomes.contains(&(slot.case_id.clone(), slot.candidate, !regraded.verified)) { + varied = true; + } + outcomes.insert((slot.case_id.clone(), slot.candidate, regraded.verified)); + } + if pair.len() == 2 { + complete_pairs += 1; + let original = pair + .iter() + .find(|(candidate, _)| !candidate) + .expect("control slot") + .1; + let treated = pair + .iter() + .find(|(candidate, _)| *candidate) + .expect("candidate slot") + .1; + wins += usize::from(treated && !original); + losses += usize::from(original && !treated); + } + } + let expected_pairs = plan.len() / 2; + let valid = blockers.is_empty() && complete_pairs == expected_pairs && expected_pairs > 0; + let control = arm(&control, expected_pairs, valid); + let candidate = arm(&candidate, expected_pairs, valid); + let mut limitations = vec![ + "instruction fragments only; workspace discovery and executable skills/tools are not exercised", + "synthetic task-shaped answer artifacts; no repository patch or customer acceptance", + "operator-declared model and provider configuration are not remote weight identity", + "no statistical significance or causal improvement claim", + ]; + if suite.repetitions < 2 { + limitations.push("one repetition does not establish stability"); + } + if varied { + limitations.push("observed outcome variation across repetitions"); + } + if valid && (control.verified == expected_pairs || candidate.verified == expected_pairs) { + limitations.push("at least one arm is saturated"); + } + if valid && wins <= losses { + limitations.push("no positive observed success delta"); + } + Report { + schema: "maestro.instruction-eval-report.v1", + protocol_sha256: protocol_sha256.into(), + comparison_valid: valid, + expected_pairs, + complete_pairs, + blockers, + control, + candidate, + candidate_only: wins, + control_only: losses, + difference_percentage_points: valid + .then(|| 100.0 * (wins as f64 - losses as f64) / expected_pairs as f64), + limitations, + promotion_allowed: false, + } +} + +pub fn unique_plan(suite: &InstructionSuite, protocol_sha256: &str) -> Result<()> { + let mut identities = HashSet::new(); + for slot in suite.plan(protocol_sha256) { + ensure!( + identities.insert((slot.case_id, slot.repetition, slot.candidate)), + "duplicate planned slot" + ); + } + Ok(()) +} diff --git a/packages/local-host-rs/examples/instruction_eval/main.rs b/packages/local-host-rs/examples/instruction_eval/main.rs new file mode 100644 index 000000000..3f065386d --- /dev/null +++ b/packages/local-host-rs/examples/instruction_eval/main.rs @@ -0,0 +1,178 @@ +//! Paired AGENTS.md instruction fragments through existing native trials. +mod instructions; +#[path = "../behavior_eval/measurement.rs"] +mod report; +#[path = "../compaction_eval/suite.rs"] +mod suite; +#[cfg(test)] +mod tests; +#[path = "../compaction_eval/trial.rs"] +mod trial; + +use anyhow::{Context, Result, ensure}; +use clap::Parser; +use instructions::{AssignedTrial, InstructionSuite}; +use maestro_local_host as host; +use maestro_local_host::agent::{ + CredentialVault, ModelDynamicsConfig, NativeAgent, NativeAgentConfig, +}; +use maestro_runtime::agent::MaxTokensSource; +use sha2::{Digest, Sha256}; +use std::{collections::HashSet, path::Path, path::PathBuf, time::Duration}; + +#[derive(Parser)] +struct Args { + #[arg(long)] + suite: PathBuf, + /// Explicit provider-qualified model; no routing or fallback. + #[arg(long)] + model: String, + /// New directory for the frozen plan, arm artifacts and partial reports. + #[arg(long)] + output: PathBuf, + #[arg(long, default_value_t = 60, value_parser = clap::value_parser!(u64).range(1..=600))] + timeout_seconds: u64, + #[arg(long, default_value_t = 1024, value_parser = clap::value_parser!(u32).range(1..=4096))] + max_tokens: u32, + #[arg(long)] + context_window: u64, + /// Validate and write the plan without authentication or model calls. + #[arg(long)] + plan_only: bool, +} + +fn hash(bytes: &[u8]) -> String { + format!("sha256:{:x}", Sha256::digest(bytes)) +} + +fn save_report( + output: &Path, + suite: &InstructionSuite, + protocol: &str, + rows: &[AssignedTrial], +) -> Result { + let report = instructions::paired(suite, protocol, rows); + // A later interruption cannot leave a stale complete report. Each arm is + // persisted before the report advances; rename replaces a complete file. + std::fs::write( + output.join("report.pending.json"), + serde_json::to_vec_pretty(&report)?, + )?; + std::fs::rename( + output.join("report.pending.json"), + output.join("report.json"), + )?; + Ok(report) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + let (provider, model) = args + .model + .split_once('/') + .context("qualify model with its provider")?; + ensure!( + !provider.trim().is_empty() && !model.trim().is_empty(), + "qualify model with its provider" + ); + let raw = std::fs::read(&args.suite)?; + ensure!(raw.len() <= 2_000_000, "suite exceeds byte budget"); + let suite: InstructionSuite = serde_json::from_slice(&raw)?; + suite.validate(args.context_window, args.max_tokens)?; + let binary = std::env::current_exe()?; + let protocol_inputs = serde_json::json!({ + "schema": "maestro.instruction-eval-protocol.v1", "suite_sha256": hash(&raw), + "model": args.model, "executable_sha256": hash(&std::fs::read(binary)?), + "timeout_seconds": args.timeout_seconds, "max_tokens": args.max_tokens, + "context_window": args.context_window, "thinking_enabled": false, + "tools": [], "hooks_enabled": false, "automatic_model_routing": false, + "treatment_surface": "agents_md_fragment", "compaction": "none", + "isolation": "fresh_workspace_and_native_agent_per_arm" + }); + let protocol = hash(&serde_json::to_vec(&protocol_inputs)?); + instructions::unique_plan(&suite, &protocol)?; + let plan = suite.plan(&protocol); + std::fs::create_dir(&args.output).context("output must be new and its parent must exist")?; + std::fs::write( + args.output.join("manifest.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "protocol_sha256": protocol, "protocol_inputs": protocol_inputs, "suite": suite, + "expected_runs": plan, "promotion_allowed": false + }))?, + )?; + let mut rows = Vec::new(); + save_report(&args.output, &suite, &protocol, &rows)?; + if args.plan_only { + println!( + "validated {} assigned native trials; no model calls", + plan.len() + ); + return Ok(()); + } + for slot in plan { + let case = suite + .cohort + .cases + .iter() + .find(|case| case.id == slot.case_id) + .expect("planned case"); + let cwd = tempfile::tempdir()?; + let config = NativeAgentConfig { + model: args.model.clone(), + max_tokens: args.max_tokens, + max_tokens_source: MaxTokensSource::Explicit, + context_window: Some(args.context_window), + cwd: cwd.path().to_string_lossy().into_owned(), + system_prompt: Some(suite.system_prompt(slot.candidate)), + thinking_enabled: false, + model_dynamics: ModelDynamicsConfig::default(), + ..Default::default() + }; + let (agent, mut events) = NativeAgent::new_with_allowed_tools_and_credential_vault( + config, + &HashSet::new(), + CredentialVault::new(), + )?; + agent.set_hooks_enabled(false)?; + let folder = args.output.join(format!( + "{}--{}--{}", + slot.case_id, + slot.repetition, + if slot.candidate { + "candidate" + } else { + "control" + } + )); + std::fs::create_dir(&folder)?; + let result = trial::run( + &agent, + &mut events, + case, + false, + Duration::from_secs(args.timeout_seconds), + &folder, + ) + .await; + agent.cancel(); + agent.shutdown().await; + let assigned = AssignedTrial { + slot, + trial: result?, + }; + std::fs::write( + folder.join("result.json"), + serde_json::to_vec_pretty(&assigned)?, + )?; + rows.push(assigned); + save_report(&args.output, &suite, &protocol, &rows)?; + } + let report = save_report(&args.output, &suite, &protocol, &rows)?; + println!("{}", serde_json::to_string_pretty(&report)?); + ensure!( + report.comparison_valid, + "inconclusive comparison: inspect assigned outcomes" + ); + Ok(()) +} diff --git a/packages/local-host-rs/examples/instruction_eval/provider_tests.rs b/packages/local-host-rs/examples/instruction_eval/provider_tests.rs new file mode 100644 index 000000000..8031bb057 --- /dev/null +++ b/packages/local-host-rs/examples/instruction_eval/provider_tests.rs @@ -0,0 +1,154 @@ +//! Actual native turns against loopback fixtures, never paid model evaluations. +#[path = "instructions.rs"] +mod instructions; + +// Reuse the lib test's existing modules: loading these files a second time +// forks their type identities and trips the duplicate-module contract. +use super::compaction_eval_provider_tests::{report, request, sse, suite, trial}; +use crate::agent::{CredentialVault, ModelDynamicsConfig, NativeAgent, NativeAgentConfig}; +use maestro_ai::{OpenAiClient, UnifiedClient}; +use maestro_runtime::agent::MaxTokensSource; +use sha2::{Digest, Sha256}; +use std::{collections::HashSet, time::Duration}; +use tokio::{io::AsyncWriteExt, net::TcpListener}; + +fn hash(bytes: &[u8]) -> String { + format!("sha256:{:x}", Sha256::digest(bytes)) +} + +#[tokio::test] +async fn matched_native_requests_change_only_instruction_fragment_and_never_send_grader() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut sent = Vec::new(); + for _ in 0..4 { + let (mut stream, _) = listener.accept().await.unwrap(); + sent.push(request(&mut stream).await); + let body = sse(r#"{"answer":"GRADER_ONLY_SENTINEL"}"#); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + } + sent + }); + let suite: instructions::InstructionSuite = serde_json::from_value(serde_json::json!({ + "schema":"maestro.instruction-eval-suite.v1", "repetitions":2, + "shared_system_prompt":"SHARED_SYSTEM_MARKER", + "control_agents_md":"CONTROL_INSTRUCTION_MARKER", + "candidate_agents_md":"CANDIDATE_INSTRUCTION_MARKER", + "cohort": { + "schema":"maestro.compaction-eval-suite.v1", + "cases":[{ + "id":"native-fixture", "family":"request-mechanics", + "history":[{"role":"user", "content":"SHARED_HISTORY_MARKER"}, {"role":"assistant", "content":"Acknowledged."}], + "question":"Return a JSON object with the answer string.", + "expected":{"answer":"GRADER_ONLY_SENTINEL"} + }] + } + })).unwrap(); + suite.validate(128_000, 1024).unwrap(); + instructions::unique_plan(&suite, "fixture-protocol").unwrap(); + let output = tempfile::tempdir().unwrap(); + let mut rows = Vec::new(); + for (index, slot) in suite.plan("fixture-protocol").into_iter().enumerate() { + let cwd = tempfile::tempdir().unwrap(); + let config = NativeAgentConfig { + model: "openai/gpt-4o".into(), + max_tokens: 1024, + max_tokens_source: MaxTokensSource::Explicit, + context_window: Some(128_000), + cwd: cwd.path().to_string_lossy().into_owned(), + system_prompt: Some(suite.system_prompt(slot.candidate)), + thinking_enabled: false, + model_dynamics: ModelDynamicsConfig::default(), + ..Default::default() + }; + let client = UnifiedClient::OpenAI( + OpenAiClient::with_base_url("fixture-key", format!("http://{address}/v1")).unwrap(), + ); + let (agent, mut events) = NativeAgent::start( + config, + vec![], + CredentialVault::new(), + Some(&HashSet::new()), + Some(super::ClientOverride::UnverifiedTest(client)), + None, + None, + ) + .unwrap(); + assert!(agent.runtime_audit_snapshot().tools.is_empty()); + agent.set_hooks_enabled(false).unwrap(); + let folder = output.path().join(index.to_string()); + std::fs::create_dir(&folder).unwrap(); + let result = trial::run( + &agent, + &mut events, + &suite.cohort.cases[0], + false, + Duration::from_secs(10), + &folder, + ) + .await; + agent.cancel(); + agent.shutdown().await; + rows.push(instructions::AssignedTrial { + slot, + trial: result.unwrap(), + }); + for line in std::fs::read_to_string(folder.join("events.jsonl")) + .unwrap() + .lines() + { + serde_json::from_str::(line).unwrap(); + } + } + let sent = tokio::time::timeout(Duration::from_secs(5), server) + .await + .unwrap() + .unwrap(); + for (index, (request, row)) in sent.iter().zip(&rows).enumerate() { + let text = request["messages"].to_string(); + assert!(text.contains("SHARED_SYSTEM_MARKER")); + assert!(text.contains("SHARED_HISTORY_MARKER")); + assert!(text.contains(if row.slot.candidate { + "CANDIDATE_INSTRUCTION_MARKER" + } else { + "CONTROL_INSTRUCTION_MARKER" + })); + assert!(!request.to_string().contains("GRADER_ONLY_SENTINEL")); + assert!( + request + .get("tools") + .is_none_or(|tools| tools.as_array().is_some_and(Vec::is_empty)) + ); + assert_eq!( + request["model"], sent[0]["model"], + "fixed model in request {index}" + ); + let nonsystem = |request: &serde_json::Value| { + request["messages"] + .as_array() + .unwrap() + .iter() + .filter(|message| message["role"] != "system") + .cloned() + .collect::>() + }; + assert_eq!(nonsystem(request), nonsystem(&sent[0])); + assert!(row.trial.verified && row.trial.terminal); + assert!(!row.trial.forced_compaction_applied); + assert_eq!(row.trial.measurement.response_count, 1); + } + let report = instructions::paired(&suite, "fixture-protocol", &rows); + assert!(report.comparison_valid); + assert_eq!(report.complete_pairs, 2); + assert_eq!(report.difference_percentage_points, Some(0.0)); + assert_eq!(report.control.input_tokens, Some(200)); + assert_eq!(report.candidate.input_tokens, Some(200)); + assert!(report.control.provider_reported_cost_usd.is_none()); + assert!(report.candidate.user_interventions.is_none()); + assert!(!report.promotion_allowed); +} diff --git a/packages/local-host-rs/examples/instruction_eval/tests.rs b/packages/local-host-rs/examples/instruction_eval/tests.rs new file mode 100644 index 000000000..136929050 --- /dev/null +++ b/packages/local-host-rs/examples/instruction_eval/tests.rs @@ -0,0 +1,249 @@ +//! Synthetic observations prove experiment mechanics, never model efficacy. +use super::{ + hash, + instructions::{AssignedTrial, InstructionSuite, paired}, + report::{Measurement, Trial}, +}; +use maestro_runtime::agent::TokenUsage; +use serde_json::json; + +fn fixture_suite() -> InstructionSuite { + serde_json::from_str(include_str!( + "../../../../evals/instruction-behavior-v1.json" + )) + .unwrap() +} + +fn fixture_rows(suite: &InstructionSuite) -> Vec { + suite + .plan("protocol") + .into_iter() + .map(|slot| { + let case = suite + .cohort + .cases + .iter() + .find(|case| case.id == slot.case_id) + .unwrap(); + let mut measurement = Measurement::default(); + measurement.add(Some(&TokenUsage { + input_tokens: 100, + output_tokens: 20, + cache_read_tokens: 50, + cache_write_tokens: 0, + cost: Some(0.1), + })); + measurement.complete = true; + AssignedTrial { + trial: Trial { + case_id: slot.case_id.clone(), + compacted: false, + verified: true, + terminal: true, + failure: None, + elapsed_seconds: 1.0, + retries_started: 0, + forced_compaction_applied: false, + answer: case.expected.to_string(), + measurement, + }, + slot, + } + }) + .collect() +} + +#[test] +fn task_shaped_cohort_validates_before_inference_and_repeats_alternate() { + let suite = fixture_suite(); + suite.validate(128_000, 1024).unwrap(); + super::instructions::unique_plan(&suite, "protocol").unwrap(); + let plan = suite.plan("protocol"); + assert_eq!(plan.len(), 18); + assert_eq!( + plan.iter() + .take(6) + .map(|slot| (slot.repetition, slot.candidate)) + .collect::>(), + [ + (1, false), + (1, true), + (2, true), + (2, false), + (3, false), + (3, true) + ] + ); + for pair in plan.chunks_exact(2) { + assert_eq!(pair[0].case_id, pair[1].case_id); + assert_eq!(pair[0].repetition, pair[1].repetition); + assert_ne!(pair[0].instruction_sha256, pair[1].instruction_sha256); + assert_eq!(pair[0].protocol_sha256, pair[1].protocol_sha256); + } + assert_eq!( + plan[0].instruction_sha256, + hash(suite.system_prompt(false).as_bytes()) + ); + let mut duplicate = fixture_suite(); + duplicate.cohort.cases[1].id = duplicate.cohort.cases[0].id.clone(); + assert!(duplicate.validate(128_000, 1024).is_err()); +} + +#[test] +fn changed_instruction_and_safe_cohort_are_required() { + for repetitions in [0, 101] { + let mut bad = fixture_suite(); + bad.repetitions = repetitions; + assert!(bad.validate(128_000, 1024).is_err()); + } + let mut bad = fixture_suite(); + bad.candidate_agents_md = bad.control_agents_md.clone(); + assert!(bad.validate(128_000, 1024).is_err()); + let mut bad = fixture_suite(); + bad.shared_system_prompt = String::new(); + assert!(bad.validate(128_000, 1024).is_err()); + let mut bad = fixture_suite(); + bad.candidate_agents_md = "x".repeat(64 * 1024); + assert!(bad.validate(128_000, 1024).is_err()); + assert!(fixture_suite().validate(8192, 1024).is_err()); + let mut bad = fixture_suite(); + bad.cohort.cases[1].history.remove(1); + assert!(bad.validate(128_000, 1024).is_err()); +} + +#[test] +fn actual_artifacts_override_claimed_success_and_keep_failed_attempt_cost() { + let suite = fixture_suite(); + let mut rows = fixture_rows(&suite); + rows[0].trial.answer = + json!({"query":"SELECT id FROM documents WHERE workspace_id = $1", "verified":true}) + .to_string(); + assert!( + rows[0].trial.verified, + "stored success is deliberately untrusted" + ); + rows[1].trial.verified = false; + let report = paired(&suite, "protocol", &rows); + assert!(report.comparison_valid); + assert_eq!(report.expected_pairs, 9); + assert_eq!(report.complete_pairs, 9); + assert_eq!(report.control.verified, 8); + assert_eq!(report.candidate.verified, 9); + assert_eq!(report.candidate_only, 1); + assert!((report.difference_percentage_points.unwrap() - 100.0 / 9.0).abs() < 1e-10); + assert!((report.control.cost_per_verified_task_usd.unwrap() - 0.9 / 8.0).abs() < 1e-10); + assert!( + report + .limitations + .contains(&"observed outcome variation across repetitions") + ); + assert!(!report.promotion_allowed); +} + +#[test] +fn missing_duplicate_extra_or_changed_protocol_withholds_headline() { + let suite = fixture_suite(); + let assert_blocked = |rows: &[AssignedTrial]| { + let report = paired(&suite, "protocol", rows); + assert!(!report.comparison_valid); + assert!(!report.blockers.is_empty()); + assert!(report.difference_percentage_points.is_none()); + assert!(report.control.success_rate.is_none()); + assert!(report.candidate.success_rate.is_none()); + assert!(report.control.input_tokens.is_none()); + }; + let mut missing = fixture_rows(&suite); + missing.pop(); + assert_blocked(&missing); + let mut duplicate = fixture_rows(&suite); + duplicate[1].slot = duplicate[0].slot.clone(); + assert_blocked(&duplicate); + let mut extra = fixture_rows(&suite); + extra[0].slot.case_id = "unexpected".into(); + assert_blocked(&extra); + let mut changed = fixture_rows(&suite); + changed[0].slot.protocol_sha256 = "wrong".into(); + assert_blocked(&changed); + let mut changed = fixture_rows(&suite); + changed[0].slot.instruction_sha256 = "wrong".into(); + assert_blocked(&changed); + assert_blocked(&[]); + let empty = paired(&suite, "protocol", &[]); + assert!(empty.control.elapsed_seconds_observed.is_none()); + assert!(empty.candidate.retries_started_observed.is_none()); +} + +#[test] +fn runtime_failure_timeout_unscored_or_compacted_arm_blocks_comparison() { + let suite = fixture_suite(); + for failure in ["timeout", "runtime_or_summary_failure"] { + let mut bad = fixture_rows(&suite); + bad[0].trial.failure = Some(failure.into()); + bad[0].trial.terminal = false; + let report = paired(&suite, "protocol", &bad); + assert!(!report.comparison_valid); + assert!(report.difference_percentage_points.is_none()); + assert_eq!(report.expected_pairs, 9); + assert_eq!(report.control.observed, 9); + assert_eq!(report.control.elapsed_seconds_observed, Some(9.0)); + } + for mutation in 0..5 { + let mut bad = fixture_rows(&suite); + match mutation { + 0 => bad[0].trial.terminal = false, + 1 => bad[0].trial.compacted = true, + 2 => bad[0].trial.forced_compaction_applied = true, + 3 => bad[0].trial.elapsed_seconds = f64::NAN, + _ => bad[0].trial.measurement.response_count = 0, + } + assert!(!paired(&suite, "protocol", &bad).comparison_valid); + } +} + +#[test] +fn unobserved_or_partial_metrics_stay_unavailable() { + let suite = fixture_suite(); + let mut rows = fixture_rows(&suite); + rows[0].trial.measurement.add(None); + let report = paired(&suite, "protocol", &rows); + assert!(report.comparison_valid); + assert!(report.control.input_tokens.is_none()); + assert!(report.control.output_tokens.is_none()); + assert!(report.control.cache_read_tokens.is_none()); + assert!(report.control.provider_reported_cost_usd.is_none()); + assert!(report.control.cost_per_verified_task_usd.is_none()); + assert_eq!(report.candidate.input_tokens, Some(900)); + assert!(report.control.user_interventions.is_none()); + assert!(report.candidate.user_interventions.is_none()); + let mut rows = fixture_rows(&suite); + rows[0].trial.measurement.complete = false; + assert!( + paired(&suite, "protocol", &rows) + .control + .input_tokens + .is_none() + ); +} + +#[test] +fn no_positive_delta_single_repeat_and_saturation_are_explicit() { + let mut suite = fixture_suite(); + suite.repetitions = 1; + let report = paired(&suite, "protocol", &fixture_rows(&suite)); + assert_eq!(report.difference_percentage_points, Some(0.0)); + assert!( + report + .limitations + .contains(&"one repetition does not establish stability") + ); + assert!( + report + .limitations + .contains(&"at least one arm is saturated") + ); + assert!( + report + .limitations + .contains(&"no positive observed success delta") + ); +} diff --git a/packages/local-host-rs/src/agent/mod.rs b/packages/local-host-rs/src/agent/mod.rs index af50e3bb7..844f09ac1 100644 --- a/packages/local-host-rs/src/agent/mod.rs +++ b/packages/local-host-rs/src/agent/mod.rs @@ -14,6 +14,9 @@ mod compaction_eval_provider_tests; #[cfg(test)] pub mod harness; #[cfg(test)] +#[path = "../../examples/instruction_eval/provider_tests.rs"] +mod instruction_eval_provider_tests; +#[cfg(test)] mod native_admission_tests; #[cfg(test)] mod native_codex_tests; diff --git a/vendor/dex-loop/src/content_policy.rs b/vendor/dex-loop/src/content_policy.rs index dbddcf0e7..cc9b1d1f9 100644 --- a/vendor/dex-loop/src/content_policy.rs +++ b/vendor/dex-loop/src/content_policy.rs @@ -25,11 +25,11 @@ pub enum AuthoredFieldKind { #[derive(Clone, Debug, PartialEq, Eq)] pub struct AuthoredField { - pub path: String, + path: String, pub prose: String, - pub kind: AuthoredFieldKind, - pub items: usize, - pub citation_urls: Vec, + kind: AuthoredFieldKind, + items: usize, + citation_urls: Vec, } impl AuthoredContent {