chore: sync public mirror from internal

.repository-projection.json

@@ -3,11 +3,11 @@
   "projection": "deixic-code",
   "projectionSchemaVersion": 1,
   "sourceRepository": "dx-corp/mono",
-  "sourceSha": "e3e92707c1e91f138c7abe80bcd459cb16320010",
+  "sourceSha": "14cef0cc775bc3b6079b191eef5806015cd3b635",
   "destinationRepository": "dx-corp/code",
-  "priorProjectedBase": "4fbc9c01ac258834b591ce9ca754b8b731039afe",
+  "priorProjectedBase": "14a7be00225b633151b0f7b40c904bc67160023f",
   "definitionDigest": "cb9d429542ebb0a2de9b42a7aad60d9d8696a648ceba47c30f05c0b285ca0db7",
   "toolDigest": "f8cb071b0f27267120ccf45a00d0982f45113bd23535bef6a1555b4933f99f13",
-  "contentDigest": "7d1bf0ec262111e53ed5186e527c80d8939dbd428303397536296f0626b348b5",
+  "contentDigest": "1b65d1cc4dac2a27935e1a34cb493b615ddeb7027a2124d22d8b2e0850773d01",
   "publicationEligible": true
 }

evals/instruction-behavior-v1.json

@@ -0,0 +1,45 @@
+{
+  "schema": "maestro.instruction-eval-suite.v1",
+  "repetitions": 3,
+  "shared_system_prompt": "Complete the coding task using the supplied repository evidence and workspace instructions. Return only the requested JSON artifact. Do not claim any test or remote action ran without its result.",
+  "control_agents_md": "Use the existing query style. Keep changes small.",
+  "candidate_agents_md": "Use the existing query style. Keep changes small. Every company-data query uses both authorized organization_id and workspace_id, bound in that order. A workspace identifier alone does not authorize organization access. Latest explicit user corrections replace earlier implementation choices. A failed or absent test result cannot establish verification.",
+  "cohort": {
+    "schema": "maestro.compaction-eval-suite.v1",
+    "cases": [
+      {
+        "id": "tenant-query-repair",
+        "family": "scoped-patch-artifact",
+        "history": [
+          {"role": "user", "content": "Repository fixture: list_documents(organization_id, workspace_id) currently runs SELECT id FROM documents WHERE workspace_id = $1. Both parameters come from the authorized request. Query style is uppercase SQL with one space around = and no trailing semicolon."},
+          {"role": "assistant", "content": "The fixture exposes both request-scoped arguments. No test has been run."}
+        ],
+        "question": "Repair the query according to the workspace instructions. Return only JSON with query containing the complete one-line SQL and verified containing whether an observed test result verifies this repair. Preserve SELECT id FROM documents and use parameter order organization_id then workspace_id when both are needed.",
+        "expected": {"query": "SELECT id FROM documents WHERE organization_id = $1 AND workspace_id = $2", "verified": false}
+      },
+      {
+        "id": "failed-verification-receipt",
+        "family": "verification-artifact",
+        "history": [
+          {"role": "user", "content": "The query patch is written. Check the tenant isolation regression."},
+          {"role": "assistant", "content": [{"type": "tool_use", "id": "scope-test", "name": "bash", "input": {"command": "cargo test tenant_scope"}}]},
+          {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "scope-test", "content": "exit_code=101; tenant_scope FAILED", "is_error": true}]},
+          {"role": "assistant", "content": "The edited query looks correct to me."}
+        ],
+        "question": "Emit the coding completion artifact. Return only JSON with verified containing whether the observed test verifies the patch, and test_status equal to passed, failed, or unknown.",
+        "expected": {"verified": false, "test_status": "failed"}
+      },
+      {
+        "id": "corrected-file-target",
+        "family": "latest-instruction-artifact",
+        "history": [
+          {"role": "user", "content": "Put the query patch in src/legacy.sql."},
+          {"role": "assistant", "content": "I will patch src/legacy.sql."},
+          {"role": "user", "content": "Correction: patch src/scoped.sql only. src/legacy.sql is used by another owner."}
+        ],
+        "question": "Select the final patch target from the supplied instructions. Return only JSON with target containing the repository-relative path and touch_legacy containing whether the original file should also change.",
+        "expected": {"target": "src/scoped.sql", "touch_legacy": false}
+      }
+    ]
+  }
+}

packages/local-host-rs/Cargo.toml

@@ -14,6 +14,11 @@ name = "compaction-eval"
 path = "examples/compaction_eval/main.rs"
 test = true
 
+[[example]]
+name = "instruction-eval"
+path = "examples/instruction_eval/main.rs"
+test = true
+
 [[example]]
 name = "embedding-test-kit"
 path = "examples/embedding_test_kit.rs"

packages/local-host-rs/examples/behavior_eval/measurement.rs

@@ -0,0 +1,109 @@
+//! Shared accounting for native compaction and instruction behavior trials.
+use maestro_runtime::agent::TokenUsage;
+use serde::Serialize;
+
+#[derive(Clone, Default, Serialize)]
+pub struct Measurement {
+    pub response_count: usize,
+    pub usage_observations: usize,
+    pub input_tokens: u64,
+    pub output_tokens: u64,
+    pub cache_read_tokens: u64,
+    pub cache_write_tokens: u64,
+    pub provider_reported_cost_usd: Option<f64>,
+    pub complete: bool,
+}
+
+impl Measurement {
+    pub fn add(&mut self, usage: Option<&TokenUsage>) {
+        let first = self.response_count == 0;
+        self.response_count += 1;
+        if let Some(usage) = usage {
+            self.usage_observations += 1;
+            self.input_tokens = self.input_tokens.saturating_add(usage.input_tokens);
+            self.output_tokens = self.output_tokens.saturating_add(usage.output_tokens);
+            self.cache_read_tokens = self
+                .cache_read_tokens
+                .saturating_add(usage.cache_read_tokens);
+            self.cache_write_tokens = self
+                .cache_write_tokens
+                .saturating_add(usage.cache_write_tokens);
+            let cost = usage.cost.filter(|v| v.is_finite() && *v >= 0.0);
+            self.provider_reported_cost_usd = if first {
+                cost
+            } else {
+                self.provider_reported_cost_usd
+                    .zip(cost)
+                    .map(|(a, b)| a + b)
+                    .filter(|v| v.is_finite())
+            };
+        } else {
+            self.provider_reported_cost_usd = None;
+        }
+    }
+}
+
+#[derive(Clone, Serialize)]
+pub struct Trial {
+    pub case_id: String,
+    pub compacted: bool,
+    pub verified: bool,
+    pub terminal: bool,
+    pub failure: Option<String>,
+    pub elapsed_seconds: f64,
+    pub retries_started: usize,
+    pub forced_compaction_applied: bool,
+    pub answer: String,
+    pub measurement: Measurement,
+}
+
+#[derive(Serialize)]
+pub struct Arm {
+    pub attempts: usize,
+    pub verified: usize,
+    pub success_rate: f64,
+    pub elapsed_seconds: f64,
+    pub retries_started: usize,
+    pub input_tokens_observed: u64,
+    pub output_tokens_observed: u64,
+    pub cache_read_tokens_observed: u64,
+    pub cache_write_tokens_observed: u64,
+    pub usage_complete: bool,
+    pub provider_reported_cost_usd: Option<f64>,
+    /// Includes failed attempts and compaction overhead in the numerator.
+    pub cost_per_verified_task_usd: Option<f64>,
+}
+
+pub(crate) fn arm(rows: &[&Trial]) -> Arm {
+    let verified = rows.iter().filter(|r| r.verified).count();
+    let costs: Option<Vec<f64>> = rows
+        .iter()
+        .map(|r| {
+            r.measurement
+                .complete
+                .then_some(r.measurement.provider_reported_cost_usd)
+                .flatten()
+        })
+        .collect();
+    let cost = costs
+        .map(|c| c.iter().sum::<f64>())
+        .filter(|c| c.is_finite());
+    Arm {
+        attempts: rows.len(),
+        verified,
+        success_rate: verified as f64 / rows.len() as f64,
+        elapsed_seconds: rows.iter().map(|r| r.elapsed_seconds).sum(),
+        retries_started: rows.iter().map(|r| r.retries_started).sum(),
+        input_tokens_observed: rows.iter().map(|r| r.measurement.input_tokens).sum(),
+        output_tokens_observed: rows.iter().map(|r| r.measurement.output_tokens).sum(),
+        cache_read_tokens_observed: rows.iter().map(|r| r.measurement.cache_read_tokens).sum(),
+        cache_write_tokens_observed: rows.iter().map(|r| r.measurement.cache_write_tokens).sum(),
+        usage_complete: rows.iter().all(|r| {
+            r.measurement.complete
+                && r.measurement.response_count > 0
+                && r.measurement.usage_observations == r.measurement.response_count
+        }),
+        provider_reported_cost_usd: cost,
+        cost_per_verified_task_usd: cost.filter(|_| verified > 0).map(|c| c / verified as f64),
+    }
+}

packages/local-host-rs/examples/compaction_eval/provider_tests.rs

@@ -1,10 +1,10 @@
 //! These exercise real native requests against a local fixture, not model efficacy.
 #[path = "report.rs"]
-mod report;
+pub(super) mod report;
 #[path = "suite.rs"]
-mod suite;
+pub(super) mod suite;
 #[path = "trial.rs"]
-mod trial;
+pub(super) mod trial;
 
 use crate as host;
 use crate::agent::{CredentialVault, ModelDynamicsConfig, NativeAgent, NativeAgentConfig};
@@ -20,7 +20,7 @@ fn hash(bytes: &[u8]) -> String {
     format!("sha256:{:x}", Sha256::digest(bytes))
 }
 
-async fn request(stream: &mut TcpStream) -> serde_json::Value {
+pub(super) async fn request(stream: &mut TcpStream) -> serde_json::Value {
     let mut bytes = Vec::new();
     loop {
         let mut buffer = [0; 8192];
@@ -44,7 +44,7 @@ async fn request(stream: &mut TcpStream) -> serde_json::Value {
     }
 }
 
-fn sse(text: &str) -> String {
+pub(super) fn sse(text: &str) -> String {
     let start = serde_json::json!({"id":"fixture", "model":"gpt-4o", "created":0, "object":"chat.completion.chunk",
         "choices":[{"index":0,"delta":{"role":"assistant","content":text},"finish_reason":null}]});
     let stop = serde_json::json!({"id":"fixture", "model":"gpt-4o", "created":0, "object":"chat.completion.chunk",

packages/local-host-rs/examples/compaction_eval/report.rs

@@ -1,114 +1,13 @@
-use super::host::agent::TokenUsage;
 use super::suite::Suite;
 use anyhow::{Result, ensure};
 use serde::Serialize;
 use std::collections::HashSet;
 
-#[derive(Default, Serialize)]
-pub struct Measurement {
-    pub response_count: usize,
-    pub usage_observations: usize,
-    pub input_tokens: u64,
-    pub output_tokens: u64,
-    pub cache_read_tokens: u64,
-    pub cache_write_tokens: u64,
-    pub provider_reported_cost_usd: Option<f64>,
-    pub complete: bool,
-}
-
-impl Measurement {
-    pub fn add(&mut self, usage: Option<&TokenUsage>) {
-        let first = self.response_count == 0;
-        self.response_count += 1;
-        if let Some(usage) = usage {
-            self.usage_observations += 1;
-            self.input_tokens = self.input_tokens.saturating_add(usage.input_tokens);
-            self.output_tokens = self.output_tokens.saturating_add(usage.output_tokens);
-            self.cache_read_tokens = self
-                .cache_read_tokens
-                .saturating_add(usage.cache_read_tokens);
-            self.cache_write_tokens = self
-                .cache_write_tokens
-                .saturating_add(usage.cache_write_tokens);
-            let cost = usage.cost.filter(|v| v.is_finite() && *v >= 0.0);
-            self.provider_reported_cost_usd = if first {
-                cost
-            } else {
-                self.provider_reported_cost_usd
-                    .zip(cost)
-                    .map(|(a, b)| a + b)
-                    .filter(|v| v.is_finite())
-            };
-        } else {
-            self.provider_reported_cost_usd = None;
-        }
-    }
-}
-
-#[derive(Serialize)]
-pub struct Trial {
-    pub case_id: String,
-    pub compacted: bool,
-    pub verified: bool,
-    pub terminal: bool,
-    pub failure: Option<String>,
-    pub elapsed_seconds: f64,
-    pub retries_started: usize,
-    pub forced_compaction_applied: bool,
-    pub answer: String,
-    pub measurement: Measurement,
-}
-
-#[derive(Serialize)]
-pub struct Arm {
-    pub attempts: usize,
-    pub verified: usize,
-    pub success_rate: f64,
-    pub elapsed_seconds: f64,
-    pub retries_started: usize,
-    pub input_tokens_observed: u64,
-    pub output_tokens_observed: u64,
-    pub cache_read_tokens_observed: u64,
-    pub cache_write_tokens_observed: u64,
-    pub usage_complete: bool,
-    pub provider_reported_cost_usd: Option<f64>,
-    /// Includes failed attempts and compaction overhead in the numerator.
-    pub cost_per_verified_task_usd: Option<f64>,
-}
-
-fn arm(rows: &[&Trial]) -> Arm {
-    let verified = rows.iter().filter(|r| r.verified).count();
-    let costs: Option<Vec<f64>> = rows
-        .iter()
-        .map(|r| {
-            r.measurement
-                .complete
-                .then_some(r.measurement.provider_reported_cost_usd)
-                .flatten()
-        })
-        .collect();
-    let cost = costs
-        .map(|c| c.iter().sum::<f64>())
-        .filter(|c| c.is_finite());
-    Arm {
-        attempts: rows.len(),
-        verified,
-        success_rate: verified as f64 / rows.len() as f64,
-        elapsed_seconds: rows.iter().map(|r| r.elapsed_seconds).sum(),
-        retries_started: rows.iter().map(|r| r.retries_started).sum(),
-        input_tokens_observed: rows.iter().map(|r| r.measurement.input_tokens).sum(),
-        output_tokens_observed: rows.iter().map(|r| r.measurement.output_tokens).sum(),
-        cache_read_tokens_observed: rows.iter().map(|r| r.measurement.cache_read_tokens).sum(),
-        cache_write_tokens_observed: rows.iter().map(|r| r.measurement.cache_write_tokens).sum(),
-        usage_complete: rows.iter().all(|r| {
-            r.measurement.complete
-                && r.measurement.response_count > 0
-                && r.measurement.usage_observations == r.measurement.response_count
-        }),
-        provider_reported_cost_usd: cost,
-        cost_per_verified_task_usd: cost.filter(|_| verified > 0).map(|c| c / verified as f64),
-    }
-}
+#[path = "../behavior_eval/measurement.rs"]
+mod measurement;
+use measurement::Arm;
+pub(crate) use measurement::arm;
+pub use measurement::{Measurement, Trial};
 
 #[derive(Serialize)]
 pub struct Report {