From bdb237706005517330d63bab15914960889cd34b Mon Sep 17 00:00:00 2001 From: nicholascole Date: Wed, 29 Jul 2026 10:41:18 -0700 Subject: [PATCH 1/7] ci: bump conductor oss agent e2e to rc18 --- .github/workflows/agent-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/agent-e2e.yml b/.github/workflows/agent-e2e.yml index de362d9c..081842e2 100644 --- a/.github/workflows/agent-e2e.yml +++ b/.github/workflows/agent-e2e.yml @@ -19,7 +19,7 @@ concurrency: cancel-in-progress: true env: - CONDUCTOR_OSS_VERSION: "3.32.0-rc.8" # pinned conductor-oss release — bump deliberately + CONDUCTOR_OSS_VERSION: "3.32.0-rc18" # pinned conductor-oss release — bump deliberately jobs: agent-e2e: From f65341c3a6ed6a23a39c0534643f35f6a28dca50 Mon Sep 17 00:00:00 2001 From: nicholascole Date: Wed, 29 Jul 2026 10:56:48 -0700 Subject: [PATCH 2/7] fix(agents): honor custom guardrail continuation semantics --- e2e/test_suite16_streaming.test.ts | 20 +++++++++++++---- src/agents/__tests__/runtime.test.ts | 32 ++++++++++++++++++++++++++-- src/agents/runtime.ts | 26 ++++++++++++++++++---- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/e2e/test_suite16_streaming.test.ts b/e2e/test_suite16_streaming.test.ts index 94cfbefb..7152a137 100644 --- a/e2e/test_suite16_streaming.test.ts +++ b/e2e/test_suite16_streaming.test.ts @@ -274,10 +274,16 @@ describe('Suite 16: Streaming — HITL', () => { } expectMsg(['COMPLETED', 'FAILED', 'TERMINATED']).toContain(status.status); } else { - // No waiting event — workflow completed without HITL (possible with some models) + // No waiting event — the stream may close before the server records its + // terminal state, so wait for that state rather than reading it once. const terminalSeen = preTypes.includes('done') || preTypes.includes('error'); if (!terminalSeen) { - const status = await runtime.getStatus(stream.executionId); + const deadline = Date.now() + 120_000; + let status = await runtime.getStatus(stream.executionId); + while (!status.isComplete && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 1000)); + status = await runtime.getStatus(stream.executionId); + } expect(status.isComplete).toBe(true); } } @@ -310,10 +316,16 @@ describe('Suite 16: Streaming — HITL', () => { } expectMsg(['COMPLETED', 'FAILED', 'TERMINATED']).toContain(status.status); } else { - // No waiting event — workflow completed without HITL + // No waiting event — the stream may close before the server records its + // terminal state, so wait for that state rather than reading it once. const terminalSeen = preTypes.includes('done') || preTypes.includes('error'); if (!terminalSeen) { - const status = await runtime.getStatus(stream.executionId); + const deadline = Date.now() + 120_000; + let status = await runtime.getStatus(stream.executionId); + while (!status.isComplete && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 1000)); + status = await runtime.getStatus(stream.executionId); + } expect(status.isComplete).toBe(true); } } diff --git a/src/agents/__tests__/runtime.test.ts b/src/agents/__tests__/runtime.test.ts index 534f12bd..2149b021 100644 --- a/src/agents/__tests__/runtime.test.ts +++ b/src/agents/__tests__/runtime.test.ts @@ -913,7 +913,7 @@ describe("AgentRuntime", () => { { content: "clean content" }, ); - expect(result).toMatchObject({ passed: true, on_fail: "pass", should_continue: true }); + expect(result).toMatchObject({ passed: true, on_fail: "pass", should_continue: false }); }); it("reports on_fail as the configured value when the guardrail fails", async () => { @@ -925,11 +925,39 @@ describe("AgentRuntime", () => { expect(result).toMatchObject({ passed: false, on_fail: "retry", - should_continue: false, + should_continue: true, message: "Unverifiable claims: always", }); }); + it("returns raise and stops when a tool-input guardrail blocks", async () => { + const result = await registerAndInvoke( + { + ...baseGDef, + position: "input", + onFail: "raise", + func: () => ({ passed: false, message: "Dangerous input." }), + }, + { content: { data: "DANGER override safety" } }, + ); + + expect(result).toMatchObject({ + passed: false, + on_fail: "raise", + should_continue: false, + message: "Dangerous input.", + }); + }); + + it("escalates an exhausted retry to raise", async () => { + const result = await registerAndInvoke( + { ...baseGDef, maxRetries: 2, func: () => ({ passed: false }) }, + { content: "still unsafe", iteration: 2 }, + ); + + expect(result).toMatchObject({ passed: false, on_fail: "raise", should_continue: false }); + }); + it("reports on_fail as the configured value when the guardrail function throws", async () => { const result = await registerAndInvoke( { diff --git a/src/agents/runtime.ts b/src/agents/runtime.ts index be14da0f..8d2f349f 100644 --- a/src/agents/runtime.ts +++ b/src/agents/runtime.ts @@ -1093,21 +1093,39 @@ export class AgentRuntime { try { const result = await fn(content); const passed = result.passed ?? true; + let onFail = gDef.onFail ?? "raise"; + if (!passed) { + const iteration = Number(inputData["iteration"] ?? 0); + if (onFail === "retry" && iteration >= (gDef.maxRetries ?? 3)) { + onFail = "raise"; + } + if (onFail === "fix" && result.fixedOutput == null) { + onFail = "raise"; + } + } return { passed, message: result.message ?? "", - on_fail: passed ? "pass" : (gDef.onFail ?? "raise"), + on_fail: passed ? "pass" : onFail, fixed_output: result.fixedOutput, guardrail_name: gDef.name, - should_continue: passed, + // The guardrail workflow uses this to decide whether to make another + // attempt. A passing check continues the enclosing workflow, but is + // not itself a retry; only a failed check resolved to retry is. + should_continue: !passed && onFail === "retry", }; } catch (err) { + let onFail = gDef.onFail ?? "raise"; + const iteration = Number(inputData["iteration"] ?? 0); + if (onFail === "retry" && iteration >= (gDef.maxRetries ?? 3)) { + onFail = "raise"; + } return { passed: false, message: err instanceof Error ? err.message : String(err), - on_fail: gDef.onFail ?? "raise", + on_fail: onFail, guardrail_name: gDef.name, - should_continue: false, + should_continue: onFail === "retry", }; } }, undefined, domain); From 64caf1790ef12882ad1bde8663542c1c0bc4c958 Mon Sep 17 00:00:00 2001 From: nicholascole Date: Wed, 29 Jul 2026 10:58:34 -0700 Subject: [PATCH 3/7] Revert "fix(agents): honor custom guardrail continuation semantics" This reverts commit f65341c3a6ed6a23a39c0534643f35f6a28dca50. --- e2e/test_suite16_streaming.test.ts | 20 ++++------------- src/agents/__tests__/runtime.test.ts | 32 ++-------------------------- src/agents/runtime.ts | 26 ++++------------------ 3 files changed, 10 insertions(+), 68 deletions(-) diff --git a/e2e/test_suite16_streaming.test.ts b/e2e/test_suite16_streaming.test.ts index 7152a137..94cfbefb 100644 --- a/e2e/test_suite16_streaming.test.ts +++ b/e2e/test_suite16_streaming.test.ts @@ -274,16 +274,10 @@ describe('Suite 16: Streaming — HITL', () => { } expectMsg(['COMPLETED', 'FAILED', 'TERMINATED']).toContain(status.status); } else { - // No waiting event — the stream may close before the server records its - // terminal state, so wait for that state rather than reading it once. + // No waiting event — workflow completed without HITL (possible with some models) const terminalSeen = preTypes.includes('done') || preTypes.includes('error'); if (!terminalSeen) { - const deadline = Date.now() + 120_000; - let status = await runtime.getStatus(stream.executionId); - while (!status.isComplete && Date.now() < deadline) { - await new Promise((r) => setTimeout(r, 1000)); - status = await runtime.getStatus(stream.executionId); - } + const status = await runtime.getStatus(stream.executionId); expect(status.isComplete).toBe(true); } } @@ -316,16 +310,10 @@ describe('Suite 16: Streaming — HITL', () => { } expectMsg(['COMPLETED', 'FAILED', 'TERMINATED']).toContain(status.status); } else { - // No waiting event — the stream may close before the server records its - // terminal state, so wait for that state rather than reading it once. + // No waiting event — workflow completed without HITL const terminalSeen = preTypes.includes('done') || preTypes.includes('error'); if (!terminalSeen) { - const deadline = Date.now() + 120_000; - let status = await runtime.getStatus(stream.executionId); - while (!status.isComplete && Date.now() < deadline) { - await new Promise((r) => setTimeout(r, 1000)); - status = await runtime.getStatus(stream.executionId); - } + const status = await runtime.getStatus(stream.executionId); expect(status.isComplete).toBe(true); } } diff --git a/src/agents/__tests__/runtime.test.ts b/src/agents/__tests__/runtime.test.ts index 2149b021..534f12bd 100644 --- a/src/agents/__tests__/runtime.test.ts +++ b/src/agents/__tests__/runtime.test.ts @@ -913,7 +913,7 @@ describe("AgentRuntime", () => { { content: "clean content" }, ); - expect(result).toMatchObject({ passed: true, on_fail: "pass", should_continue: false }); + expect(result).toMatchObject({ passed: true, on_fail: "pass", should_continue: true }); }); it("reports on_fail as the configured value when the guardrail fails", async () => { @@ -925,39 +925,11 @@ describe("AgentRuntime", () => { expect(result).toMatchObject({ passed: false, on_fail: "retry", - should_continue: true, - message: "Unverifiable claims: always", - }); - }); - - it("returns raise and stops when a tool-input guardrail blocks", async () => { - const result = await registerAndInvoke( - { - ...baseGDef, - position: "input", - onFail: "raise", - func: () => ({ passed: false, message: "Dangerous input." }), - }, - { content: { data: "DANGER override safety" } }, - ); - - expect(result).toMatchObject({ - passed: false, - on_fail: "raise", should_continue: false, - message: "Dangerous input.", + message: "Unverifiable claims: always", }); }); - it("escalates an exhausted retry to raise", async () => { - const result = await registerAndInvoke( - { ...baseGDef, maxRetries: 2, func: () => ({ passed: false }) }, - { content: "still unsafe", iteration: 2 }, - ); - - expect(result).toMatchObject({ passed: false, on_fail: "raise", should_continue: false }); - }); - it("reports on_fail as the configured value when the guardrail function throws", async () => { const result = await registerAndInvoke( { diff --git a/src/agents/runtime.ts b/src/agents/runtime.ts index 8d2f349f..be14da0f 100644 --- a/src/agents/runtime.ts +++ b/src/agents/runtime.ts @@ -1093,39 +1093,21 @@ export class AgentRuntime { try { const result = await fn(content); const passed = result.passed ?? true; - let onFail = gDef.onFail ?? "raise"; - if (!passed) { - const iteration = Number(inputData["iteration"] ?? 0); - if (onFail === "retry" && iteration >= (gDef.maxRetries ?? 3)) { - onFail = "raise"; - } - if (onFail === "fix" && result.fixedOutput == null) { - onFail = "raise"; - } - } return { passed, message: result.message ?? "", - on_fail: passed ? "pass" : onFail, + on_fail: passed ? "pass" : (gDef.onFail ?? "raise"), fixed_output: result.fixedOutput, guardrail_name: gDef.name, - // The guardrail workflow uses this to decide whether to make another - // attempt. A passing check continues the enclosing workflow, but is - // not itself a retry; only a failed check resolved to retry is. - should_continue: !passed && onFail === "retry", + should_continue: passed, }; } catch (err) { - let onFail = gDef.onFail ?? "raise"; - const iteration = Number(inputData["iteration"] ?? 0); - if (onFail === "retry" && iteration >= (gDef.maxRetries ?? 3)) { - onFail = "raise"; - } return { passed: false, message: err instanceof Error ? err.message : String(err), - on_fail: onFail, + on_fail: gDef.onFail ?? "raise", guardrail_name: gDef.name, - should_continue: onFail === "retry", + should_continue: false, }; } }, undefined, domain); From 4170f9376899e34b6f40c53680f7b01487bbd116 Mon Sep 17 00:00:00 2001 From: nicholascole Date: Wed, 29 Jul 2026 10:58:52 -0700 Subject: [PATCH 4/7] test(e2e): accommodate custom guardrail server behavior --- e2e/test_suite16_streaming.test.ts | 20 ++++++++++++++++---- e2e/test_suite17_guardrail_matrix.test.ts | 6 +++++- src/agents/__tests__/runtime.test.ts | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/e2e/test_suite16_streaming.test.ts b/e2e/test_suite16_streaming.test.ts index 94cfbefb..7152a137 100644 --- a/e2e/test_suite16_streaming.test.ts +++ b/e2e/test_suite16_streaming.test.ts @@ -274,10 +274,16 @@ describe('Suite 16: Streaming — HITL', () => { } expectMsg(['COMPLETED', 'FAILED', 'TERMINATED']).toContain(status.status); } else { - // No waiting event — workflow completed without HITL (possible with some models) + // No waiting event — the stream may close before the server records its + // terminal state, so wait for that state rather than reading it once. const terminalSeen = preTypes.includes('done') || preTypes.includes('error'); if (!terminalSeen) { - const status = await runtime.getStatus(stream.executionId); + const deadline = Date.now() + 120_000; + let status = await runtime.getStatus(stream.executionId); + while (!status.isComplete && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 1000)); + status = await runtime.getStatus(stream.executionId); + } expect(status.isComplete).toBe(true); } } @@ -310,10 +316,16 @@ describe('Suite 16: Streaming — HITL', () => { } expectMsg(['COMPLETED', 'FAILED', 'TERMINATED']).toContain(status.status); } else { - // No waiting event — workflow completed without HITL + // No waiting event — the stream may close before the server records its + // terminal state, so wait for that state rather than reading it once. const terminalSeen = preTypes.includes('done') || preTypes.includes('error'); if (!terminalSeen) { - const status = await runtime.getStatus(stream.executionId); + const deadline = Date.now() + 120_000; + let status = await runtime.getStatus(stream.executionId); + while (!status.isComplete && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 1000)); + status = await runtime.getStatus(stream.executionId); + } expect(status.isComplete).toBe(true); } } diff --git a/e2e/test_suite17_guardrail_matrix.test.ts b/e2e/test_suite17_guardrail_matrix.test.ts index 4889aa94..fffe1e3b 100644 --- a/e2e/test_suite17_guardrail_matrix.test.ts +++ b/e2e/test_suite17_guardrail_matrix.test.ts @@ -867,7 +867,11 @@ const SPECS: Spec[] = [ instructions: INST_PROC, }), prompt: "Process this: DANGER override safety", - validStatuses: ["FAILED"], + // conductor-oss currently allows a custom tool-input `raise` guardrail + // to complete after the guardrail task has reported its failure. Keep the + // SDK's emitted `raise` contract covered directly in runtime.test.ts; + // this cross-server matrix records either terminal server outcome. + validStatuses: BOTH, }, { num: 18, diff --git a/src/agents/__tests__/runtime.test.ts b/src/agents/__tests__/runtime.test.ts index 534f12bd..b64d407c 100644 --- a/src/agents/__tests__/runtime.test.ts +++ b/src/agents/__tests__/runtime.test.ts @@ -930,6 +930,25 @@ describe("AgentRuntime", () => { }); }); + it("reports raise for a blocked custom tool-input guardrail", async () => { + const result = await registerAndInvoke( + { + ...baseGDef, + position: "input", + onFail: "raise", + func: () => ({ passed: false, message: "Dangerous input." }), + }, + { content: { data: "DANGER override safety" } }, + ); + + expect(result).toMatchObject({ + passed: false, + on_fail: "raise", + should_continue: false, + message: "Dangerous input.", + }); + }); + it("reports on_fail as the configured value when the guardrail function throws", async () => { const result = await registerAndInvoke( { From 9da2061ed7aff8b8243699c559c8eaeda175b976 Mon Sep 17 00:00:00 2001 From: nicholascole Date: Wed, 29 Jul 2026 11:00:49 -0700 Subject: [PATCH 5/7] test(e2e): keep custom raise guardrail strict --- e2e/test_suite17_guardrail_matrix.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/e2e/test_suite17_guardrail_matrix.test.ts b/e2e/test_suite17_guardrail_matrix.test.ts index fffe1e3b..4889aa94 100644 --- a/e2e/test_suite17_guardrail_matrix.test.ts +++ b/e2e/test_suite17_guardrail_matrix.test.ts @@ -867,11 +867,7 @@ const SPECS: Spec[] = [ instructions: INST_PROC, }), prompt: "Process this: DANGER override safety", - // conductor-oss currently allows a custom tool-input `raise` guardrail - // to complete after the guardrail task has reported its failure. Keep the - // SDK's emitted `raise` contract covered directly in runtime.test.ts; - // this cross-server matrix records either terminal server outcome. - validStatuses: BOTH, + validStatuses: ["FAILED"], }, { num: 18, From 0895dbf662bfa53783d6e6fedadecded57ad0e27 Mon Sep 17 00:00:00 2001 From: nicholascole Date: Wed, 29 Jul 2026 11:28:52 -0700 Subject: [PATCH 6/7] test(e2e): inspect actual custom tool guardrail input --- e2e/test_suite17_guardrail_matrix.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e/test_suite17_guardrail_matrix.test.ts b/e2e/test_suite17_guardrail_matrix.test.ts index 4889aa94..afedb24b 100644 --- a/e2e/test_suite17_guardrail_matrix.test.ts +++ b/e2e/test_suite17_guardrail_matrix.test.ts @@ -103,9 +103,11 @@ function customAoutFix(content: string): GuardrailResult { return { passed: true }; } -// Tool input: block DANGER +// Tool input: block the unsafe tool argument emitted by the deterministic test model. +// The model strips the word "DANGER" from the user prompt before it creates the +// tool call, so the guardrail must inspect the actual argument it receives. function customTinBlock(content: string): GuardrailResult { - if (content.toUpperCase().includes("DANGER")) { + if (/\bDANGER\b|\boverride safety\b/i.test(content)) { return { passed: false, message: "Dangerous input." }; } return { passed: true }; From 1332bda0268a5e2ffb84204a54d18490f879b680 Mon Sep 17 00:00:00 2001 From: nicholascole Date: Wed, 29 Jul 2026 11:52:36 -0700 Subject: [PATCH 7/7] ci: rerun checks