From cc025ed0793af9abad5f89c1c842ee84ff8ed398 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Tue, 14 Jul 2026 10:57:34 -0700 Subject: [PATCH 1/3] fix(compiler): guardrail retry->raise escalation (live loop iteration + custom coercion) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two compiler bugs made custom OUTPUT guardrails with onFail=RETRY never escalate to raise after maxRetries — the agent ran to COMPLETED instead of FAILED/TERMINATED (Suite8/Suite8b escalation asserts). (a) Wrong iteration reference path. The compiler read the live loop counter as ${.iteration}, but Conductor exposes {input, output} per task ref, so the counter lives under output. The bare reference resolves to null mid-loop, silently disabling every iteration-based escalation. Use ${.output.iteration} in GuardrailCompiler (regex/llm/custom), TerminationCompiler, and the MultiAgentCompiler round-robin selector. Verified by an engine probe: a loop-body task reading ${loop.output.iteration} resolves to the live int 1,2,3; the bare form resolves to null. (b) Custom-guardrail normalize never escalated. customGuardrailNormalizeScript returned on_fail unchanged and compileCustomGuardrail passed neither iteration nor max_retries. Add the retry->raise (and fix->raise) coercion the regex/llm scripts already have, and wire iteration (= ${.output.iteration}) + max_retries (= guard.getMaxRetries()) into the normalize INLINE. Validated end-to-end without an LLM: a DO_WHILE whose body is [normalize INLINE (coercion), route SWITCH -> TERMINATE] escalates retry->raise at max_retries and the workflow ends FAILED, not COMPLETED. Full module suite green (703/703); added a regression test asserting the normalize wiring. Co-Authored-By: Claude Opus 4.8 --- .../runtime/compiler/GuardrailCompiler.java | 10 ++++++- .../runtime/compiler/MultiAgentCompiler.java | 2 +- .../runtime/compiler/TerminationCompiler.java | 8 ++--- .../runtime/util/JavaScriptBuilder.java | 17 ++++++++--- .../compiler/GuardrailCompilerTest.java | 29 +++++++++++++++++++ .../compiler/TerminationCompilerTest.java | 3 +- 6 files changed, 58 insertions(+), 11 deletions(-) diff --git a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/GuardrailCompiler.java b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/GuardrailCompiler.java index 1f0b60606..300b76213 100644 --- a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/GuardrailCompiler.java +++ b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/GuardrailCompiler.java @@ -83,7 +83,11 @@ public List compileGuardrailTasks( return new ArrayList<>(); } - String iterationRef = "${" + agentName + "_loop.iteration}"; + // NOTE: the live loop counter lives under the DO_WHILE task's OUTPUT + // (Conductor exposes {input, output} per task ref), so it must be read as + // ${.output.iteration}. The bare ${.iteration} resolves to null + // mid-loop, which silently disabled every iteration-based escalation below. + String iterationRef = "${" + agentName + "_loop.output.iteration}"; return compileGuardrailTasksInternal(outputGuardrails, agentName, contentRef, iterationRef); } @@ -262,6 +266,10 @@ private GuardrailTaskResult compileCustomGuardrail( normalizeInputs.put("worker_output", "${" + workerRef + ".output}"); normalizeInputs.put("guardrail_name", guard.getName()); normalizeInputs.put("default_on_fail", guard.getOnFail()); + // Wire the live loop counter + retry budget so the normalize script can escalate + // retry -> raise once maxRetries is exhausted (parity with regex/llm guardrails). + normalizeInputs.put("iteration", iterationRef); + normalizeInputs.put("max_retries", guard.getMaxRetries()); normalizeTask.setInputParameters(normalizeInputs); return new GuardrailTaskResult(List.of(task, normalizeTask), refName, true); diff --git a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/MultiAgentCompiler.java b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/MultiAgentCompiler.java index 87a5020d7..1af1b1b3d 100644 --- a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/MultiAgentCompiler.java +++ b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/MultiAgentCompiler.java @@ -1050,7 +1050,7 @@ private WorkflowDef compileRotation(AgentConfig config, boolean random) { Map selectInputs = new LinkedHashMap<>(); selectInputs.put("evaluatorType", "graaljs"); selectInputs.put("expression", selectScript); - selectInputs.put("iteration", ref(loopRef + ".iteration")); + selectInputs.put("iteration", ref(loopRef + ".output.iteration")); if (config.getAllowedTransitions() != null) { selectInputs.put("last_agent", "${workflow.variables.last_agent}"); } diff --git a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/TerminationCompiler.java b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/TerminationCompiler.java index 79515e817..9f7dfa44f 100644 --- a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/TerminationCompiler.java +++ b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/compiler/TerminationCompiler.java @@ -38,7 +38,7 @@ public static WorkflowTask compileTermination(TerminationConfig config, String a String taskName = agentName + "_termination"; String refName = agentName + "_termination"; String resultRef = "${" + llmRef + ".output.result}"; - String iterationRef = "${" + agentName + "_loop.iteration}"; + String iterationRef = "${" + agentName + "_loop.output.iteration}"; // Match local compiler: emit a SIMPLE worker task. // The Python runtime registers a worker that evaluates the termination condition. @@ -69,7 +69,7 @@ public static WorkflowTask compileStopWhen(String taskName, String agentName, St agentName = AgentCompiler.toRef(agentName); String refName = agentName + "_stop_when"; String resultRef = "${" + llmRef + ".output.result}"; - String iterationRef = "${" + agentName + "_loop.iteration}"; + String iterationRef = "${" + agentName + "_loop.output.iteration}"; WorkflowTask task = new WorkflowTask(); task.setName(taskName); @@ -106,7 +106,7 @@ public static WorkflowTask compileStopWhenForConversation(String taskName, Strin Map inputs = new LinkedHashMap<>(); inputs.put("result", "${workflow.variables.conversation}"); - inputs.put("iteration", "${" + loopRef + ".iteration}"); + inputs.put("iteration", "${" + loopRef + ".output.iteration}"); task.setInputParameters(inputs); return task; @@ -134,7 +134,7 @@ public static WorkflowTask compileTerminationForConversation( Map inputs = new LinkedHashMap<>(); inputs.put("result", "${workflow.variables.conversation}"); - inputs.put("iteration", "${" + loopRef + ".iteration}"); + inputs.put("iteration", "${" + loopRef + ".output.iteration}"); task.setInputParameters(inputs); return task; diff --git a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java index 9883d4ba9..4e29ea9e6 100644 --- a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java +++ b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java @@ -431,6 +431,13 @@ public static String guardrailFixScript() { public static String customGuardrailNormalizeScript() { return iife(" var raw = $.worker_output;" + " var guardrailName = $.guardrail_name || 'guardrail';" + " var defaultOnFail = $.default_on_fail || 'retry';" + + " var iteration = $.iteration || 0;" + + " var max_retries = $.max_retries || 0;" + + " function escalate(of) {" + + " if (of === 'retry' && iteration >= max_retries) return 'raise';" + + " if (of === 'fix') return 'raise';" + + " return of;" + + " }" + " if (raw == null) {" + " return {passed: true, message: '', on_fail: null," + " fixed_output: null, guardrail_name: guardrailName," @@ -449,9 +456,10 @@ public static String customGuardrailNormalizeScript() { + " var existingOnFail = raw.on_fail !== undefined ? raw.on_fail : raw.onFail;" + " var fixedOutput = raw.fixed_output !== undefined ? raw.fixed_output : raw.fixedOutput;" + " var passed = raw.passed !== false && (existingOnFail == null || existingOnFail === 'pass');" - + " return {passed: passed, message: raw.message || '', on_fail: existingOnFail," + + " var actualOnFail = passed ? existingOnFail : escalate(existingOnFail);" + + " return {passed: passed, message: raw.message || '', on_fail: actualOnFail," + " fixed_output: fixedOutput, guardrail_name: raw.guardrail_name || raw.guardrailName || guardrailName," - + " should_continue: existingOnFail === 'retry'};" + + " should_continue: actualOnFail === 'retry'};" + " }" + " if (raw != null && typeof raw === 'object'" + " && (raw.tripwire_triggered !== undefined || raw.tripwireTriggered !== undefined" @@ -469,9 +477,10 @@ public static String customGuardrailNormalizeScript() { + " fixed_output: null, guardrail_name: guardrailName," + " should_continue: false};" + " }" + + " var tripwireOnFail = escalate(defaultOnFail);" + " return {passed: false, message: reason || (guardrailName + ' triggered')," - + " on_fail: defaultOnFail, fixed_output: null," - + " guardrail_name: guardrailName, should_continue: defaultOnFail === 'retry'};" + + " on_fail: tripwireOnFail, fixed_output: null," + + " guardrail_name: guardrailName, should_continue: tripwireOnFail === 'retry'};" + " }" + " return {passed: true, message: '', on_fail: null," + " fixed_output: null, guardrail_name: guardrailName," diff --git a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java index 0342ea0f6..ca4eed2ec 100644 --- a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java +++ b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java @@ -80,6 +80,35 @@ void testCustomGuardrail() { assertThat(results.get(0).getTasks().get(1).getType()).isEqualTo("INLINE"); } + @Test + void testCustomGuardrailNormalizeWiresLiveIterationAndMaxRetriesForEscalation() { + GuardrailConfig g = GuardrailConfig.builder() + .name("custom_check") + .guardrailType("custom") + .position("output") + .taskName("my_guardrail_worker") + .onFail("retry") + .maxRetries(3) + .build(); + + GuardrailCompiler gc = new GuardrailCompiler(); + var results = gc.compileGuardrailTasks(List.of(g), "agent", "${ref}"); + + WorkflowTask normalize = results.get(0).getTasks().get(1); + assertThat(normalize.getType()).isEqualTo("INLINE"); + // The live loop counter must be read from the DO_WHILE task's OUTPUT + // (${.output.iteration}); the bare ${.iteration} resolves to null + // mid-loop and silently disables retry->raise escalation. + assertThat((String) normalize.getInputParameters().get("iteration")) + .isEqualTo("${agent_loop.output.iteration}"); + assertThat(normalize.getInputParameters().get("max_retries")).isEqualTo(3); + // The normalize script must apply the same retry->raise coercion the regex/llm + // scripts use once the retry budget is exhausted. + assertThat((String) normalize.getInputParameters().get("expression")) + .contains("iteration >= max_retries") + .contains("'raise'"); + } + @Test void testCustomGuardrailIncludesOpenAICompatibleInputAliases() { GuardrailConfig g = GuardrailConfig.builder() diff --git a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java index 5a50da56b..7890c1ca7 100644 --- a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java +++ b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java @@ -89,7 +89,8 @@ void testStopWhen() { assertThat(task.getTaskReferenceName()).isEqualTo("agent_stop_when"); // Inputs bind to LLM result, loop iteration, and messages (stop_when needs conversation history) assertThat((String) task.getInputParameters().get("result")).contains("agent_llm.output.result"); - assertThat((String) task.getInputParameters().get("iteration")).contains("agent_loop.iteration"); + assertThat((String) task.getInputParameters().get("iteration")) + .contains("agent_loop.output.iteration"); assertThat((String) task.getInputParameters().get("messages")).contains("agent_llm.input.messages"); } From 190c727f57932d92c8ad1380ccbd911d5b95305c Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Tue, 14 Jul 2026 17:39:45 -0700 Subject: [PATCH 2/3] fix(compiler): custom guardrail fix->raise only when no fixed_output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The escalation coercion added for retry->raise also coerced fix->raise unconditionally, which wrongly terminated custom `fix` guardrails that provide a fixedOutput (they should apply the fix and complete, not raise). This broke the Suite17 custom-fix cells (aout_custom_fix, tout_custom_fix: expected COMPLETED with REDACTED output, got FAILED). Make fix->raise conditional on fixedOutput being null/undefined — matching the java worker's own rule (AgentRuntime: `if fix && fixedOutput == null -> raise`). retry->raise escalation is unchanged, so Suite8 max_retries escalation still works. Verified the script logic against fix-with-output, fix-without-output, retry-at-max, and retry-below-max cases. Co-Authored-By: Claude Opus 4.8 --- .../dev/agentspan/runtime/util/JavaScriptBuilder.java | 8 ++++---- .../agentspan/runtime/compiler/GuardrailCompilerTest.java | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java index 4e29ea9e6..42524a74f 100644 --- a/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java +++ b/server/conductor-agentspan/src/main/java/dev/agentspan/runtime/util/JavaScriptBuilder.java @@ -433,9 +433,9 @@ public static String customGuardrailNormalizeScript() { + " var defaultOnFail = $.default_on_fail || 'retry';" + " var iteration = $.iteration || 0;" + " var max_retries = $.max_retries || 0;" - + " function escalate(of) {" + + " function escalate(of, fixedOutput) {" + " if (of === 'retry' && iteration >= max_retries) return 'raise';" - + " if (of === 'fix') return 'raise';" + + " if (of === 'fix' && (fixedOutput === null || fixedOutput === undefined)) return 'raise';" + " return of;" + " }" + " if (raw == null) {" @@ -456,7 +456,7 @@ public static String customGuardrailNormalizeScript() { + " var existingOnFail = raw.on_fail !== undefined ? raw.on_fail : raw.onFail;" + " var fixedOutput = raw.fixed_output !== undefined ? raw.fixed_output : raw.fixedOutput;" + " var passed = raw.passed !== false && (existingOnFail == null || existingOnFail === 'pass');" - + " var actualOnFail = passed ? existingOnFail : escalate(existingOnFail);" + + " var actualOnFail = passed ? existingOnFail : escalate(existingOnFail, fixedOutput);" + " return {passed: passed, message: raw.message || '', on_fail: actualOnFail," + " fixed_output: fixedOutput, guardrail_name: raw.guardrail_name || raw.guardrailName || guardrailName," + " should_continue: actualOnFail === 'retry'};" @@ -477,7 +477,7 @@ public static String customGuardrailNormalizeScript() { + " fixed_output: null, guardrail_name: guardrailName," + " should_continue: false};" + " }" - + " var tripwireOnFail = escalate(defaultOnFail);" + + " var tripwireOnFail = escalate(defaultOnFail, null);" + " return {passed: false, message: reason || (guardrailName + ' triggered')," + " on_fail: tripwireOnFail, fixed_output: null," + " guardrail_name: guardrailName, should_continue: tripwireOnFail === 'retry'};" diff --git a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java index ca4eed2ec..253e62d08 100644 --- a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java +++ b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/GuardrailCompilerTest.java @@ -106,7 +106,11 @@ void testCustomGuardrailNormalizeWiresLiveIterationAndMaxRetriesForEscalation() // scripts use once the retry budget is exhausted. assertThat((String) normalize.getInputParameters().get("expression")) .contains("iteration >= max_retries") - .contains("'raise'"); + .contains("'raise'") + // fix -> raise must be conditional on there being NO fixed output, otherwise + // a custom `fix` guardrail with a fixedOutput would be wrongly terminated + // instead of applying the fix (regression guard). + .contains("fixedOutput === null"); } @Test From 3668ab0c149014f99ca0e58356875046783eaf66 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Wed, 15 Jul 2026 21:28:30 -0700 Subject: [PATCH 3/3] style: apply spotless formatting Co-Authored-By: Claude Fable 5 --- .../agentspan/runtime/compiler/TerminationCompilerTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java index 7890c1ca7..fe4b8f1a0 100644 --- a/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java +++ b/server/conductor-agentspan/src/test/java/dev/agentspan/runtime/compiler/TerminationCompilerTest.java @@ -89,8 +89,7 @@ void testStopWhen() { assertThat(task.getTaskReferenceName()).isEqualTo("agent_stop_when"); // Inputs bind to LLM result, loop iteration, and messages (stop_when needs conversation history) assertThat((String) task.getInputParameters().get("result")).contains("agent_llm.output.result"); - assertThat((String) task.getInputParameters().get("iteration")) - .contains("agent_loop.output.iteration"); + assertThat((String) task.getInputParameters().get("iteration")).contains("agent_loop.output.iteration"); assertThat((String) task.getInputParameters().get("messages")).contains("agent_llm.input.messages"); }