From 9639a3b4c2b2bd613075b967755669533cc9258d Mon Sep 17 00:00:00 2001 From: dezsow <13318493+dezsow@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:06:05 -0400 Subject: [PATCH 1/3] test(rules): mirror LC-112 in fixture --- .../rules-fixture/langchain/agent_safety.yaml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/testdata/rules-fixture/langchain/agent_safety.yaml b/testdata/rules-fixture/langchain/agent_safety.yaml index 2350f547..a9897cc5 100644 --- a/testdata/rules-fixture/langchain/agent_safety.yaml +++ b/testdata/rules-fixture/langchain/agent_safety.yaml @@ -83,3 +83,37 @@ rules: Pass maxIterations to the AgentExecutor options, sized to the task, and set handleParsingErrors so a malformed step is surfaced rather than retried indefinitely. + + - id: LC-112 + title: LangChain privileged create_agent lacks human-approval prerequisites + severity: high + confidence: 0.7 + language: python + applies_to: + - langchain_agent + scope: agent + match: + all: + - agent_class: + - CreateAgent + - agent_uses_hosted_tool_class: + - PythonREPLTool + - PythonAstREPLTool + - ShellTool + - any: + - agent_kwarg_list_empty: + - middleware + - agent_kwarg_missing: + - checkpointer + explanation: > + This LangChain v1 create_agent exposes a built-in that can execute arbitrary + Python or shell commands, but it lacks middleware and/or checkpoint state + needed to support a resumable human approval boundary. Without those + structural prerequisites, the model can reach a high-impact execution tool + without an explicit interrupt/resume control point that Trustabl can verify. + fix: > + Add human-in-the-loop approval middleware for the privileged tool and pass a + checkpointer so interrupted execution can be persisted and resumed after the + decision. Keep the REPL/shell tool sandboxed and narrowly scoped. This rule + only verifies the structural prerequisites; review that the middleware is in + fact configured to require approval for the privileged tool. From e06293e3ead4c03c941a59696c7e3079770c2b73 Mon Sep 17 00:00:00 2001 From: dezsow <13318493+dezsow@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:06:23 -0400 Subject: [PATCH 2/3] test(rules): cover LC-112 human approval prerequisites --- internal/rules/lc112_policy_test.go | 101 ++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 internal/rules/lc112_policy_test.go diff --git a/internal/rules/lc112_policy_test.go b/internal/rules/lc112_policy_test.go new file mode 100644 index 00000000..a6088c34 --- /dev/null +++ b/internal/rules/lc112_policy_test.go @@ -0,0 +1,101 @@ +package rules_test + +import ( + "testing" + + "github.com/trustabl/trustabl/internal/models" +) + +func TestLC112HumanApprovalPrerequisites(t *testing.T) { + middleware := func(items ...models.Expr) *models.KwargTree { + return &models.KwargTree{ + Value: &models.Expr{ + Kind: models.ExprList, + List: items, + }, + } + } + checkpointer := &models.KwargTree{ + Value: &models.Expr{Kind: models.ExprCall, Text: "InMemorySaver()"}, + } + + cases := []struct { + name string + agent models.AgentDef + wantFires bool + }{ + { + name: "fires with privileged tool and no approval prerequisites", + agent: models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "ShellTool"}}, + }, + wantFires: true, + }, + { + name: "fires with middleware but no checkpointer", + agent: models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "ShellTool"}}, + Kwargs: &models.KwargTree{Children: map[string]*models.KwargTree{ + "middleware": middleware(models.Expr{Kind: models.ExprCall, Text: "HumanInTheLoopMiddleware(...)"}), + }}, + }, + wantFires: true, + }, + { + name: "fires with empty middleware even when checkpointer is present", + agent: models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "PythonREPLTool"}}, + Kwargs: &models.KwargTree{Children: map[string]*models.KwargTree{ + "middleware": middleware(), + "checkpointer": checkpointer, + }}, + }, + wantFires: true, + }, + { + name: "silent with middleware and checkpointer", + agent: models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "ShellTool"}}, + Kwargs: &models.KwargTree{Children: map[string]*models.KwargTree{ + "middleware": middleware(models.Expr{Kind: models.ExprCall, Text: "HumanInTheLoopMiddleware(...)"}), + "checkpointer": checkpointer, + }}, + }, + wantFires: false, + }, + { + name: "silent without a privileged execution tool", + agent: models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + }, + wantFires: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := loadAgentRule(t, "LC-112") + if !d.Applies(tc.agent) { + if tc.wantFires { + t.Fatalf("LC-112 does not apply to %s/%s", tc.agent.SDK, tc.agent.Class) + } + return + } + + fired := false + for _, f := range d.Detect(tc.agent, models.RepoInventory{}) { + if f.RuleID == "LC-112" { + fired = true + break + } + } + if fired != tc.wantFires { + t.Fatalf("LC-112 fired=%v, want %v", fired, tc.wantFires) + } + }) + } +} From f79c56b5fc64eaca59e001eee2ae04a5885686aa Mon Sep 17 00:00:00 2001 From: sairenchristianbuerano Date: Thu, 15 Oct 2026 07:01:38 +0800 Subject: [PATCH 3/3] fix(rules): renumber the human-approval rule to LC-117 LC-112 was free when this branch was opened on 24 August. It is not now: a tracing rule took it in langchain/tracing.yaml during the 260 commits since, so the loader refused the pack with a duplicate id and every subtest failed. Renumbered to LC-117, the next free id across the whole langchain pack rather than the next free one in agent_safety.yaml alone. The test file is renamed to match. Also adds the four cases to policies_test.go that TestPolicyRules_AllRulesCovered requires. A standalone test file is not enough on its own; the gate reads the central table, and it failed with "rules with no policy_test coverage at all: [LC-117]" until these were added. check-rules-sync.sh reports in sync across 130 files, and the paired change is on the same branch name in the rules repository so both CI jobs resolve it. --- ...12_policy_test.go => lc117_policy_test.go} | 10 ++--- internal/rules/policies_test.go | 39 +++++++++++++++++++ .../rules-fixture/langchain/agent_safety.yaml | 2 +- 3 files changed, 45 insertions(+), 6 deletions(-) rename internal/rules/{lc112_policy_test.go => lc117_policy_test.go} (91%) diff --git a/internal/rules/lc112_policy_test.go b/internal/rules/lc117_policy_test.go similarity index 91% rename from internal/rules/lc112_policy_test.go rename to internal/rules/lc117_policy_test.go index a6088c34..b760cda5 100644 --- a/internal/rules/lc112_policy_test.go +++ b/internal/rules/lc117_policy_test.go @@ -6,7 +6,7 @@ import ( "github.com/trustabl/trustabl/internal/models" ) -func TestLC112HumanApprovalPrerequisites(t *testing.T) { +func TestLC117HumanApprovalPrerequisites(t *testing.T) { middleware := func(items ...models.Expr) *models.KwargTree { return &models.KwargTree{ Value: &models.Expr{ @@ -78,23 +78,23 @@ func TestLC112HumanApprovalPrerequisites(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - d := loadAgentRule(t, "LC-112") + d := loadAgentRule(t, "LC-117") if !d.Applies(tc.agent) { if tc.wantFires { - t.Fatalf("LC-112 does not apply to %s/%s", tc.agent.SDK, tc.agent.Class) + t.Fatalf("LC-117 does not apply to %s/%s", tc.agent.SDK, tc.agent.Class) } return } fired := false for _, f := range d.Detect(tc.agent, models.RepoInventory{}) { - if f.RuleID == "LC-112" { + if f.RuleID == "LC-117" { fired = true break } } if fired != tc.wantFires { - t.Fatalf("LC-112 fired=%v, want %v", fired, tc.wantFires) + t.Fatalf("LC-117 fired=%v, want %v", fired, tc.wantFires) } }) } diff --git a/internal/rules/policies_test.go b/internal/rules/policies_test.go index c1fd2f3b..46ac9dab 100644 --- a/internal/rules/policies_test.go +++ b/internal/rules/policies_test.go @@ -4947,6 +4947,45 @@ var policyAgentRuleCases = []policyAgentCase{ }, models.RepoInventory{}, false}, + {"LC-117 fires when a privileged create_agent has no approval prerequisites", "LC-117", + models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "ShellTool"}}, + }, + models.RepoInventory{}, + true}, + {"LC-117 fires when middleware is present but no checkpointer", "LC-117", + models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "PythonREPLTool"}}, + Kwargs: &models.KwargTree{Children: map[string]*models.KwargTree{ + "middleware": {Value: &models.Expr{Kind: models.ExprList, List: []models.Expr{ + {Kind: models.ExprCall, Text: "HumanInTheLoopMiddleware(...)"}, + }}}, + }}, + }, + models.RepoInventory{}, + true}, + {"LC-117 silent with both middleware and a checkpointer", "LC-117", + models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "ShellTool"}}, + Kwargs: &models.KwargTree{Children: map[string]*models.KwargTree{ + "middleware": {Value: &models.Expr{Kind: models.ExprList, List: []models.Expr{ + {Kind: models.ExprCall, Text: "HumanInTheLoopMiddleware(...)"}, + }}}, + "checkpointer": {Value: &models.Expr{Kind: models.ExprCall, Text: "InMemorySaver()"}}, + }}, + }, + models.RepoInventory{}, + false}, + {"LC-117 silent without a privileged execution tool", "LC-117", + models.AgentDef{ + SDK: models.SDKLangChain, Class: "CreateAgent", Language: models.LanguagePython, + HostedToolRefs: []models.HostedToolRef{{Class: "TavilySearchResults"}}, + }, + models.RepoInventory{}, + false}, {"VAI-101 fires when an instrumented repo has a call with no telemetry", "VAI-101", models.AgentDef{ SDK: models.SDKVercelAI, Class: "GenerateText", Language: models.LanguageTypeScript, diff --git a/testdata/rules-fixture/langchain/agent_safety.yaml b/testdata/rules-fixture/langchain/agent_safety.yaml index 57f83294..b61a0de8 100644 --- a/testdata/rules-fixture/langchain/agent_safety.yaml +++ b/testdata/rules-fixture/langchain/agent_safety.yaml @@ -220,7 +220,7 @@ rules: config={"configurable": {"thread_id": ...}} so each conversation resumes from its own persisted thread. - - id: LC-112 + - id: LC-117 title: LangChain privileged create_agent lacks human-approval prerequisites severity: high confidence: 0.7