From 05028a1f2f7e3bd35e2ea637d754964f4b6905ed Mon Sep 17 00:00:00 2001
From: Matt Pua <matt.p@posthog.com>
Date: Thu, 23 Jul 2026 11:10:51 -0400
Subject: [PATCH 1/4] fix: require structured blocking questions

Generated-By: PostHog Code
Task-Id: 1b1b8d63-c777-47f5-9fa9-3a15479a627f
---
 .../src/services/agent/agent.test.ts          |  23 +++
 .../src/services/agent/agent.ts               |   3 +
 scripts/bench-bash-tokens.mjs                 | 183 ++++++++++++++++++
 3 files changed, 209 insertions(+)
 create mode 100644 scripts/bench-bash-tokens.mjs

diff --git a/packages/workspace-server/src/services/agent/agent.test.ts b/packages/workspace-server/src/services/agent/agent.test.ts
index 293ac3854e..0b780ca8d7 100644
--- a/packages/workspace-server/src/services/agent/agent.test.ts
+++ b/packages/workspace-server/src/services/agent/agent.test.ts
@@ -845,6 +845,29 @@ describe("AgentService", () => {
       expect(prompt).toContain("If the user names a folder or path");
     });
   });
+
+  describe("system prompt questions", () => {
+    it("requires blocking questions to use AskUserQuestion", () => {
+      const prompt = (
+        service as unknown as {
+          buildSystemPrompt: (
+            credentials: { apiHost: string; projectId: number },
+            taskId: string,
+          ) => { append: string };
+        }
+      ).buildSystemPrompt(
+        { apiHost: "https://app.posthog.com", projectId: 1 },
+        "task-1",
+      ).append;
+
+      expect(prompt).toContain(
+        "When you need an answer from the user before you can continue, use the AskUserQuestion tool.",
+      );
+      expect(prompt).toContain(
+        "plain-text questions mark the task as finished",
+      );
+    });
+  });
 });
 
 describe("buildAutoApproveOutcome", () => {
diff --git a/packages/workspace-server/src/services/agent/agent.ts b/packages/workspace-server/src/services/agent/agent.ts
index a352357e3b..d42894d8c5 100644
--- a/packages/workspace-server/src/services/agent/agent.ts
+++ b/packages/workspace-server/src/services/agent/agent.ts
@@ -652,6 +652,9 @@ When creating pull requests, add the following footer at the end of the PR descr
 
 When you mention a pull request in any reply or summary, always hyperlink it to its full URL (e.g. a Markdown link like [#123](https://github.com/org/repo/pull/123)) rather than plain text, so readers can open it directly.
 
+## Questions
+When you need an answer from the user before you can continue, use the AskUserQuestion tool. Never end a turn with a blocking question in a normal assistant message because plain-text questions mark the task as finished instead of waiting for the user's response.
+
 ## Shell efficiency
 Optimize for the fewest shell round trips.
 - Batch related commands into one Bash invocation using \`&&\` (e.g. \`npm run typecheck && npm run lint && npm test\`).
diff --git a/scripts/bench-bash-tokens.mjs b/scripts/bench-bash-tokens.mjs
new file mode 100644
index 0000000000..80d1a21aa3
--- /dev/null
+++ b/scripts/bench-bash-tokens.mjs
@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const root = mkdtempSync(join(tmpdir(), "posthog-bash-token-bench-"));
+const fixture = join(root, "fixture");
+mkdirSync(fixture);
+
+function run(bin, args, cwd = fixture) {
+  const startedAt = performance.now();
+  const result = spawnSync(bin, args, {
+    cwd,
+    encoding: "utf8",
+    env: { ...process.env, NO_COLOR: "1", FORCE_COLOR: "0" },
+    maxBuffer: 64 * 1024 * 1024,
+  });
+  return {
+    output: `${result.stdout ?? ""}${result.stderr ?? ""}`,
+    status: result.status,
+    durationMs: performance.now() - startedAt,
+    error: result.error?.message,
+  };
+}
+
+function estimatedTokens(output) {
+  return Math.ceil(Buffer.byteLength(output, "utf8") / 4);
+}
+
+function createFixture() {
+  const sourceDir = join(fixture, "src");
+  mkdirSync(sourceDir);
+  for (let fileIndex = 0; fileIndex < 80; fileIndex++) {
+    const lines = [];
+    for (let lineIndex = 0; lineIndex < 80; lineIndex++) {
+      lines.push(
+        `export const repeated_${fileIndex}_${lineIndex} = "TOKEN_BENCH_MATCH shared diagnostic payload ${lineIndex}";`,
+      );
+    }
+    lines.push(`export const unique_${fileIndex} = "UNIQUE_FILE_${fileIndex}";`);
+    writeFileSync(join(sourceDir, `module-${String(fileIndex).padStart(3, "0")}.ts`), `${lines.join("\n")}\n`);
+  }
+
+  const generated = {};
+  for (let index = 0; index < 1200; index++) {
+    generated[`property_${String(index).padStart(4, "0")}`] = {
+      status: index % 3 === 0 ? "active" : "inactive",
+      value: `TOKEN_BENCH_JSON_${index}`,
+    };
+  }
+  writeFileSync(join(fixture, "large.json"), JSON.stringify(generated, null, 2));
+  writeFileSync(
+    join(fixture, "package.json"),
+    JSON.stringify({
+      name: "bash-token-benchmark",
+      private: true,
+      scripts: { test: "node noisy-test.mjs" },
+    }),
+  );
+  writeFileSync(
+    join(fixture, "noisy-test.mjs"),
+    `for (let i = 0; i < 1500; i++) console.log(\`PASS test case \${i}: repeated assertion details\`);\nconsole.error("FAIL intentional TOKEN_BENCH_TEST_FAILURE");\nprocess.exit(1);\n`,
+  );
+
+  run("git", ["init", "-q"]);
+  run("git", ["config", "user.email", "benchmark@example.com"]);
+  run("git", ["config", "user.name", "Benchmark"]);
+  run("git", ["add", "."]);
+  run("git", ["commit", "-qm", "benchmark baseline"]);
+  for (let index = 0; index < 40; index++) {
+    writeFileSync(join(sourceDir, `untracked-${String(index).padStart(3, "0")}.txt`), `TOKEN_BENCH_UNTRACKED_${index}\n`);
+  }
+  writeFileSync(
+    join(sourceDir, "module-000.ts"),
+    `${Array.from({ length: 600 }, (_, index) => `export const changed_${index} = "TOKEN_BENCH_DIFF_${index}";`).join("\n")}\n`,
+  );
+}
+
+createFixture();
+
+const cases = [
+  {
+    name: "recursive search",
+    raw: ["rg", ["-n", "TOKEN_BENCH_MATCH", "src"]],
+    optimized: ["rtk", ["rg", "-n", "TOKEN_BENCH_MATCH", "src"]],
+    required: ["TOKEN_BENCH_MATCH", "module-000.ts", "module-079.ts"],
+  },
+  {
+    name: "file discovery",
+    raw: ["find", ["src", "-type", "f"]],
+    optimized: ["rtk", ["find", "src", "-type", "f"]],
+    required: ["module-000.ts", "module-079.ts"],
+  },
+  {
+    name: "directory listing",
+    raw: ["ls", ["-la", "src"]],
+    optimized: ["rtk", ["ls", "-la", "src"]],
+    required: ["module-000.ts", "module-079.ts"],
+  },
+  {
+    name: "git status",
+    raw: ["git", ["status", "--short"]],
+    optimized: ["rtk", ["git", "status", "--short"]],
+    required: ["module-000.ts", "untracked-000.txt"],
+  },
+  {
+    name: "git diff",
+    raw: ["git", ["diff", "--", "src/module-000.ts"]],
+    optimized: ["rtk", ["git", "diff", "--", "src/module-000.ts"]],
+    required: ["module-000.ts", "TOKEN_BENCH_DIFF_599"],
+  },
+  {
+    name: "json response",
+    raw: ["curl", ["-sS", `file://${join(fixture, "large.json")}`]],
+    optimized: ["rtk", ["curl", "-sS", `file://${join(fixture, "large.json")}`]],
+    required: ["property_0000", "property_1199"],
+  },
+  {
+    name: "failing test output",
+    raw: ["pnpm", ["test"]],
+    optimized: ["rtk", ["pnpm", "test"]],
+    required: ["TOKEN_BENCH_TEST_FAILURE"],
+  },
+];
+
+try {
+  const results = [];
+  for (const benchmarkCase of cases) {
+    const raw = run(...benchmarkCase.raw);
+    const optimized = run(...benchmarkCase.optimized);
+    const missing = benchmarkCase.required.filter(
+      (marker) => !optimized.output.includes(marker),
+    );
+    const valid =
+      !raw.error &&
+      !optimized.error &&
+      raw.status === optimized.status &&
+      missing.length === 0;
+    results.push({
+      name: benchmarkCase.name,
+      valid,
+      rawExit: raw.status,
+      optimizedExit: optimized.status,
+      missing,
+      rawTokens: estimatedTokens(raw.output),
+      optimizedTokens: estimatedTokens(optimized.output),
+      savingsPercent:
+        raw.output.length === 0
+          ? 0
+          : Number(
+              (
+                (1 - estimatedTokens(optimized.output) / estimatedTokens(raw.output)) *
+                100
+              ).toFixed(2),
+            ),
+      rawDurationMs: Number(raw.durationMs.toFixed(1)),
+      optimizedDurationMs: Number(optimized.durationMs.toFixed(1)),
+    });
+  }
+
+  const invalid = results.filter((result) => !result.valid);
+  const rawTokens = results.reduce((sum, result) => sum + result.rawTokens, 0);
+  const optimizedTokens = results.reduce(
+    (sum, result) => sum + result.optimizedTokens,
+    0,
+  );
+  const report = {
+    corpusCases: results.length,
+    validCases: results.length - invalid.length,
+    rawTokens,
+    optimizedTokens,
+    savingsPercent: Number(((1 - optimizedTokens / rawTokens) * 100).toFixed(2)),
+    estimator: "ceil(UTF-8 bytes / 4)",
+    results,
+  };
+  console.log(JSON.stringify(report, null, 2));
+  console.log(`BASH_OUTPUT_TOKENS=${optimizedTokens}`);
+  if (invalid.length > 0) process.exitCode = 1;
+} finally {
+  rmSync(root, { recursive: true, force: true });
+}

From 4fdbfbfa6594a30da69cbf9e0d5a2e865e0f37bb Mon Sep 17 00:00:00 2001
From: Matt Pua <matt.p@posthog.com>
Date: Thu, 23 Jul 2026 11:11:35 -0400
Subject: [PATCH 2/4] chore: keep benchmark work out of question fix

Generated-By: PostHog Code
Task-Id: 1b1b8d63-c777-47f5-9fa9-3a15479a627f
---
 scripts/bench-bash-tokens.mjs | 183 ----------------------------------
 1 file changed, 183 deletions(-)
 delete mode 100644 scripts/bench-bash-tokens.mjs

diff --git a/scripts/bench-bash-tokens.mjs b/scripts/bench-bash-tokens.mjs
deleted file mode 100644
index 80d1a21aa3..0000000000
--- a/scripts/bench-bash-tokens.mjs
+++ /dev/null
@@ -1,183 +0,0 @@
-#!/usr/bin/env node
-
-import { spawnSync } from "node:child_process";
-import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-const root = mkdtempSync(join(tmpdir(), "posthog-bash-token-bench-"));
-const fixture = join(root, "fixture");
-mkdirSync(fixture);
-
-function run(bin, args, cwd = fixture) {
-  const startedAt = performance.now();
-  const result = spawnSync(bin, args, {
-    cwd,
-    encoding: "utf8",
-    env: { ...process.env, NO_COLOR: "1", FORCE_COLOR: "0" },
-    maxBuffer: 64 * 1024 * 1024,
-  });
-  return {
-    output: `${result.stdout ?? ""}${result.stderr ?? ""}`,
-    status: result.status,
-    durationMs: performance.now() - startedAt,
-    error: result.error?.message,
-  };
-}
-
-function estimatedTokens(output) {
-  return Math.ceil(Buffer.byteLength(output, "utf8") / 4);
-}
-
-function createFixture() {
-  const sourceDir = join(fixture, "src");
-  mkdirSync(sourceDir);
-  for (let fileIndex = 0; fileIndex < 80; fileIndex++) {
-    const lines = [];
-    for (let lineIndex = 0; lineIndex < 80; lineIndex++) {
-      lines.push(
-        `export const repeated_${fileIndex}_${lineIndex} = "TOKEN_BENCH_MATCH shared diagnostic payload ${lineIndex}";`,
-      );
-    }
-    lines.push(`export const unique_${fileIndex} = "UNIQUE_FILE_${fileIndex}";`);
-    writeFileSync(join(sourceDir, `module-${String(fileIndex).padStart(3, "0")}.ts`), `${lines.join("\n")}\n`);
-  }
-
-  const generated = {};
-  for (let index = 0; index < 1200; index++) {
-    generated[`property_${String(index).padStart(4, "0")}`] = {
-      status: index % 3 === 0 ? "active" : "inactive",
-      value: `TOKEN_BENCH_JSON_${index}`,
-    };
-  }
-  writeFileSync(join(fixture, "large.json"), JSON.stringify(generated, null, 2));
-  writeFileSync(
-    join(fixture, "package.json"),
-    JSON.stringify({
-      name: "bash-token-benchmark",
-      private: true,
-      scripts: { test: "node noisy-test.mjs" },
-    }),
-  );
-  writeFileSync(
-    join(fixture, "noisy-test.mjs"),
-    `for (let i = 0; i < 1500; i++) console.log(\`PASS test case \${i}: repeated assertion details\`);\nconsole.error("FAIL intentional TOKEN_BENCH_TEST_FAILURE");\nprocess.exit(1);\n`,
-  );
-
-  run("git", ["init", "-q"]);
-  run("git", ["config", "user.email", "benchmark@example.com"]);
-  run("git", ["config", "user.name", "Benchmark"]);
-  run("git", ["add", "."]);
-  run("git", ["commit", "-qm", "benchmark baseline"]);
-  for (let index = 0; index < 40; index++) {
-    writeFileSync(join(sourceDir, `untracked-${String(index).padStart(3, "0")}.txt`), `TOKEN_BENCH_UNTRACKED_${index}\n`);
-  }
-  writeFileSync(
-    join(sourceDir, "module-000.ts"),
-    `${Array.from({ length: 600 }, (_, index) => `export const changed_${index} = "TOKEN_BENCH_DIFF_${index}";`).join("\n")}\n`,
-  );
-}
-
-createFixture();
-
-const cases = [
-  {
-    name: "recursive search",
-    raw: ["rg", ["-n", "TOKEN_BENCH_MATCH", "src"]],
-    optimized: ["rtk", ["rg", "-n", "TOKEN_BENCH_MATCH", "src"]],
-    required: ["TOKEN_BENCH_MATCH", "module-000.ts", "module-079.ts"],
-  },
-  {
-    name: "file discovery",
-    raw: ["find", ["src", "-type", "f"]],
-    optimized: ["rtk", ["find", "src", "-type", "f"]],
-    required: ["module-000.ts", "module-079.ts"],
-  },
-  {
-    name: "directory listing",
-    raw: ["ls", ["-la", "src"]],
-    optimized: ["rtk", ["ls", "-la", "src"]],
-    required: ["module-000.ts", "module-079.ts"],
-  },
-  {
-    name: "git status",
-    raw: ["git", ["status", "--short"]],
-    optimized: ["rtk", ["git", "status", "--short"]],
-    required: ["module-000.ts", "untracked-000.txt"],
-  },
-  {
-    name: "git diff",
-    raw: ["git", ["diff", "--", "src/module-000.ts"]],
-    optimized: ["rtk", ["git", "diff", "--", "src/module-000.ts"]],
-    required: ["module-000.ts", "TOKEN_BENCH_DIFF_599"],
-  },
-  {
-    name: "json response",
-    raw: ["curl", ["-sS", `file://${join(fixture, "large.json")}`]],
-    optimized: ["rtk", ["curl", "-sS", `file://${join(fixture, "large.json")}`]],
-    required: ["property_0000", "property_1199"],
-  },
-  {
-    name: "failing test output",
-    raw: ["pnpm", ["test"]],
-    optimized: ["rtk", ["pnpm", "test"]],
-    required: ["TOKEN_BENCH_TEST_FAILURE"],
-  },
-];
-
-try {
-  const results = [];
-  for (const benchmarkCase of cases) {
-    const raw = run(...benchmarkCase.raw);
-    const optimized = run(...benchmarkCase.optimized);
-    const missing = benchmarkCase.required.filter(
-      (marker) => !optimized.output.includes(marker),
-    );
-    const valid =
-      !raw.error &&
-      !optimized.error &&
-      raw.status === optimized.status &&
-      missing.length === 0;
-    results.push({
-      name: benchmarkCase.name,
-      valid,
-      rawExit: raw.status,
-      optimizedExit: optimized.status,
-      missing,
-      rawTokens: estimatedTokens(raw.output),
-      optimizedTokens: estimatedTokens(optimized.output),
-      savingsPercent:
-        raw.output.length === 0
-          ? 0
-          : Number(
-              (
-                (1 - estimatedTokens(optimized.output) / estimatedTokens(raw.output)) *
-                100
-              ).toFixed(2),
-            ),
-      rawDurationMs: Number(raw.durationMs.toFixed(1)),
-      optimizedDurationMs: Number(optimized.durationMs.toFixed(1)),
-    });
-  }
-
-  const invalid = results.filter((result) => !result.valid);
-  const rawTokens = results.reduce((sum, result) => sum + result.rawTokens, 0);
-  const optimizedTokens = results.reduce(
-    (sum, result) => sum + result.optimizedTokens,
-    0,
-  );
-  const report = {
-    corpusCases: results.length,
-    validCases: results.length - invalid.length,
-    rawTokens,
-    optimizedTokens,
-    savingsPercent: Number(((1 - optimizedTokens / rawTokens) * 100).toFixed(2)),
-    estimator: "ceil(UTF-8 bytes / 4)",
-    results,
-  };
-  console.log(JSON.stringify(report, null, 2));
-  console.log(`BASH_OUTPUT_TOKENS=${optimizedTokens}`);
-  if (invalid.length > 0) process.exitCode = 1;
-} finally {
-  rmSync(root, { recursive: true, force: true });
-}

From d3043144701395d7b6857b41a89bb3116f1ab6ad Mon Sep 17 00:00:00 2001
From: Matt Pua <matt.p@posthog.com>
Date: Thu, 23 Jul 2026 11:14:15 -0400
Subject: [PATCH 3/4] fix: document question tools for both adapters

Generated-By: PostHog Code
Task-Id: 1b1b8d63-c777-47f5-9fa9-3a15479a627f
---
 packages/workspace-server/src/services/agent/agent.test.ts | 4 ++--
 packages/workspace-server/src/services/agent/agent.ts      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/workspace-server/src/services/agent/agent.test.ts b/packages/workspace-server/src/services/agent/agent.test.ts
index 0b780ca8d7..40ae4fb9f6 100644
--- a/packages/workspace-server/src/services/agent/agent.test.ts
+++ b/packages/workspace-server/src/services/agent/agent.test.ts
@@ -847,7 +847,7 @@ describe("AgentService", () => {
   });
 
   describe("system prompt questions", () => {
-    it("requires blocking questions to use AskUserQuestion", () => {
+    it("requires blocking questions to use the adapter's question tool", () => {
       const prompt = (
         service as unknown as {
           buildSystemPrompt: (
@@ -861,7 +861,7 @@ describe("AgentService", () => {
       ).append;
 
       expect(prompt).toContain(
-        "When you need an answer from the user before you can continue, use the AskUserQuestion tool.",
+        "AskUserQuestion for Claude, or request_user_input in Codex Plan mode",
       );
       expect(prompt).toContain(
         "plain-text questions mark the task as finished",
diff --git a/packages/workspace-server/src/services/agent/agent.ts b/packages/workspace-server/src/services/agent/agent.ts
index d42894d8c5..42cdd402e5 100644
--- a/packages/workspace-server/src/services/agent/agent.ts
+++ b/packages/workspace-server/src/services/agent/agent.ts
@@ -653,7 +653,7 @@ When creating pull requests, add the following footer at the end of the PR descr
 When you mention a pull request in any reply or summary, always hyperlink it to its full URL (e.g. a Markdown link like [#123](https://github.com/org/repo/pull/123)) rather than plain text, so readers can open it directly.
 
 ## Questions
-When you need an answer from the user before you can continue, use the AskUserQuestion tool. Never end a turn with a blocking question in a normal assistant message because plain-text questions mark the task as finished instead of waiting for the user's response.
+When you need an answer from the user before you can continue, use the structured question tool available to you: AskUserQuestion for Claude, or request_user_input in Codex Plan mode. Never end a turn with a blocking question in a normal assistant message because plain-text questions mark the task as finished instead of waiting for the user's response.
 
 ## Shell efficiency
 Optimize for the fewest shell round trips.

From dc0d803c6671c80f1443b42480b616dd00bb955d Mon Sep 17 00:00:00 2001
From: Matt Pua <matt.p@posthog.com>
Date: Thu, 23 Jul 2026 11:17:44 -0400
Subject: [PATCH 4/4] fix: make question guidance provider independent

Generated-By: PostHog Code
Task-Id: 1b1b8d63-c777-47f5-9fa9-3a15479a627f
---
 packages/workspace-server/src/services/agent/agent.test.ts | 4 ++--
 packages/workspace-server/src/services/agent/agent.ts      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/workspace-server/src/services/agent/agent.test.ts b/packages/workspace-server/src/services/agent/agent.test.ts
index 40ae4fb9f6..14204e4065 100644
--- a/packages/workspace-server/src/services/agent/agent.test.ts
+++ b/packages/workspace-server/src/services/agent/agent.test.ts
@@ -847,7 +847,7 @@ describe("AgentService", () => {
   });
 
   describe("system prompt questions", () => {
-    it("requires blocking questions to use the adapter's question tool", () => {
+    it("requires blocking questions to use a structured user-input tool", () => {
       const prompt = (
         service as unknown as {
           buildSystemPrompt: (
@@ -861,7 +861,7 @@ describe("AgentService", () => {
       ).append;
 
       expect(prompt).toContain(
-        "AskUserQuestion for Claude, or request_user_input in Codex Plan mode",
+        "use the structured user-input tool available in your current mode",
       );
       expect(prompt).toContain(
         "plain-text questions mark the task as finished",
diff --git a/packages/workspace-server/src/services/agent/agent.ts b/packages/workspace-server/src/services/agent/agent.ts
index 42cdd402e5..9fa5c7db8a 100644
--- a/packages/workspace-server/src/services/agent/agent.ts
+++ b/packages/workspace-server/src/services/agent/agent.ts
@@ -653,7 +653,7 @@ When creating pull requests, add the following footer at the end of the PR descr
 When you mention a pull request in any reply or summary, always hyperlink it to its full URL (e.g. a Markdown link like [#123](https://github.com/org/repo/pull/123)) rather than plain text, so readers can open it directly.
 
 ## Questions
-When you need an answer from the user before you can continue, use the structured question tool available to you: AskUserQuestion for Claude, or request_user_input in Codex Plan mode. Never end a turn with a blocking question in a normal assistant message because plain-text questions mark the task as finished instead of waiting for the user's response.
+When you need an answer from the user before you can continue, use the structured user-input tool available in your current mode. Never end a turn with a blocking question in a normal assistant message because plain-text questions mark the task as finished instead of waiting for the user's response.
 
 ## Shell efficiency
 Optimize for the fewest shell round trips.