Local Publish decides its own next step, measured first

apps/aflow-executor-ai/src/handlers/ai/handlers/decide.test.ts

@@ -0,0 +1,129 @@
+/**
+ * ai.decision.decide as a step: the questions a request asks reach the model
+ * as one pass, an item the model declines is never selected, and an `each`
+ * over an empty list is answered without a model or the credential for one.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import type { ExecutorContext } from '@aflow/executor-runtime';
+import {
+  AiDecideOutputSchema,
+  DECISION_NO_MODEL,
+  type AiDecideInput,
+  type TenantId,
+} from '@aflow/schemas';
+import { getAIClientForContext } from '../aiClient.js';
+import { handleDecide } from './decide.js';
+import { resolveDecisionModel } from './decisionModel.js';
+import type { HandlerDeps } from './types.js';
+
+vi.mock('../aiClient.js', () => ({ getAIClientForContext: vi.fn() }));
+vi.mock('./decisionModel.js', () => ({ resolveDecisionModel: vi.fn() }));
+vi.mock('@aflow/executor-runtime', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@aflow/executor-runtime')>()),
+  successWithData: vi.fn((_ctx: unknown, data: unknown) =>
+    Promise.resolve({ status: 'SUCCEEDED', data }),
+  ),
+}));
+
+const decide = vi.fn();
+const getModel = vi.fn(() => ({ decision: { confidence: 'calibrated' } }));
+
+function fakeCtx(): ExecutorContext {
+  return {
+    job: { tenantId: 'tenant-d', stepExecutionId: 'step-d', attempt: 1 },
+    tenantId: 'tenant-d' as TenantId,
+    runId: 'run-d',
+    log: { warn: () => {}, error: () => {}, info: () => {}, debug: () => {} },
+  } as unknown as ExecutorContext;
+}
+
+const deps = {
+  payloadStore: {} as HandlerDeps['payloadStore'],
+  handleError: (_ctx: ExecutorContext, label: string, error: unknown) => {
+    throw new Error(`${label}: ${String(error)}`);
+  },
+  validateToolArgs: () => null,
+} satisfies HandlerDeps;
+
+const FINDING = {
+  type: 'yes_no' as const,
+  instructions: 'Names a defect, established by evidence shown',
+  minConfidence: 0.8,
+};
+
+async function run(input: AiDecideInput) {
+  const result = (await handleDecide(fakeCtx(), input, deps)) as unknown as { data: unknown };
+  return AiDecideOutputSchema.parse(result.data);
+}
+
+describe('ai.decision.decide — a question of each item', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(getAIClientForContext).mockResolvedValue({ decide, getModel } as never);
+    vi.mocked(resolveDecisionModel).mockResolvedValue('jev');
+  });
+
+  it('asks every item in the one call, and hands back only the items judged true and decided', async () => {
+    decide.mockResolvedValue({
+      model: 'jev-1.13.0',
+      answers: {
+        each_0: { type: 'yes_no', probability: 0.97 },
+        each_1: { type: 'yes_no', probability: 0.6 },
+      },
+      usage: { promptTokens: 10, completionTokens: 0, totalTokens: 10 },
+    });
+    const output = await run({
+      state: { verdict: 'request_changes' },
+      each: { items: [{ claim: 'leaks' }, { claim: 'maybe' }], question: FINDING },
+    });
+    expect(decide).toHaveBeenCalledOnce();
+    expect(Object.keys(decide.mock.calls[0]![0].questions as object)).toEqual(['each_0', 'each_1']);
+    expect(output.answers).toEqual({});
+    expect(output.each?.selected).toEqual([{ claim: 'leaks' }]);
+    expect(output.each?.undecidedCount).toBe(1);
+  });
+
+  it('never selects an item the model declined, and counts it undecided, beside a named question', async () => {
+    decide.mockResolvedValue({
+      model: 'luna-decisions',
+      answers: {
+        route: { type: 'choice', choice: 'fix', confidence: 0.9, probabilities: { fix: 0.9 } },
+        each_0: { type: 'refusal' },
+        each_1: { type: 'yes_no', probability: 0.95 },
+      },
+      usage: { promptTokens: 10, completionTokens: 0, totalTokens: 10 },
+    });
+    const output = await run({
+      state: { verdict: 'request_changes' },
+      questions: {
+        route: {
+          type: 'choice',
+          options: { fix: 'Fix it.', ship: 'Ship it.' },
+          minConfidence: 0.8,
+        },
+      },
+      each: { items: [{ claim: 'declined' }, { claim: 'leaks' }], question: FINDING },
+    });
+    expect(output.answers['route']).toMatchObject({ decidedValue: 'fix', decided: true });
+    expect(output.each?.answers[0]).toEqual({
+      type: 'refusal',
+      decidedValue: null,
+      decided: false,
+    });
+    expect(output.each?.selected).toEqual([{ claim: 'leaks' }]);
+    expect(output.each?.undecidedCount).toBe(1);
+    expect(output.calibration).toBe('calibrated');
+  });
+
+  it('asks no model, and needs no credential, for an empty list — and selects nothing', async () => {
+    const output = await run({ state: {}, each: { items: [], question: FINDING } });
+    expect(resolveDecisionModel).not.toHaveBeenCalled();
+    expect(getAIClientForContext).not.toHaveBeenCalled();
+    expect(decide).not.toHaveBeenCalled();
+    expect(output.each).toMatchObject({ selected: [], selectedCount: 0, selectedJson: '[]' });
+    expect(output.usage.totalTokens).toBe(0);
+    // Nothing was asked, so no model answered and no confidence is claimed calibrated.
+    expect(output.model).toBe(DECISION_NO_MODEL);
+    expect(output.calibration).toBe('self_reported');
+  });
+});

apps/aflow-executor-ai/src/handlers/ai/handlers/decide.ts

@@ -3,8 +3,18 @@
  */
 import type { ExecutorContext, StepResult } from '@aflow/executor-runtime';
 import { failureWithError, successWithData } from '@aflow/executor-runtime';
-import type { AiDecideInput, AiDecideOutput, TenantId, StepExecutionId } from '@aflow/schemas';
-import { resolveDecisionAnswers } from '@aflow/ai-client';
+import {
+  type AiDecideInput,
+  type AiDecideOutput,
+  DECISION_NO_MODEL,
+  type TenantId,
+  type StepExecutionId,
+} from '@aflow/schemas';
+import {
+  decisionRequestQuestions,
+  resolveDecisionAnswers,
+  splitDecisionAnswers,
+} from '@aflow/ai-client';
 import { getAIClientForContext } from '../aiClient.js';
 import { buildUsageBreakdown } from '../helpers.js';
 import { resolveDecisionModel } from './decisionModel.js';
@@ -18,6 +28,22 @@ export async function handleDecide(
   deps: HandlerDeps,
 ): Promise<StepResult> {
   try {
+    const questions = decisionRequestQuestions(params);
+    // Only an empty `each` asks nothing; the answer is known without a model,
+    // and so without the credential resolving one would need.
+    if (Object.keys(questions).length === 0) {
+      const { answers, each } = splitDecisionAnswers(params, {});
+      const output: AiDecideOutput = {
+        answers,
+        ...(each !== undefined ? { each } : {}),
+        model: params.model ?? DECISION_NO_MODEL,
+        calibration: 'self_reported',
+        usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+        latencyMs: 0,
+      };
+      return await successWithData(ctx, output);
+    }
+
     const imageRefs = params.images ?? [];
     const model = await resolveDecisionModel(ctx, deps, {
       model: params.model,
@@ -45,16 +71,21 @@ export async function handleDecide(
     const response = await client.decide({
       model,
       state: params.state,
-      questions: params.questions,
+      questions,
       ...(images.length > 0 ? { images } : {}),
       tenantId: ctx.job.tenantId as TenantId,
       runId: ctx.runId,
       stepExecutionId: ctx.job.stepExecutionId as StepExecutionId,
       attempt: ctx.job.attempt,
     });
 
+    const { answers, each } = splitDecisionAnswers(
+      params,
+      resolveDecisionAnswers(questions, response.answers),
+    );
     const output: AiDecideOutput = {
-      answers: resolveDecisionAnswers(params.questions, response.answers),
+      answers,
+      ...(each !== undefined ? { each } : {}),
       model: response.model,
       calibration: client.getModel(model)?.decision?.confidence ?? 'self_reported',
       usage: response.usage,

apps/aflow-executor-host/src/__tests__/commitCheck.test.ts

@@ -589,7 +589,7 @@ describe('host.commit.check — a run lost only to tests running out of time', (
     );
   }
 
-  it('names the tests and the commit a publication runs the checks on again, and clears nothing', async () => {
+  it('names the tests a publication runs the checks again for, and clears nothing', async () => {
     const world = await fixture({
       branchPrefix: 'aflow/',
       checks: reportingTimeouts("console.log('FAIL tests'); process.exit(1);"),
@@ -598,7 +598,6 @@ describe('host.commit.check — a run lost only to tests running out of time', (
     expect(output.passed).toBe(false);
     expect(output.clearedSha).toBeUndefined();
     expect(output.testTimeouts).toEqual(TIMED_OUT);
-    expect(output.clearedOrRetriableSha).toBe(world.sha);
     expect(output.summary).toContain('nothing failed but 2 tests running out of time');
     expect(output.summary).toContain(TIMED_OUT.join('; '));
     expect(output.summary).toContain('a publication runs the checks once more');
@@ -612,7 +611,6 @@ describe('host.commit.check — a run lost only to tests running out of time', (
     });
     const passed = HostCommitCheckOutputSchema.parse((await check(passing)).captured.output);
     expect(passed.testTimeouts).toBeUndefined();
-    expect(passed.clearedOrRetriableSha).toBe(passing.sha);
     expect(passed.clearedSha).toBe(passing.sha);
 
     const stopped = await fixture({
@@ -623,14 +621,47 @@ describe('host.commit.check — a run lost only to tests running out of time', (
     const timedOut = HostCommitCheckOutputSchema.parse((await check(stopped)).captured.output);
     expect(timedOut).toMatchObject({ passed: false, timedOut: true });
     expect(timedOut.testTimeouts).toBeUndefined();
-    expect(timedOut.clearedOrRetriableSha).toBeUndefined();
   });
 
   it('is not claimed where the checks reported none', async () => {
     const world = await fixture({ branchPrefix: 'aflow/', checks: FAILING });
     const output = HostCommitCheckOutputSchema.parse((await check(world)).captured.output);
     expect(output.testTimeouts).toBeUndefined();
-    expect(output.clearedOrRetriableSha).toBeUndefined();
+  });
+});
+
+describe('host.commit.check — the step the checks name as failing', () => {
+  /** A check whose report names `guard` as the failing step, and ends as `exit` says. */
+  function reportingGuard(guard: string, exit: string): string[] {
+    return nodeCheck(`${reportScript(JSON.stringify({ failingGuard: guard }))}\n${exit}`);
+  }
+
+  it('names the failing step on checks that failed, from a report that carries no listener count', async () => {
+    const world = await fixture({
+      branchPrefix: 'aflow/',
+      checks: reportingGuard('tsc apps/x/tsconfig.json', 'process.exit(2);'),
+    });
+    const output = HostCommitCheckOutputSchema.parse((await check(world)).captured.output);
+    expect(output.passed).toBe(false);
+    expect(output.failingGuard).toBe('tsc apps/x/tsconfig.json');
+    // The summary is what a fix is briefed with, so it names the step too.
+    expect(output.summary).toContain('the check failed at `tsc apps/x/tsconfig.json`.');
+    // Unknown, not zero: the run failed before the tests that would count them.
+    expect(output.skippedListenerTests).toBeUndefined();
+    expect(output.receipt).toBeDefined();
+  });
+
+  it('names none on checks that passed, whatever the report says, nor where the checks named none', async () => {
+    const passing = await fixture({
+      branchPrefix: 'aflow/',
+      checks: reportingGuard('tsc apps/x/tsconfig.json', 'process.exit(0);'),
+    });
+    const passed = HostCommitCheckOutputSchema.parse((await check(passing)).captured.output);
+    expect(passed.failingGuard).toBeUndefined();
+
+    const silent = await fixture({ branchPrefix: 'aflow/', checks: FAILING });
+    const failed = HostCommitCheckOutputSchema.parse((await check(silent)).captured.output);
+    expect(failed.failingGuard).toBeUndefined();
   });
 });