Skip to content
2 changes: 2 additions & 0 deletions api/server/services/ToolService.js
Original file line number Diff line number Diff line change
Expand Up @@ -2321,6 +2321,8 @@ async function loadToolsForExecution({
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(
codeExecutionContext.codeEnvironmentConfigSchema,
),
minCommandAdmissionMs:
codeExecutionContext.codeEnvironmentConfigSchema?.limits?.minCommandAdmissionMs,
})
: createBashExecutionTool({
authHeaders,
Expand Down
13 changes: 11 additions & 2 deletions api/server/services/__tests__/ToolService.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -2980,7 +2980,14 @@ describe('ToolService - Action Capability Gating', () => {
environmentType: 'attached',
environmentId: 'personal-machine',
bridgeWorkerId: 'worker-abc',
codeEnvironmentConfigSchema: { limits: { maxCommandTimeoutMs: 120000, maxQueueWaitMs: 0 } },
codeEnvironmentConfigSchema: {
limits: {
maxCommandTimeoutMs: 80_000,
maxQueueWaitMs: 0,
maxRequestTimeoutMs: 90_000,
minCommandAdmissionMs: 15_000,
},
},
});
const toolRegistry = new Map([
[AgentConstants.BASH_TOOL, { name: AgentConstants.BASH_TOOL }],
Expand All @@ -3006,9 +3013,11 @@ describe('ToolService - Action Capability Gating', () => {
baseUrl: 'http://attached-code.test/v1',
workspaceId: 'project-a',
gitIdentity: { name: 'LibreChat Agent', email: 'agent@example.com' },
maxTimeoutMs: 120000,
maxTimeoutMs: 65_000,
maxQueueWaitMs: 0,
codeApiMaxRetryWaitMs: 0,
maxRequestTimeoutMs: 90_000,
minCommandAdmissionMs: 15_000,
});
expect(mockResolveCodeExecutionWorkspaceContext).toHaveBeenCalledWith(
expect.objectContaining({ requestedSelections: req.body.codeWorkspaces }),
Expand Down
2 changes: 1 addition & 1 deletion e2e/specs/mock/activity-fold.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ test.describe('activity fold', () => {
const pill = messagesView(page).getByTestId('failed-reveal-pill');
const peek = messagesView(page).getByTestId('activity-phase-failed-peek');
await expect(pill).toBeVisible();
await expect(pill).toHaveAccessibleName('Show failed call');
await expect(pill).toHaveAccessibleName('Show 1 failed call out of 4 calls');
await expect(peek).toBeVisible();
await expect(peek).toContainText('Failed:');
await expect(peek).toContainText('Show error');
Expand Down
4 changes: 2 additions & 2 deletions e2e/specs/mock/steering-escalation.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -206,14 +206,14 @@ test.describe('escalating waiting messages to an interrupt', () => {
// The toggle lives in the row menu's separated Preferences section.
await row.getByRole('button', { name: 'More options' }).click();
await expect(page.getByText('Preferences', { exact: true })).toBeVisible({ timeout: 5000 });
await page.getByRole('menuitem', { name: 'Always interrupt instead', exact: true }).click();
await page.getByRole('menuitem', { name: 'Steer sooner by default', exact: true }).click();
Comment thread
lia-by-librechat[bot] marked this conversation as resolved.

// Verify the preference flips while this row is guaranteed to remain
// parked. After the interrupt is submitted the run may seal and auto-drain
// the row before another locator action can observe it.
await row.getByRole('button', { name: 'More options' }).click();
await expect(
page.getByRole('menuitem', { name: 'Wait for tool steps instead', exact: true }),
page.getByRole('menuitem', { name: 'Wait for the next step instead', exact: true }),
).toBeVisible({ timeout: 5000 });
await page.keyboard.press('Escape');

Expand Down
11 changes: 9 additions & 2 deletions librechat.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -728,9 +728,16 @@ endpoints:
# # ingress, and caller can all outlast this value. The Nginx example
# # (120s) cannot establish the live limit. Omission keeps 30s admission.
# # With 125s total, a read can queue <=90s, a default command <=85s,
# # a 90s command <=25s. Commands >115s cannot fit; the 120s command
# # ceiling above requires a larger VERIFIED total budget to be usable.
# # a 90s command <=25s. With the default 10s admission reserve,
# # commands >105s cannot fit; the 120s command ceiling above requires
# # a VERIFIED total budget of at least 140s to be usable.
# # maxRequestTimeoutMs: 125000
# # Optional command admission allowance before local dispatch overhead.
# # Omission reserves 10s; valid range: 1000–300000 ms. The HTTP budget
# # must exceed this allowance plus 10s settlement/delivery grace;
# # with the 10s default, a budget of 20s or less is rejected.
# # Smaller allowances give a busy worker less time to accept the call.
# # minCommandAdmissionMs: 10000
# permissions:
# fileWrite:
# allowed: [allow, ask, deny]
Expand Down
147 changes: 147 additions & 0 deletions packages/api/src/code/command.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,153 @@ describe('createAttachedWorkspaceBashTool', () => {
expect(foregroundRequest).toMatchObject({ timeoutMs: 30_000 });
});

test('fits the command ceiling inside a configured total HTTP budget', () => {
expect(
resolveAttachedWorkspaceCommandTimeoutMax({
limits: { maxCommandTimeoutMs: 80_000, maxRequestTimeoutMs: 90_000 },
}),
).toBe(70_000);
expect(
resolveAttachedWorkspaceCommandTimeoutMax({
limits: { maxCommandTimeoutMs: 60_000, maxRequestTimeoutMs: 90_000 },
}),
).toBe(60_000);
expect(
resolveAttachedWorkspaceCommandTimeoutMax({
limits: {
maxCommandTimeoutMs: 80_000,
maxRequestTimeoutMs: 90_000,
minCommandAdmissionMs: 15_000,
},
}),
).toBe(65_000);
expect(
resolveAttachedWorkspaceCommandTimeoutMax({
limits: {
maxCommandTimeoutMs: 80_000,
maxRequestTimeoutMs: 90_000,
minCommandAdmissionMs: 1_000,
},
}),
).toBe(79_000);
expect(
resolveAttachedWorkspaceCommandTimeoutMax(
{ limits: { maxCommandTimeoutMs: 120_000, maxRequestTimeoutMs: 125_000 } },
90_000,
),
).toBe(90_000);
expect(
resolveAttachedWorkspaceCommandTimeoutMax({ limits: { maxCommandTimeoutMs: 120_000 } }),
).toBe(120_000);
});

test('starts an omitted background timeout that the configured budget can carry', async () => {
jest.spyOn(Date, 'now').mockReturnValue(1_000);
const configSchema = { limits: { maxCommandTimeoutMs: 80_000, maxRequestTimeoutMs: 90_000 } };
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
const bashTool = createAttachedWorkspaceBashTool({
baseUrl: 'https://code.example.com/v1',
authHeaders: () => ({}),
workspaceId: 'project-a',
maxTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax(configSchema),
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(configSchema),
fetchImpl,
});

await bashTool.invoke(
{ command: 'npm test' },
{ configurable: { [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true } },
);

const [, init] = (fetchImpl as jest.Mock).mock.calls[0];
expect(JSON.parse(String(init?.body))).toMatchObject({ timeoutMs: 70_000 });
expect(init?.headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('10000');
jest.restoreAllMocks();
});

test('reserves the configured admission allowance for an omitted background timeout', async () => {
jest.spyOn(Date, 'now').mockReturnValue(1_000);
const configSchema = {
limits: {
maxCommandTimeoutMs: 80_000,
maxRequestTimeoutMs: 90_000,
minCommandAdmissionMs: 15_000,
},
};
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
const bashTool = createAttachedWorkspaceBashTool({
baseUrl: 'https://code.example.com/v1',
authHeaders: () => ({}),
workspaceId: 'project-a',
maxTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax(configSchema),
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(configSchema),
minCommandAdmissionMs: configSchema.limits.minCommandAdmissionMs,
fetchImpl,
});

expect(bashTool.schema).toMatchObject({
properties: { timeoutMs: expect.objectContaining({ maximum: 65_000 }) },
});
await bashTool.invoke(
{ command: 'npm test' },
{ configurable: { [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true } },
);
const [, init] = (fetchImpl as jest.Mock).mock.calls[0];
expect(JSON.parse(String(init?.body))).toMatchObject({ timeoutMs: 65_000 });
expect(init?.headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('15000');
await expect(
bashTool.func({ command: 'npm test', timeoutMs: 80_000 }, undefined, {}),
).rejects.toThrow('deployment limit of 65000 milliseconds');
expect(fetchImpl).toHaveBeenCalledTimes(1);
jest.restoreAllMocks();
});

test('still dispatches at the smallest supported reserve after credential-signing time', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
const bashTool = createAttachedWorkspaceBashTool({
baseUrl: 'https://code.example.com/v1',
authHeaders: () => {
now.mockReturnValue(1_200);
return {};
},
workspaceId: 'project-a',
maxTimeoutMs: 80_000,
maxRequestTimeoutMs: 90_000,
minCommandAdmissionMs: 1_000,
fetchImpl,
});

await bashTool.invoke(
{ command: 'npm test' },
{ configurable: { [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true } },
);
const [, init] = (fetchImpl as jest.Mock).mock.calls[0];
expect(JSON.parse(String(init?.body))).toMatchObject({ timeoutMs: 79_000 });
expect(init?.headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('800');
jest.restoreAllMocks();
});

test('advertises and enforces only the timeout that fits when given both limits directly', async () => {
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
const bashTool = createAttachedWorkspaceBashTool({
baseUrl: 'https://code.example.com/v1',
authHeaders: () => ({}),
workspaceId: 'project-a',
maxTimeoutMs: 80_000,
maxRequestTimeoutMs: 90_000,
fetchImpl,
});

expect(bashTool.schema).toMatchObject({
properties: { timeoutMs: expect.objectContaining({ maximum: 70_000 }) },
});
await expect(
bashTool.func({ command: 'npm test', timeoutMs: 80_000 }, undefined, {}),
).rejects.toThrow('deployment limit of 70000 milliseconds');
expect(fetchImpl).not.toHaveBeenCalled();
});

test('resolves the administrator-configured retry horizon', () => {
expect(resolveAttachedWorkspaceQueueWaitMs()).toBe(5 * 60_000);
expect(resolveAttachedWorkspaceQueueWaitMs({ limits: { maxQueueWaitMs: 30_000 } })).toBe(
Expand Down
28 changes: 26 additions & 2 deletions packages/api/src/code/command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import type { WorkspaceExecuteCommandResult } from './workspace';
import type { CodeExecutionContext } from '~/agents/execution';
import type { CodeBridgeFetch } from './bridge';
import {
fitWorkspaceCommandTimeoutToBudget,
executeWorkspaceTool,
WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
WORKSPACE_COMMAND_MAX_TIMEOUT_MS,
Expand Down Expand Up @@ -102,7 +103,23 @@ export function resolveAttachedWorkspaceCommandTimeoutMax(
} else if (upstreamMaxTimeoutMs != null) {
requested = upstream;
}
return Math.min(requested, upstream);
return fitCommandTimeoutMaxToBudget(
Math.min(requested, upstream),
resolveAttachedWorkspaceRequestTimeoutMs(configSchema),
configSchema?.limits?.minCommandAdmissionMs,
);
}

function fitCommandTimeoutMaxToBudget(
maxTimeoutMs: number,
maxRequestTimeoutMs?: number,
minCommandAdmissionMs?: number,
): number {
if (maxRequestTimeoutMs == null) return maxTimeoutMs;
return Math.min(
maxTimeoutMs,
fitWorkspaceCommandTimeoutToBudget(maxRequestTimeoutMs, minCommandAdmissionMs),
);
}

/**
Expand Down Expand Up @@ -304,6 +321,7 @@ export function createAttachedWorkspaceBashTool({
maxQueueWaitMs,
codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
minCommandAdmissionMs,
fetchImpl,
}: {
baseUrl: string;
Expand All @@ -319,9 +337,15 @@ export function createAttachedWorkspaceBashTool({
codeApiMaxRetryWaitMs?: number;
/** Verified total HTTP budget; omission keeps the legacy per-attempt timeout. */
maxRequestTimeoutMs?: number;
/** Minimum time for command admission inside an opted-in HTTP budget. */
minCommandAdmissionMs?: number;
fetchImpl?: CodeBridgeFetch;
}): DynamicStructuredTool {
const effectiveMaxTimeoutMs = normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs);
const effectiveMaxTimeoutMs = fitCommandTimeoutMaxToBudget(
normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs),
maxRequestTimeoutMs,
minCommandAdmissionMs,
);
const schema = structuredClone(
buildAttachedWorkspaceBashSchema(effectiveMaxTimeoutMs, environment),
);
Expand Down
19 changes: 19 additions & 0 deletions packages/api/src/code/workspace.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
CODE_ENVIRONMENT_ADMISSION_MAX_MS,
CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS,
CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS,
CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS,
} from 'librechat-data-provider';
Expand Down Expand Up @@ -31,6 +32,24 @@ const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms';
export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS;
const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000;
const WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS = 5_000;

/**
* Longest command timeout a total HTTP budget can carry: the budget minus settlement and delivery
* grace and a minimum admission allowance. A command whose reserve reaches the budget is refused
* before dispatch, so a larger ceiling would only advertise timeouts that can never start.
*/
export function fitWorkspaceCommandTimeoutToBudget(
maxRequestTimeoutMs: number,
minCommandAdmissionMs: number = CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS,
): number {
return Math.max(
1,
maxRequestTimeoutMs -
WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS -
WORKSPACE_COMMAND_TRANSPORT_GRACE_MS -
minCommandAdmissionMs,
Comment thread
lia-by-librechat[bot] marked this conversation as resolved.
);
}
const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
const MAX_ERROR_BODY_BYTES = 4096;
const ERROR_BODY_TIMEOUT_MS = 1000;
Expand Down
Loading
Loading