From 06779465ce25d7776a574bcf730bfad8a1b85bad Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:23:32 +0200 Subject: [PATCH 01/25] feat: Add PR Automation Config and Record Adds the endpoints.agents.prAutomation config block (off by default) and a per-conversation record that holds the automation state, round counter, trust level and per-repository bot allowlist. Round claims are a single conditional update so racing or duplicate deliveries cannot exceed the cap. --- packages/data-provider/src/config.ts | 15 + packages/data-provider/src/index.ts | 1 + .../data-provider/src/prAutomation.spec.ts | 71 +++++ .../data-provider/src/types/prAutomation.ts | 42 +++ packages/data-schemas/src/methods/index.ts | 5 + .../src/methods/prAutomation.spec.ts | 294 ++++++++++++++++++ .../data-schemas/src/methods/prAutomation.ts | 238 ++++++++++++++ packages/data-schemas/src/models/index.ts | 3 + .../data-schemas/src/models/prAutomation.ts | 14 + .../data-schemas/src/schema/prAutomation.ts | 70 +++++ packages/data-schemas/src/types/index.ts | 1 + .../data-schemas/src/types/prAutomation.ts | 70 +++++ 12 files changed, 824 insertions(+) create mode 100644 packages/data-provider/src/prAutomation.spec.ts create mode 100644 packages/data-provider/src/types/prAutomation.ts create mode 100644 packages/data-schemas/src/methods/prAutomation.spec.ts create mode 100644 packages/data-schemas/src/methods/prAutomation.ts create mode 100644 packages/data-schemas/src/models/prAutomation.ts create mode 100644 packages/data-schemas/src/schema/prAutomation.ts create mode 100644 packages/data-schemas/src/types/prAutomation.ts diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 9b8f8fcb8e3..2f0a45540c1 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -69,6 +69,7 @@ import { } from './types/pullRequest'; import type { PullRequestProvider } from './types/pullRequest'; import { isActionTool } from './types/tools'; +import { PR_AUTOMATION_TRUST_LEVELS } from './types/prAutomation'; import { apiBaseUrl } from './api-endpoints'; import { FileSources } from './types/files'; import { MCPServersSchema } from './mcp'; @@ -2014,6 +2015,20 @@ export const agentsEndpointSchema = baseEndpointSchema sharedCacheSlots: z.number().int().min(64).max(1_000_000).optional().default(65_536), }) .optional(), + /** Auto-fix CI and address review comments on a conversation's pull request. Off unless an + * administrator opts in; a user can only narrow `maxTrust`, never widen it. */ + prAutomation: z + .object({ + enabled: z.boolean().optional().default(false), + /** Fix rounds one pull request may run before the automation stops. */ + maxRounds: z.number().int().min(1).max(20).optional().default(5), + /** Wall-clock minutes from the first automated round before the automation stops. */ + maxMinutes: z.number().int().min(5).max(1440).optional().default(120), + /** Widest author trust a user may select: `approvedBots` acts on the linked account and + * allowlisted bots only, `collaborators` adds write-access users, `anyone` acts on all. */ + maxTrust: z.enum(PR_AUTOMATION_TRUST_LEVELS).optional().default('approvedBots'), + }) + .optional(), /** Conversational background-task delivery policy. Automatic completion wakeups are * enabled unless an administrator explicitly restores poll-only behavior. */ backgroundTasks: z diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index 1c9dc09f08e..af2d9de1bd9 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -66,6 +66,7 @@ export type { export * from './types/subagents'; export * from './types/background'; export * from './types/pullRequest'; +export * from './types/prAutomation'; export * from './types/queuedTurns'; /* access permissions */ export * from './accessPermissions'; diff --git a/packages/data-provider/src/prAutomation.spec.ts b/packages/data-provider/src/prAutomation.spec.ts new file mode 100644 index 00000000000..69fcbf39804 --- /dev/null +++ b/packages/data-provider/src/prAutomation.spec.ts @@ -0,0 +1,71 @@ +import { configSchema } from './config'; +import { clampPRAutomationTrust } from './types/prAutomation'; +import type { PRAutomationTrustLevel } from './types/prAutomation'; + +describe('clampPRAutomationTrust', () => { + it('defaults to the narrowest level when the user chose nothing', () => { + expect(clampPRAutomationTrust(undefined, 'anyone')).toBe('approvedBots'); + }); + + it.each<[PRAutomationTrustLevel, PRAutomationTrustLevel, PRAutomationTrustLevel]>([ + ['anyone', 'approvedBots', 'approvedBots'], + ['collaborators', 'approvedBots', 'approvedBots'], + ['anyone', 'collaborators', 'collaborators'], + ['approvedBots', 'anyone', 'approvedBots'], + ['collaborators', 'anyone', 'collaborators'], + ['anyone', 'anyone', 'anyone'], + ])('requested %s under a %s ceiling gives %s', (requested, ceiling, expected) => { + expect(clampPRAutomationTrust(requested, ceiling)).toBe(expected); + }); + + it('falls back to the narrowest level for a value it does not know', () => { + expect(clampPRAutomationTrust('everyone' as unknown as PRAutomationTrustLevel, 'anyone')).toBe( + 'approvedBots', + ); + }); +}); + +describe('agent PR automation config', () => { + const parse = (prAutomation?: unknown) => + configSchema.safeParse({ version: '1.0', endpoints: { agents: { prAutomation } } }); + + it('is absent when not configured, which leaves it off', () => { + const absent = configSchema.parse({ version: '1.0', endpoints: { agents: {} } }); + expect(absent.endpoints?.agents?.prAutomation).toBeUndefined(); + }); + + it('defaults to disabled with the narrowest ceiling', () => { + const empty = configSchema.parse({ + version: '1.0', + endpoints: { agents: { prAutomation: {} } }, + }); + expect(empty.endpoints?.agents?.prAutomation).toEqual({ + enabled: false, + maxRounds: 5, + maxMinutes: 120, + maxTrust: 'approvedBots', + }); + }); + + it('accepts an administrator raising the limits and the ceiling', () => { + const result = parse({ + enabled: true, + maxRounds: 10, + maxMinutes: 240, + maxTrust: 'collaborators', + }); + expect(result.success).toBe(true); + }); + + it.each([0, 21, 1.5])('rejects %s fix rounds', (maxRounds) => { + expect(parse({ maxRounds }).success).toBe(false); + }); + + it.each([4, 1441, 1.5])('rejects a %s minute window', (maxMinutes) => { + expect(parse({ maxMinutes }).success).toBe(false); + }); + + it.each(['everyone', 'bots', ''])('rejects %s as a trust ceiling', (maxTrust) => { + expect(parse({ maxTrust }).success).toBe(false); + }); +}); diff --git a/packages/data-provider/src/types/prAutomation.ts b/packages/data-provider/src/types/prAutomation.ts new file mode 100644 index 00000000000..7495bac18a2 --- /dev/null +++ b/packages/data-provider/src/types/prAutomation.ts @@ -0,0 +1,42 @@ +export const PR_AUTOMATION_STATES = ['idle', 'waiting', 'fixing', 'needs_user', 'stopped'] as const; + +/** + * Stable, machine-readable reasons an automation stopped. The client maps each + * code to localized copy; none of them carries provider text or a comment body. + */ +export const PR_AUTOMATION_STOP_CODES = [ + 'user_stopped', + 'round_cap', + 'time_cap', + 'fork_pull_request', + 'untrusted_head_commit', + 'approval_expired', + 'pull_request_closed', + 'disabled_by_admin', +] as const; + +/** Ordered from narrowest to widest: who may be acted on. */ +export const PR_AUTOMATION_TRUST_LEVELS = ['approvedBots', 'collaborators', 'anyone'] as const; + +export type PRAutomationState = (typeof PR_AUTOMATION_STATES)[number]; +export type PRAutomationStopCode = (typeof PR_AUTOMATION_STOP_CODES)[number]; +export type PRAutomationTrustLevel = (typeof PR_AUTOMATION_TRUST_LEVELS)[number]; + +/** + * An administrator's ceiling bounds what a user may choose. A user can only + * narrow it, so the result is whichever level is narrower. + */ +export function clampPRAutomationTrust( + requested: PRAutomationTrustLevel | undefined, + ceiling: PRAutomationTrustLevel, +): PRAutomationTrustLevel { + if (requested == null) { + return PR_AUTOMATION_TRUST_LEVELS[0]; + } + const requestedRank = PR_AUTOMATION_TRUST_LEVELS.indexOf(requested); + const ceilingRank = PR_AUTOMATION_TRUST_LEVELS.indexOf(ceiling); + if (requestedRank < 0) { + return PR_AUTOMATION_TRUST_LEVELS[0]; + } + return PR_AUTOMATION_TRUST_LEVELS[Math.min(requestedRank, ceilingRank)]; +} diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index 8be120e86dc..b30a5fd9d0d 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -43,6 +43,8 @@ import { MAX_TOOL_FAVORITES, type ToolFavoriteMethods, } from './favorite'; +/* PR Automation */ +import { createPRAutomationMethods, type PRAutomationMethods } from './prAutomation'; /* Agent Categories */ import { createAgentCategoryMethods, type AgentCategoryMethods } from './agentCategory'; /* Agent API Keys */ @@ -264,6 +266,7 @@ export type AllMethods = ToolApprovalGrantStorage & FileMethods & MemoryMethods & ToolFavoriteMethods & + PRAutomationMethods & AgentCategoryMethods & AgentApiKeyMethods & MCPServerMethods & @@ -504,6 +507,7 @@ export function createMethods( ...createFileMethods(mongoose), ...createMemoryMethods(mongoose), ...createToolFavoriteMethods(mongoose), + ...createPRAutomationMethods(mongoose), ...createAgentCategoryMethods(mongoose), ...createAgentApiKeyMethods(mongoose), ...createMCPServerMethods(mongoose, { @@ -578,6 +582,7 @@ export type { ProjectFileRecord, ProjectFilesOptions, ToolFavoriteMethods, + PRAutomationMethods, AgentCategoryMethods, AgentApiKeyMethods, MCPServerMethods, diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts new file mode 100644 index 00000000000..7b7becb50c9 --- /dev/null +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -0,0 +1,294 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { logger, createModels } from '..'; +import { MAX_PR_AUTOMATION_BOTS } from '../types/prAutomation'; +import { createPRAutomationMethods } from './prAutomation'; + +logger.silent = true; + +let PRAutomation: mongoose.Model; +let methods: ReturnType; +let mongoServer: MongoMemoryServer; + +const userId = new mongoose.Types.ObjectId().toString(); +const otherUserId = new mongoose.Types.ObjectId().toString(); +const key = { userId, conversationId: 'convo-1' }; +const limits = { maxRounds: 3, maxMinutes: 60 }; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + createModels(mongoose); + PRAutomation = mongoose.models.PRAutomation; + await PRAutomation.syncIndexes(); + methods = createPRAutomationMethods(mongoose); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(async () => { + await PRAutomation.deleteMany({}); +}); + +describe('getPRAutomation', () => { + test('returns null for a conversation that never enabled it', async () => { + expect(await methods.getPRAutomation(key)).toBeNull(); + }); +}); + +describe('enablePRAutomation', () => { + test('creates an idle record on the narrowest trust level', async () => { + const record = await methods.enablePRAutomation(key); + expect(record).toMatchObject({ + conversationId: 'convo-1', + state: 'idle', + round: 0, + trust: 'approvedBots', + trustedBots: [], + }); + }); + + test('is idempotent and keeps one record per user and conversation', async () => { + await methods.enablePRAutomation(key); + await methods.enablePRAutomation(key); + expect(await PRAutomation.countDocuments({ user: userId })).toBe(1); + }); + + test('does not change an active record that is enabled again', async () => { + await methods.enablePRAutomation(key); + await methods.claimPRAutomationRound({ ...key, ...limits, headSha: 'a'.repeat(40) }); + const again = await methods.enablePRAutomation(key); + expect(again).toMatchObject({ state: 'fixing', round: 1 }); + }); + + test('restarts a stopped record with a fresh round counter and window', async () => { + await methods.enablePRAutomation(key); + await methods.claimPRAutomationRound({ ...key, ...limits, headSha: 'a'.repeat(40) }); + await methods.setPRAutomationState(key, { state: 'stopped', stopCode: 'user_stopped' }); + + const restarted = await methods.enablePRAutomation(key); + expect(restarted).toMatchObject({ state: 'idle', round: 0 }); + expect(restarted.stopCode).toBeUndefined(); + expect(restarted.startedAt).toBeUndefined(); + expect(restarted.lastHeadSha).toBeUndefined(); + }); + + test('keeps records of different users apart', async () => { + await methods.enablePRAutomation(key); + expect(await methods.getPRAutomation({ ...key, userId: otherUserId })).toBeNull(); + }); +}); + +describe('claimPRAutomationRound', () => { + const head = (n: number) => String(n).repeat(40).slice(0, 40); + + test('starts a round, counts it and records the head', async () => { + await methods.enablePRAutomation(key); + const result = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); + expect(result).toMatchObject({ + ok: true, + value: { state: 'fixing', round: 1, lastHeadSha: head(1) }, + }); + }); + + test('rejects a second delivery for a head that was already claimed', async () => { + await methods.enablePRAutomation(key); + await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); + await methods.setPRAutomationState(key, { state: 'waiting' }); + const duplicate = await methods.claimPRAutomationRound({ + ...key, + ...limits, + headSha: head(1), + }); + expect(duplicate).toEqual({ ok: false, error: { code: 'stale_head' } }); + expect((await methods.getPRAutomation(key))?.round).toBe(1); + }); + + test('rejects a claim while a round is already running', async () => { + await methods.enablePRAutomation(key); + await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); + const overlapping = await methods.claimPRAutomationRound({ + ...key, + ...limits, + headSha: head(2), + }); + expect(overlapping).toEqual({ ok: false, error: { code: 'not_active' } }); + }); + + test('stops at the round cap and never goes past it', async () => { + await methods.enablePRAutomation(key); + for (let round = 1; round <= limits.maxRounds; round++) { + const claimed = await methods.claimPRAutomationRound({ + ...key, + ...limits, + headSha: head(round), + }); + expect(claimed.ok).toBe(true); + await methods.setPRAutomationState(key, { state: 'waiting' }); + } + const over = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(9) }); + expect(over).toEqual({ ok: false, error: { code: 'round_cap' } }); + expect((await methods.getPRAutomation(key))?.round).toBe(limits.maxRounds); + }); + + test('lets exactly one of several concurrent deliveries claim the same head', async () => { + await methods.enablePRAutomation(key); + const results = await Promise.all( + Array.from({ length: 8 }, () => + methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }), + ), + ); + expect(results.filter((result) => result.ok)).toHaveLength(1); + expect((await methods.getPRAutomation(key))?.round).toBe(1); + }); + + test('never exceeds the cap under concurrent deliveries for different heads', async () => { + await methods.enablePRAutomation(key); + const results = await Promise.all( + Array.from({ length: 8 }, (_, index) => + methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(index + 1) }), + ), + ); + expect(results.filter((result) => result.ok)).toHaveLength(1); + expect((await methods.getPRAutomation(key))?.round).toBeLessThanOrEqual(limits.maxRounds); + }); + + test('stops once the wall-clock window has passed', async () => { + await methods.enablePRAutomation(key); + const start = new Date('2026-01-01T00:00:00Z'); + await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1), now: start }); + await methods.setPRAutomationState(key, { state: 'waiting' }); + + const later = new Date(start.getTime() + (limits.maxMinutes + 1) * 60_000); + const expired = await methods.claimPRAutomationRound({ + ...key, + ...limits, + headSha: head(2), + now: later, + }); + expect(expired).toEqual({ ok: false, error: { code: 'time_cap' } }); + }); + + test('reports not_found for a conversation with no record', async () => { + const result = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); + expect(result).toEqual({ ok: false, error: { code: 'not_found' } }); + }); + + test('rejects a claim on a stopped record', async () => { + await methods.enablePRAutomation(key); + await methods.setPRAutomationState(key, { state: 'stopped', stopCode: 'user_stopped' }); + const result = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); + expect(result).toEqual({ ok: false, error: { code: 'not_active' } }); + }); +}); + +describe('setPRAutomationState', () => { + test('records a stop code only with the stopped state', async () => { + await methods.enablePRAutomation(key); + const stopped = await methods.setPRAutomationState(key, { + state: 'stopped', + stopCode: 'round_cap', + }); + expect(stopped).toMatchObject({ state: 'stopped', stopCode: 'round_cap' }); + }); + + test('does not revive a stopped record', async () => { + await methods.enablePRAutomation(key); + await methods.setPRAutomationState(key, { state: 'stopped', stopCode: 'user_stopped' }); + const revived = await methods.setPRAutomationState(key, { state: 'waiting' }); + expect(revived).toBeNull(); + expect((await methods.getPRAutomation(key))?.state).toBe('stopped'); + }); + + test('clears a stale stop code when the state moves on', async () => { + await methods.enablePRAutomation(key); + await methods.setPRAutomationState(key, { state: 'needs_user' }); + const next = await methods.setPRAutomationState(key, { state: 'waiting' }); + expect(next?.stopCode).toBeUndefined(); + }); + + test('returns null for a conversation with no record', async () => { + expect(await methods.setPRAutomationState(key, { state: 'waiting' })).toBeNull(); + }); +}); + +describe('setPRAutomationTrust', () => { + test('changes the trust level', async () => { + await methods.enablePRAutomation(key); + const updated = await methods.setPRAutomationTrust(key, 'collaborators'); + expect(updated?.trust).toBe('collaborators'); + }); + + test('rejects a level outside the enum', async () => { + await methods.enablePRAutomation(key); + await expect( + methods.setPRAutomationTrust(key, 'everyone' as unknown as 'anyone'), + ).rejects.toThrow(); + }); +}); + +describe('approved bots', () => { + test('adds a bot by numeric id', async () => { + await methods.enablePRAutomation(key); + const result = await methods.addPRAutomationBot(key, { id: 101, login: 'review-bot[bot]' }); + expect(result).toMatchObject({ + ok: true, + value: { trustedBots: [{ id: 101, login: 'review-bot[bot]' }] }, + }); + }); + + test('is idempotent by id even when the login was renamed', async () => { + await methods.enablePRAutomation(key); + await methods.addPRAutomationBot(key, { id: 101, login: 'old-name[bot]' }); + const again = await methods.addPRAutomationBot(key, { id: 101, login: 'new-name[bot]' }); + expect(again.ok).toBe(true); + const record = await methods.getPRAutomation(key); + expect(record?.trustedBots).toEqual([{ id: 101, login: 'old-name[bot]' }]); + }); + + test('refuses a bot beyond the cap with a stable code', async () => { + await methods.enablePRAutomation(key); + for (let id = 1; id <= MAX_PR_AUTOMATION_BOTS; id++) { + const added = await methods.addPRAutomationBot(key, { id }); + expect({ id, ok: added.ok }).toEqual({ id, ok: true }); + } + const over = await methods.addPRAutomationBot(key, { id: 9999 }); + expect(over).toEqual({ ok: false, error: { code: 'bot_limit' } }); + }); + + test('reports not_found when the conversation has no record', async () => { + const result = await methods.addPRAutomationBot(key, { id: 101 }); + expect(result).toEqual({ ok: false, error: { code: 'not_found' } }); + }); + + test('removes a bot by id and leaves the others', async () => { + await methods.enablePRAutomation(key); + await methods.addPRAutomationBot(key, { id: 1 }); + await methods.addPRAutomationBot(key, { id: 2 }); + const record = await methods.removePRAutomationBot(key, 1); + expect(record?.trustedBots).toEqual([{ id: 2 }]); + }); + + test('keeps each conversation allowlist separate', async () => { + await methods.enablePRAutomation(key); + await methods.enablePRAutomation({ userId, conversationId: 'convo-2' }); + await methods.addPRAutomationBot(key, { id: 101 }); + const other = await methods.getPRAutomation({ userId, conversationId: 'convo-2' }); + expect(other?.trustedBots).toEqual([]); + }); +}); + +describe('disablePRAutomation', () => { + test('removes the record and reports it', async () => { + await methods.enablePRAutomation(key); + expect(await methods.disablePRAutomation(key)).toEqual({ removed: true }); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('reports nothing removed when there was no record', async () => { + expect(await methods.disablePRAutomation(key)).toEqual({ removed: false }); + }); +}); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts new file mode 100644 index 00000000000..8445ad9e393 --- /dev/null +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -0,0 +1,238 @@ +import type { PRAutomationState } from 'librechat-data-provider'; +import type { PRAutomationStopCode } from 'librechat-data-provider'; +import type { PRAutomationTrustLevel } from 'librechat-data-provider'; +import type * as t from '~/types/prAutomation'; +import { MAX_PR_AUTOMATION_BOTS } from '~/types/prAutomation'; + +const PROJECTION = '-_id -__v'; +const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; + +export function createPRAutomationMethods(mongoose: typeof import('mongoose')): { + getPRAutomation: (key: t.PRAutomationKey) => Promise; + enablePRAutomation: (params: t.EnablePRAutomationParams) => Promise; + disablePRAutomation: (key: t.PRAutomationKey) => Promise<{ removed: boolean }>; + claimPRAutomationRound: ( + params: t.ClaimPRAutomationRoundParams, + ) => Promise; + setPRAutomationState: ( + key: t.PRAutomationKey, + next: { state: PRAutomationState; stopCode?: PRAutomationStopCode }, + ) => Promise; + setPRAutomationTrust: ( + key: t.PRAutomationKey, + trust: PRAutomationTrustLevel, + ) => Promise; + addPRAutomationBot: ( + key: t.PRAutomationKey, + bot: t.IPRAutomationBot, + ) => Promise; + removePRAutomationBot: (key: t.PRAutomationKey, botId: number) => Promise; +} { + const keyFilter = ({ userId, conversationId }: t.PRAutomationKey) => ({ + user: userId, + conversationId, + }); + + /** Absence is a normal answer here; a query failure throws. */ + async function getPRAutomation(key: t.PRAutomationKey): Promise { + const PRAutomation = mongoose.models.PRAutomation; + return PRAutomation.findOne(keyFilter(key)).select(PROJECTION).lean(); + } + + /** + * Idempotent. An existing active record is returned unchanged. A stopped + * record is the user explicitly turning the automation back on, so it starts + * a fresh run: the round counter and the time window reset, which they could + * not do from a webhook-triggered turn. + */ + async function enablePRAutomation({ + trust, + repository, + pullNumber, + ...key + }: t.EnablePRAutomationParams): Promise { + const PRAutomation = mongoose.models.PRAutomation; + const filter = keyFilter(key); + await PRAutomation.updateOne( + filter, + { + $setOnInsert: { + ...filter, + state: 'idle', + round: 0, + trustedBots: [], + trust: trust ?? 'approvedBots', + }, + }, + { upsert: true, runValidators: true }, + ); + await PRAutomation.updateOne( + { ...filter, state: 'stopped' }, + { + $set: { state: 'idle', round: 0, ...(trust != null && { trust }) }, + $unset: { stopCode: '', startedAt: '', lastHeadSha: '' }, + }, + ); + if (repository != null || pullNumber != null) { + await PRAutomation.updateOne(filter, { + $set: { + ...(repository != null && { repository }), + ...(pullNumber != null && { pullNumber }), + }, + }); + } + const record = await getPRAutomation(key); + if (record == null) { + throw new Error('PR automation record missing after enable'); + } + return record; + } + + async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { + const PRAutomation = mongoose.models.PRAutomation; + const result = await PRAutomation.deleteOne(keyFilter(key)); + return { removed: result.deletedCount > 0 }; + } + + /** + * Starts one fix round, atomically. The round count, the time window, the + * state and the head are all conditions of a single `findOneAndUpdate`, so two + * deliveries racing for the same record cannot both pass the cap, and a + * duplicate delivery for a head that was already claimed is rejected instead + * of starting a second round. The persisted counter is what stops a new turn + * from resetting the cap. + */ + async function claimPRAutomationRound({ + maxRounds, + maxMinutes, + headSha, + now = new Date(), + ...key + }: t.ClaimPRAutomationRoundParams): Promise { + const PRAutomation = mongoose.models.PRAutomation; + const filter = keyFilter(key); + const cutoff = new Date(now.getTime() - maxMinutes * 60_000); + + /** The window opens at the first claim attempt on an active record, once. */ + await PRAutomation.updateOne( + { ...filter, state: { $in: CLAIMABLE_STATES }, startedAt: { $exists: false } }, + { $set: { startedAt: now } }, + ); + + const claimed = await PRAutomation.findOneAndUpdate( + { + ...filter, + state: { $in: CLAIMABLE_STATES }, + round: { $lt: maxRounds }, + startedAt: { $gte: cutoff }, + lastHeadSha: { $ne: headSha }, + }, + { + $inc: { round: 1 }, + $set: { state: 'fixing', lastHeadSha: headSha }, + $unset: { stopCode: '' }, + }, + { new: true, select: PROJECTION }, + ).lean(); + if (claimed != null) { + return { ok: true, value: claimed }; + } + + const current = await getPRAutomation(key); + if (current == null) { + return { ok: false, error: { code: 'not_found' } }; + } + if (!CLAIMABLE_STATES.includes(current.state)) { + return { ok: false, error: { code: 'not_active' } }; + } + if (current.round >= maxRounds) { + return { ok: false, error: { code: 'round_cap' } }; + } + if (current.startedAt != null && current.startedAt < cutoff) { + return { ok: false, error: { code: 'time_cap' } }; + } + return { ok: false, error: { code: 'stale_head' } }; + } + + /** A stopped record only leaves that state through `enablePRAutomation`. */ + async function setPRAutomationState( + key: t.PRAutomationKey, + next: { state: PRAutomationState; stopCode?: PRAutomationStopCode }, + ): Promise { + const PRAutomation = mongoose.models.PRAutomation; + const stopCode = next.state === 'stopped' ? next.stopCode : undefined; + return PRAutomation.findOneAndUpdate( + { ...keyFilter(key), state: { $ne: 'stopped' } }, + stopCode != null + ? { $set: { state: next.state, stopCode } } + : { $set: { state: next.state }, $unset: { stopCode: '' } }, + { new: true, select: PROJECTION, runValidators: true }, + ).lean(); + } + + async function setPRAutomationTrust( + key: t.PRAutomationKey, + trust: PRAutomationTrustLevel, + ): Promise { + const PRAutomation = mongoose.models.PRAutomation; + return PRAutomation.findOneAndUpdate( + keyFilter(key), + { $set: { trust } }, + { new: true, select: PROJECTION, runValidators: true }, + ).lean(); + } + + /** Idempotent by numeric id, and capped. A login is stored for display only. */ + async function addPRAutomationBot( + key: t.PRAutomationKey, + bot: t.IPRAutomationBot, + ): Promise { + const PRAutomation = mongoose.models.PRAutomation; + const updated = await PRAutomation.findOneAndUpdate( + { + ...keyFilter(key), + 'trustedBots.id': { $ne: bot.id }, + [`trustedBots.${MAX_PR_AUTOMATION_BOTS - 1}`]: { $exists: false }, + }, + { $push: { trustedBots: bot } }, + { new: true, select: PROJECTION, runValidators: true }, + ).lean(); + if (updated != null) { + return { ok: true, value: updated }; + } + + const current = await getPRAutomation(key); + if (current == null) { + return { ok: false, error: { code: 'not_found' } }; + } + if (current.trustedBots.some((existing) => existing.id === bot.id)) { + return { ok: true, value: current }; + } + return { ok: false, error: { code: 'bot_limit' } }; + } + + async function removePRAutomationBot( + key: t.PRAutomationKey, + botId: number, + ): Promise { + const PRAutomation = mongoose.models.PRAutomation; + return PRAutomation.findOneAndUpdate( + keyFilter(key), + { $pull: { trustedBots: { id: botId } } }, + { new: true, select: PROJECTION }, + ).lean(); + } + + return { + getPRAutomation, + enablePRAutomation, + disablePRAutomation, + claimPRAutomationRound, + setPRAutomationState, + setPRAutomationTrust, + addPRAutomationBot, + removePRAutomationBot, + }; +} + +export type PRAutomationMethods = ReturnType; diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts index 1d999c870ff..d27cc1422ea 100644 --- a/packages/data-schemas/src/models/index.ts +++ b/packages/data-schemas/src/models/index.ts @@ -22,6 +22,7 @@ import { createPluginAuthModel } from './pluginAuth'; import { createSharedLinkModel } from './sharedLink'; import { createAccessRoleModel } from './accessRole'; import { createToolFavoriteModel } from './favorite'; +import { createPRAutomationModel } from './prAutomation'; import { createMCPServerModel } from './mcpServer'; import { createAssistantModel } from './assistant'; import { createSkillFileModel } from './skillFile'; @@ -87,6 +88,7 @@ export function createModels(mongoose: typeof import('mongoose')): { ToolCall: ReturnType; MemoryEntry: ReturnType; ToolFavorite: ReturnType; + PRAutomation: ReturnType; AccessRole: ReturnType; AclEntry: ReturnType; SystemGrant: ReturnType; @@ -139,6 +141,7 @@ export function createModels(mongoose: typeof import('mongoose')): { ToolCall: createToolCallModel(mongoose), MemoryEntry: createMemoryModel(mongoose), ToolFavorite: createToolFavoriteModel(mongoose), + PRAutomation: createPRAutomationModel(mongoose), AccessRole: createAccessRoleModel(mongoose), AclEntry: createAclEntryModel(mongoose), SystemGrant: createSystemGrantModel(mongoose), diff --git a/packages/data-schemas/src/models/prAutomation.ts b/packages/data-schemas/src/models/prAutomation.ts new file mode 100644 index 00000000000..3c13fcdc8c8 --- /dev/null +++ b/packages/data-schemas/src/models/prAutomation.ts @@ -0,0 +1,14 @@ +import { Model } from 'mongoose'; +import type { IPRAutomationDocument } from '~/types/prAutomation'; +import { applyTenantIsolation } from '~/models/plugins/tenantIsolation'; +import prAutomationSchema from '~/schema/prAutomation'; + +export function createPRAutomationModel( + mongoose: typeof import('mongoose'), +): Model { + applyTenantIsolation(prAutomationSchema); + return ( + mongoose.models.PRAutomation || + mongoose.model('PRAutomation', prAutomationSchema) + ); +} diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts new file mode 100644 index 00000000000..584e8cb047e --- /dev/null +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -0,0 +1,70 @@ +import { Schema } from 'mongoose'; +import { PR_AUTOMATION_STATES } from 'librechat-data-provider'; +import { PR_AUTOMATION_STOP_CODES } from 'librechat-data-provider'; +import { PR_AUTOMATION_TRUST_LEVELS } from 'librechat-data-provider'; +import type { IPRAutomationDocument } from '~/types/prAutomation'; +import { MAX_PR_AUTOMATION_BOTS } from '~/types/prAutomation'; + +const trustedBotSchema = new Schema( + { + id: { type: Number, required: true, min: 1 }, + login: { type: String, maxlength: 128 }, + }, + { _id: false }, +); + +const prAutomationSchema: Schema = new Schema( + { + user: { + type: Schema.Types.ObjectId, + ref: 'User', + required: true, + index: true, + }, + tenantId: { + type: String, + index: true, + }, + conversationId: { + type: String, + required: true, + maxlength: 256, + }, + repository: { type: String, maxlength: 256 }, + pullNumber: { type: Number, min: 1 }, + state: { + type: String, + enum: PR_AUTOMATION_STATES, + default: 'idle', + required: true, + }, + stopCode: { type: String, enum: PR_AUTOMATION_STOP_CODES }, + trust: { + type: String, + enum: PR_AUTOMATION_TRUST_LEVELS, + default: 'approvedBots', + required: true, + }, + trustedBots: { + type: [trustedBotSchema], + default: [], + validate: { + validator: (bots: unknown[]) => bots.length <= MAX_PR_AUTOMATION_BOTS, + message: `At most ${MAX_PR_AUTOMATION_BOTS} approved bots`, + }, + }, + round: { type: Number, default: 0, min: 0, required: true }, + startedAt: { type: Date }, + lastHeadSha: { type: String, maxlength: 64 }, + }, + { timestamps: true }, +); + +/** + * One automation per user and conversation. The unique index makes enabling + * idempotent and race-free. It excludes `tenantId` for the same reason as + * `ToolFavorite`: user ObjectIds are globally unique. + */ +prAutomationSchema.index({ user: 1, conversationId: 1 }, { unique: true }); + +export default prAutomationSchema; diff --git a/packages/data-schemas/src/types/index.ts b/packages/data-schemas/src/types/index.ts index eba6570762e..a918aa6f5a7 100644 --- a/packages/data-schemas/src/types/index.ts +++ b/packages/data-schemas/src/types/index.ts @@ -31,6 +31,7 @@ export * from './pluginAuth'; /* Memories */ export * from './memory'; export * from './favorite'; +export * from './prAutomation'; /* Prompts */ export * from './prompts'; /* Skills */ diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts new file mode 100644 index 00000000000..bf1d5d0a4c7 --- /dev/null +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -0,0 +1,70 @@ +import type { PRAutomationState } from 'librechat-data-provider'; +import type { PRAutomationStopCode } from 'librechat-data-provider'; +import type { PRAutomationTrustLevel } from 'librechat-data-provider'; +import type { Document, Types } from 'mongoose'; + +export const MAX_PR_AUTOMATION_BOTS = 20; + +/** A bot the user approved for one repository, matched by its numeric account id. */ +export interface IPRAutomationBot { + id: number; + /** Display only. Logins can be renamed, so identity checks never read it. */ + login?: string; +} + +export interface IPRAutomation { + user: Types.ObjectId; + tenantId?: string; + conversationId: string; + /** `owner/name` of the repository the pull request belongs to. */ + repository?: string; + pullNumber?: number; + state: PRAutomationState; + stopCode?: PRAutomationStopCode; + trust: PRAutomationTrustLevel; + /** Approved bots for this repository only. */ + trustedBots: IPRAutomationBot[]; + /** Fix rounds started so far. Persisted so a new turn cannot reset it. */ + round: number; + /** First automated round, the start of the wall-clock window. */ + startedAt?: Date; + lastHeadSha?: string; + createdAt?: Date; + updatedAt?: Date; +} + +export interface IPRAutomationDocument extends IPRAutomation, Document {} + +export interface PRAutomationKey { + userId: string | Types.ObjectId; + conversationId: string; +} + +export interface EnablePRAutomationParams extends PRAutomationKey { + trust?: PRAutomationTrustLevel; + repository?: string; + pullNumber?: number; +} + +export interface ClaimPRAutomationRoundParams extends PRAutomationKey { + maxRounds: number; + maxMinutes: number; + /** The head the round will work on; recorded so a stale event can be recognized. */ + headSha: string; + now?: Date; +} + +export type PRAutomationClaimErrorCode = + | 'not_found' + | 'not_active' + | 'round_cap' + | 'time_cap' + | 'stale_head'; + +export type ClaimPRAutomationRoundResult = + | { ok: true; value: IPRAutomation } + | { ok: false; error: { code: PRAutomationClaimErrorCode } }; + +export type PRAutomationBotResult = + | { ok: true; value: IPRAutomation } + | { ok: false; error: { code: 'not_found' | 'bot_limit' } }; From e732be97af5378ee6291d6df143aef4cffdd755d Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:47:25 +0200 Subject: [PATCH 02/25] fix: Address Review Findings on PR Automation Record Round completions now settle only the round that owns them, a user stop is unconditional and requires a stop code, claimed heads are remembered so a delayed delivery of an earlier head cannot start a round, approved bots are cleared when the conversation is bound to a different repository, the approved bot limit is configurable, and the record is removed with its conversation and with its user. Import order is fixed for the static check. --- packages/data-provider/src/config.ts | 14 +- .../data-provider/src/prAutomation.spec.ts | 13 +- .../data-provider/src/types/prAutomation.ts | 4 + .../src/methods/conversation.spec.ts | 40 +++ .../data-schemas/src/methods/conversation.ts | 10 + .../src/methods/prAutomation.spec.ts | 270 ++++++++++++------ .../data-schemas/src/methods/prAutomation.ts | 118 ++++++-- .../src/methods/user.methods.spec.ts | 18 ++ packages/data-schemas/src/methods/user.ts | 1 + packages/data-schemas/src/models/index.ts | 2 +- .../data-schemas/src/schema/prAutomation.ts | 6 +- .../data-schemas/src/types/prAutomation.ts | 25 +- 12 files changed, 389 insertions(+), 132 deletions(-) diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 2f0a45540c1..2d1bf6697eb 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -69,7 +69,11 @@ import { } from './types/pullRequest'; import type { PullRequestProvider } from './types/pullRequest'; import { isActionTool } from './types/tools'; -import { PR_AUTOMATION_TRUST_LEVELS } from './types/prAutomation'; +import { + PR_AUTOMATION_TRUST_LEVELS, + DEFAULT_PR_AUTOMATION_BOTS, + MAX_PR_AUTOMATION_BOTS, +} from './types/prAutomation'; import { apiBaseUrl } from './api-endpoints'; import { FileSources } from './types/files'; import { MCPServersSchema } from './mcp'; @@ -2024,6 +2028,14 @@ export const agentsEndpointSchema = baseEndpointSchema maxRounds: z.number().int().min(1).max(20).optional().default(5), /** Wall-clock minutes from the first automated round before the automation stops. */ maxMinutes: z.number().int().min(5).max(1440).optional().default(120), + /** Bots one user may approve for one repository. */ + maxBots: z + .number() + .int() + .min(1) + .max(MAX_PR_AUTOMATION_BOTS) + .optional() + .default(DEFAULT_PR_AUTOMATION_BOTS), /** Widest author trust a user may select: `approvedBots` acts on the linked account and * allowlisted bots only, `collaborators` adds write-access users, `anyone` acts on all. */ maxTrust: z.enum(PR_AUTOMATION_TRUST_LEVELS).optional().default('approvedBots'), diff --git a/packages/data-provider/src/prAutomation.spec.ts b/packages/data-provider/src/prAutomation.spec.ts index 69fcbf39804..2d825213876 100644 --- a/packages/data-provider/src/prAutomation.spec.ts +++ b/packages/data-provider/src/prAutomation.spec.ts @@ -1,6 +1,6 @@ -import { configSchema } from './config'; -import { clampPRAutomationTrust } from './types/prAutomation'; import type { PRAutomationTrustLevel } from './types/prAutomation'; +import { clampPRAutomationTrust } from './types/prAutomation'; +import { configSchema } from './config'; describe('clampPRAutomationTrust', () => { it('defaults to the narrowest level when the user chose nothing', () => { @@ -43,6 +43,7 @@ describe('agent PR automation config', () => { enabled: false, maxRounds: 5, maxMinutes: 120, + maxBots: 20, maxTrust: 'approvedBots', }); }); @@ -65,6 +66,14 @@ describe('agent PR automation config', () => { expect(parse({ maxMinutes }).success).toBe(false); }); + it.each([0, 101, 1.5])('rejects %s approved bots per repository', (maxBots) => { + expect(parse({ maxBots }).success).toBe(false); + }); + + it('accepts the largest approved bot limit', () => { + expect(parse({ maxBots: 100 }).success).toBe(true); + }); + it.each(['everyone', 'bots', ''])('rejects %s as a trust ceiling', (maxTrust) => { expect(parse({ maxTrust }).success).toBe(false); }); diff --git a/packages/data-provider/src/types/prAutomation.ts b/packages/data-provider/src/types/prAutomation.ts index 7495bac18a2..90c2b8d70ac 100644 --- a/packages/data-provider/src/types/prAutomation.ts +++ b/packages/data-provider/src/types/prAutomation.ts @@ -15,6 +15,10 @@ export const PR_AUTOMATION_STOP_CODES = [ 'disabled_by_admin', ] as const; +/** Largest per-repository bot allowlist a deployment can configure. */ +export const MAX_PR_AUTOMATION_BOTS = 100; +export const DEFAULT_PR_AUTOMATION_BOTS = 20; + /** Ordered from narrowest to widest: who may be acted on. */ export const PR_AUTOMATION_TRUST_LEVELS = ['approvedBots', 'collaborators', 'anyone'] as const; diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index ecea2ae3596..be292eb9b0e 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -3727,6 +3727,46 @@ describe('Conversation Operations', () => { }); describe('deleteConvos', () => { + it('removes the PR automation record of a deleted conversation and keeps the others', async () => { + const conversationId = uuidv4(); + const keptConversationId = uuidv4(); + const PRAutomation = mongoose.models.PRAutomation; + for (const id of [conversationId, keptConversationId]) { + await Conversation.create({ conversationId: id, user: 'user123', endpoint: 'agents' }); + await PRAutomation.create({ user: 'user123', conversationId: id }); + } + + await deleteConvos('user123', { conversationId }); + + expect(await PRAutomation.countDocuments({ conversationId })).toBe(0); + expect(await PRAutomation.countDocuments({ conversationId: keptConversationId })).toBe(1); + }); + + it('removes the PR automation record of a root that a previous attempt already deleted', async () => { + const rootId = uuidv4(); + const childId = uuidv4(); + const PRAutomation = mongoose.models.PRAutomation; + await Conversation.create({ + conversationId: childId, + user: 'user123', + endpoint: EModelEndpoint.agents, + subagentThread: { + rootConversationId: rootId, + parentConversationId: rootId, + parentMessageId: 'message-1', + parentToolCallId: 'call-1', + subagentType: 'agent-child', + subagentKind: 'agent', + depth: 1, + }, + }); + await PRAutomation.create({ user: 'user123', conversationId: rootId }); + + await deleteConvos('user123', { conversationId: rootId }, { allowEmpty: true }); + + expect(await PRAutomation.countDocuments({ conversationId: rootId })).toBe(0); + }); + it('retires queued-turn work before each conversation deletion wave', async () => { const conversationId = uuidv4(); await Conversation.create({ diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index 350178cafe2..1123dab44cc 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -4140,6 +4140,12 @@ export function createConversationMethods( ); await options?.beforeDelete?.(waveIds); await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds); + /** Removed before the conversation, so a failure here leaves both for a retry and a + * webhook can never claim a record whose controlling conversation is gone. */ + await mongoose.models.PRAutomation?.deleteMany({ + user, + conversationId: { $in: waveIds }, + }); const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); if (result.deletedCount > 0) { /** Result erasure is irreversible. Keep receipts intact when a @@ -4174,6 +4180,10 @@ export function createConversationMethods( ]; if (recoveryConversationIds.length > 0) { + await mongoose.models.PRAutomation?.deleteMany({ + user, + conversationId: { $in: recoveryConversationIds }, + }); await deps?.deleteAgentQueuedTurns?.( user, recoveryConversationIds.map((conversationId) => ({ diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 7b7becb50c9..222a0283107 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -1,7 +1,7 @@ import mongoose from 'mongoose'; -import { MongoMemoryServer } from 'mongodb-memory-server'; import { logger, createModels } from '..'; -import { MAX_PR_AUTOMATION_BOTS } from '../types/prAutomation'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type { PRAutomationStopCode } from 'librechat-data-provider'; import { createPRAutomationMethods } from './prAutomation'; logger.silent = true; @@ -14,6 +14,21 @@ const userId = new mongoose.Types.ObjectId().toString(); const otherUserId = new mongoose.Types.ObjectId().toString(); const key = { userId, conversationId: 'convo-1' }; const limits = { maxRounds: 3, maxMinutes: 60 }; +const maxBots = 3; +const head = (n: number) => String(n).repeat(40).slice(0, 40); + +const claim = (n: number, extra: { now?: Date } = {}) => + methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(n), ...extra }); +const settle = (round: number, state: 'waiting' | 'needs_user' = 'waiting') => + methods.settlePRAutomationRound({ ...key, round, state }); +/** Test setup that does not depend on the settle method under test. */ +const toWaiting = () => + PRAutomation.updateOne( + { user: userId, conversationId: key.conversationId }, + { $set: { state: 'waiting' } }, + ); +const addBot = (id: number, login?: string, limit = maxBots) => + methods.addPRAutomationBot(key, login == null ? { id } : { id, login }, limit); beforeAll(async () => { mongoServer = await MongoMemoryServer.create(); @@ -59,15 +74,15 @@ describe('enablePRAutomation', () => { test('does not change an active record that is enabled again', async () => { await methods.enablePRAutomation(key); - await methods.claimPRAutomationRound({ ...key, ...limits, headSha: 'a'.repeat(40) }); + await claim(1); const again = await methods.enablePRAutomation(key); expect(again).toMatchObject({ state: 'fixing', round: 1 }); }); test('restarts a stopped record with a fresh round counter and window', async () => { await methods.enablePRAutomation(key); - await methods.claimPRAutomationRound({ ...key, ...limits, headSha: 'a'.repeat(40) }); - await methods.setPRAutomationState(key, { state: 'stopped', stopCode: 'user_stopped' }); + await claim(1); + await methods.stopPRAutomation(key, 'user_stopped'); const restarted = await methods.enablePRAutomation(key); expect(restarted).toMatchObject({ state: 'idle', round: 0 }); @@ -76,19 +91,53 @@ describe('enablePRAutomation', () => { expect(restarted.lastHeadSha).toBeUndefined(); }); + test('lets a restarted record claim a head it claimed before the stop', async () => { + await methods.enablePRAutomation(key); + await claim(1); + await methods.stopPRAutomation(key, 'user_stopped'); + await methods.enablePRAutomation(key); + expect((await claim(1)).ok).toBe(true); + }); + test('keeps records of different users apart', async () => { await methods.enablePRAutomation(key); expect(await methods.getPRAutomation({ ...key, userId: otherUserId })).toBeNull(); }); + + test('clears approved bots when the conversation is bound to a different repository', async () => { + await methods.enablePRAutomation({ ...key, repository: 'acme/one', pullNumber: 1 }); + await addBot(101); + const rebound = await methods.enablePRAutomation({ + ...key, + repository: 'acme/two', + pullNumber: 2, + }); + expect(rebound).toMatchObject({ repository: 'acme/two', pullNumber: 2, trustedBots: [] }); + }); + + test('does not carry a bot approved before any repository was bound', async () => { + await methods.enablePRAutomation(key); + await addBot(101); + const bound = await methods.enablePRAutomation({ ...key, repository: 'acme/one' }); + expect(bound.trustedBots).toEqual([]); + }); + + test('keeps approved bots when the same repository is bound to another pull request', async () => { + await methods.enablePRAutomation({ ...key, repository: 'acme/one', pullNumber: 1 }); + await addBot(101); + const rebound = await methods.enablePRAutomation({ + ...key, + repository: 'acme/one', + pullNumber: 2, + }); + expect(rebound).toMatchObject({ pullNumber: 2, trustedBots: [{ id: 101 }] }); + }); }); describe('claimPRAutomationRound', () => { - const head = (n: number) => String(n).repeat(40).slice(0, 40); - test('starts a round, counts it and records the head', async () => { await methods.enablePRAutomation(key); - const result = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); - expect(result).toMatchObject({ + expect(await claim(1)).toMatchObject({ ok: true, value: { state: 'fixing', round: 1, lastHeadSha: head(1) }, }); @@ -96,62 +145,48 @@ describe('claimPRAutomationRound', () => { test('rejects a second delivery for a head that was already claimed', async () => { await methods.enablePRAutomation(key); - await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); - await methods.setPRAutomationState(key, { state: 'waiting' }); - const duplicate = await methods.claimPRAutomationRound({ - ...key, - ...limits, - headSha: head(1), - }); - expect(duplicate).toEqual({ ok: false, error: { code: 'stale_head' } }); + await claim(1); + await toWaiting(); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'stale_head' } }); expect((await methods.getPRAutomation(key))?.round).toBe(1); }); + test('rejects a delayed delivery of an earlier head after a later head was claimed', async () => { + await methods.enablePRAutomation(key); + await claim(1); + await toWaiting(); + await claim(2); + await toWaiting(); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'stale_head' } }); + expect((await methods.getPRAutomation(key))?.round).toBe(2); + }); + test('rejects a claim while a round is already running', async () => { await methods.enablePRAutomation(key); - await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); - const overlapping = await methods.claimPRAutomationRound({ - ...key, - ...limits, - headSha: head(2), - }); - expect(overlapping).toEqual({ ok: false, error: { code: 'not_active' } }); + await claim(1); + expect(await claim(2)).toEqual({ ok: false, error: { code: 'not_active' } }); }); test('stops at the round cap and never goes past it', async () => { await methods.enablePRAutomation(key); for (let round = 1; round <= limits.maxRounds; round++) { - const claimed = await methods.claimPRAutomationRound({ - ...key, - ...limits, - headSha: head(round), - }); - expect(claimed.ok).toBe(true); - await methods.setPRAutomationState(key, { state: 'waiting' }); + expect((await claim(round)).ok).toBe(true); + await toWaiting(); } - const over = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(9) }); - expect(over).toEqual({ ok: false, error: { code: 'round_cap' } }); + expect(await claim(9)).toEqual({ ok: false, error: { code: 'round_cap' } }); expect((await methods.getPRAutomation(key))?.round).toBe(limits.maxRounds); }); test('lets exactly one of several concurrent deliveries claim the same head', async () => { await methods.enablePRAutomation(key); - const results = await Promise.all( - Array.from({ length: 8 }, () => - methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }), - ), - ); + const results = await Promise.all(Array.from({ length: 8 }, () => claim(1))); expect(results.filter((result) => result.ok)).toHaveLength(1); expect((await methods.getPRAutomation(key))?.round).toBe(1); }); test('never exceeds the cap under concurrent deliveries for different heads', async () => { await methods.enablePRAutomation(key); - const results = await Promise.all( - Array.from({ length: 8 }, (_, index) => - methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(index + 1) }), - ), - ); + const results = await Promise.all(Array.from({ length: 8 }, (_, index) => claim(index + 1))); expect(results.filter((result) => result.ok)).toHaveLength(1); expect((await methods.getPRAutomation(key))?.round).toBeLessThanOrEqual(limits.maxRounds); }); @@ -159,59 +194,104 @@ describe('claimPRAutomationRound', () => { test('stops once the wall-clock window has passed', async () => { await methods.enablePRAutomation(key); const start = new Date('2026-01-01T00:00:00Z'); - await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1), now: start }); - await methods.setPRAutomationState(key, { state: 'waiting' }); + await claim(1, { now: start }); + await toWaiting(); const later = new Date(start.getTime() + (limits.maxMinutes + 1) * 60_000); - const expired = await methods.claimPRAutomationRound({ - ...key, - ...limits, - headSha: head(2), - now: later, - }); - expect(expired).toEqual({ ok: false, error: { code: 'time_cap' } }); + expect(await claim(2, { now: later })).toEqual({ ok: false, error: { code: 'time_cap' } }); }); test('reports not_found for a conversation with no record', async () => { - const result = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); - expect(result).toEqual({ ok: false, error: { code: 'not_found' } }); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_found' } }); }); test('rejects a claim on a stopped record', async () => { await methods.enablePRAutomation(key); - await methods.setPRAutomationState(key, { state: 'stopped', stopCode: 'user_stopped' }); - const result = await methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(1) }); - expect(result).toEqual({ ok: false, error: { code: 'not_active' } }); + await methods.stopPRAutomation(key, 'user_stopped'); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_active' } }); }); }); -describe('setPRAutomationState', () => { - test('records a stop code only with the stopped state', async () => { +describe('settlePRAutomationRound', () => { + test('settles the round that owns it, including from needs_user', async () => { await methods.enablePRAutomation(key); - const stopped = await methods.setPRAutomationState(key, { + await claim(1); + expect(await settle(1, 'needs_user')).toMatchObject({ state: 'needs_user', round: 1 }); + expect(await settle(1, 'waiting')).toMatchObject({ state: 'waiting', round: 1 }); + }); + + test('ignores a completion that belongs to an earlier round', async () => { + await methods.enablePRAutomation(key); + await claim(1); + await settle(1); + await claim(2); + + expect(await settle(1, 'needs_user')).toBeNull(); + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'fixing', round: 2 }); + }); + + test('does not let a stale completion open the way for an overlapping claim', async () => { + await methods.enablePRAutomation(key); + await claim(1); + await settle(1); + await claim(2); + await settle(1); + expect(await claim(3)).toEqual({ ok: false, error: { code: 'not_active' } }); + }); + + test('does not settle a record that never started a round', async () => { + await methods.enablePRAutomation(key); + expect(await settle(0)).toBeNull(); + expect((await methods.getPRAutomation(key))?.state).toBe('idle'); + }); + + test('does not revive a stopped record', async () => { + await methods.enablePRAutomation(key); + await claim(1); + await methods.stopPRAutomation(key, 'user_stopped'); + expect(await settle(1)).toBeNull(); + expect((await methods.getPRAutomation(key))?.state).toBe('stopped'); + }); + + test('returns null for a conversation with no record', async () => { + expect(await settle(1)).toBeNull(); + }); +}); + +describe('stopPRAutomation', () => { + test('records the stop code', async () => { + await methods.enablePRAutomation(key); + expect(await methods.stopPRAutomation(key, 'round_cap')).toMatchObject({ state: 'stopped', stopCode: 'round_cap', }); - expect(stopped).toMatchObject({ state: 'stopped', stopCode: 'round_cap' }); }); - test('does not revive a stopped record', async () => { + test('stops a running round without waiting for it to settle', async () => { await methods.enablePRAutomation(key); - await methods.setPRAutomationState(key, { state: 'stopped', stopCode: 'user_stopped' }); - const revived = await methods.setPRAutomationState(key, { state: 'waiting' }); - expect(revived).toBeNull(); - expect((await methods.getPRAutomation(key))?.state).toBe('stopped'); + await claim(1); + expect(await methods.stopPRAutomation(key, 'user_stopped')).toMatchObject({ + state: 'stopped', + stopCode: 'user_stopped', + }); }); - test('clears a stale stop code when the state moves on', async () => { + test('keeps the first stop code when it is stopped again', async () => { await methods.enablePRAutomation(key); - await methods.setPRAutomationState(key, { state: 'needs_user' }); - const next = await methods.setPRAutomationState(key, { state: 'waiting' }); - expect(next?.stopCode).toBeUndefined(); + await methods.stopPRAutomation(key, 'round_cap'); + expect(await methods.stopPRAutomation(key, 'user_stopped')).toBeNull(); + expect((await methods.getPRAutomation(key))?.stopCode).toBe('round_cap'); + }); + + test('rejects a code outside the stop code list', async () => { + await methods.enablePRAutomation(key); + await expect( + methods.stopPRAutomation(key, 'because' as unknown as PRAutomationStopCode), + ).rejects.toThrow(); }); test('returns null for a conversation with no record', async () => { - expect(await methods.setPRAutomationState(key, { state: 'waiting' })).toBeNull(); + expect(await methods.stopPRAutomation(key, 'user_stopped')).toBeNull(); }); }); @@ -233,8 +313,7 @@ describe('setPRAutomationTrust', () => { describe('approved bots', () => { test('adds a bot by numeric id', async () => { await methods.enablePRAutomation(key); - const result = await methods.addPRAutomationBot(key, { id: 101, login: 'review-bot[bot]' }); - expect(result).toMatchObject({ + expect(await addBot(101, 'review-bot[bot]')).toMatchObject({ ok: true, value: { trustedBots: [{ id: 101, login: 'review-bot[bot]' }] }, }); @@ -242,32 +321,49 @@ describe('approved bots', () => { test('is idempotent by id even when the login was renamed', async () => { await methods.enablePRAutomation(key); - await methods.addPRAutomationBot(key, { id: 101, login: 'old-name[bot]' }); - const again = await methods.addPRAutomationBot(key, { id: 101, login: 'new-name[bot]' }); - expect(again.ok).toBe(true); + await addBot(101, 'old-name[bot]'); + expect((await addBot(101, 'new-name[bot]')).ok).toBe(true); const record = await methods.getPRAutomation(key); expect(record?.trustedBots).toEqual([{ id: 101, login: 'old-name[bot]' }]); }); - test('refuses a bot beyond the cap with a stable code', async () => { + test('refuses a bot beyond the limit it is given, with a stable code', async () => { await methods.enablePRAutomation(key); - for (let id = 1; id <= MAX_PR_AUTOMATION_BOTS; id++) { - const added = await methods.addPRAutomationBot(key, { id }); - expect({ id, ok: added.ok }).toEqual({ id, ok: true }); + for (let id = 1; id <= maxBots; id++) { + expect({ id, ok: (await addBot(id)).ok }).toEqual({ id, ok: true }); } - const over = await methods.addPRAutomationBot(key, { id: 9999 }); - expect(over).toEqual({ ok: false, error: { code: 'bot_limit' } }); + expect(await addBot(9999)).toEqual({ ok: false, error: { code: 'bot_limit' } }); + }); + + test('accepts a bot that is already approved when the list is full', async () => { + await methods.enablePRAutomation(key); + for (let id = 1; id <= maxBots; id++) { + await addBot(id); + } + expect((await addBot(1)).ok).toBe(true); + }); + + test('admits more bots once a higher limit is passed', async () => { + await methods.enablePRAutomation(key); + for (let id = 1; id <= maxBots; id++) { + await addBot(id); + } + expect((await addBot(maxBots + 1, undefined, maxBots + 2)).ok).toBe(true); + }); + + test('rejects a limit below one instead of storing without a cap', async () => { + await methods.enablePRAutomation(key); + await expect(addBot(1, undefined, 0)).rejects.toThrow(RangeError); }); test('reports not_found when the conversation has no record', async () => { - const result = await methods.addPRAutomationBot(key, { id: 101 }); - expect(result).toEqual({ ok: false, error: { code: 'not_found' } }); + expect(await addBot(101)).toEqual({ ok: false, error: { code: 'not_found' } }); }); test('removes a bot by id and leaves the others', async () => { await methods.enablePRAutomation(key); - await methods.addPRAutomationBot(key, { id: 1 }); - await methods.addPRAutomationBot(key, { id: 2 }); + await addBot(1); + await addBot(2); const record = await methods.removePRAutomationBot(key, 1); expect(record?.trustedBots).toEqual([{ id: 2 }]); }); @@ -275,7 +371,7 @@ describe('approved bots', () => { test('keeps each conversation allowlist separate', async () => { await methods.enablePRAutomation(key); await methods.enablePRAutomation({ userId, conversationId: 'convo-2' }); - await methods.addPRAutomationBot(key, { id: 101 }); + await addBot(101); const other = await methods.getPRAutomation({ userId, conversationId: 'convo-2' }); expect(other?.trustedBots).toEqual([]); }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 8445ad9e393..daf0c14207d 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -1,11 +1,13 @@ -import type { PRAutomationState } from 'librechat-data-provider'; -import type { PRAutomationStopCode } from 'librechat-data-provider'; +import { PR_AUTOMATION_STOP_CODES, MAX_PR_AUTOMATION_BOTS } from 'librechat-data-provider'; import type { PRAutomationTrustLevel } from 'librechat-data-provider'; +import type { PRAutomationStopCode } from 'librechat-data-provider'; +import type { PRAutomationState } from 'librechat-data-provider'; import type * as t from '~/types/prAutomation'; -import { MAX_PR_AUTOMATION_BOTS } from '~/types/prAutomation'; const PROJECTION = '-_id -__v'; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; +const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; +const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; export function createPRAutomationMethods(mongoose: typeof import('mongoose')): { getPRAutomation: (key: t.PRAutomationKey) => Promise; @@ -14,9 +16,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): claimPRAutomationRound: ( params: t.ClaimPRAutomationRoundParams, ) => Promise; - setPRAutomationState: ( + settlePRAutomationRound: ( + params: t.SettlePRAutomationRoundParams, + ) => Promise; + stopPRAutomation: ( key: t.PRAutomationKey, - next: { state: PRAutomationState; stopCode?: PRAutomationStopCode }, + stopCode: PRAutomationStopCode, ) => Promise; setPRAutomationTrust: ( key: t.PRAutomationKey, @@ -25,6 +30,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): addPRAutomationBot: ( key: t.PRAutomationKey, bot: t.IPRAutomationBot, + maxBots: number, ) => Promise; removePRAutomationBot: (key: t.PRAutomationKey, botId: number) => Promise; } { @@ -42,8 +48,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Idempotent. An existing active record is returned unchanged. A stopped * record is the user explicitly turning the automation back on, so it starts - * a fresh run: the round counter and the time window reset, which they could - * not do from a webhook-triggered turn. + * a fresh run: the round counter, the time window and the claimed heads reset, + * which they could not do from a webhook-triggered turn. + * + * The approved bots belong to one repository. Binding the conversation to a + * different repository, or to one for the first time, clears them, so a bot + * approved for repository A is never trusted in repository B. */ async function enablePRAutomation({ trust, @@ -61,6 +71,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): state: 'idle', round: 0, trustedBots: [], + claimedHeads: [], trust: trust ?? 'approvedBots', }, }, @@ -69,17 +80,18 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): await PRAutomation.updateOne( { ...filter, state: 'stopped' }, { - $set: { state: 'idle', round: 0, ...(trust != null && { trust }) }, + $set: { state: 'idle', round: 0, claimedHeads: [], ...(trust != null && { trust }) }, $unset: { stopCode: '', startedAt: '', lastHeadSha: '' }, }, ); - if (repository != null || pullNumber != null) { - await PRAutomation.updateOne(filter, { - $set: { - ...(repository != null && { repository }), - ...(pullNumber != null && { pullNumber }), - }, - }); + if (repository != null) { + await PRAutomation.updateOne( + { ...filter, repository: { $ne: repository } }, + { $set: { repository, trustedBots: [] } }, + ); + } + if (pullNumber != null) { + await PRAutomation.updateOne(filter, { $set: { pullNumber } }); } const record = await getPRAutomation(key); if (record == null) { @@ -97,10 +109,13 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Starts one fix round, atomically. The round count, the time window, the * state and the head are all conditions of a single `findOneAndUpdate`, so two - * deliveries racing for the same record cannot both pass the cap, and a - * duplicate delivery for a head that was already claimed is rejected instead - * of starting a second round. The persisted counter is what stops a new turn - * from resetting the cap. + * deliveries racing for the same record cannot both pass the cap. A head that + * any earlier round already claimed is rejected, so a delayed delivery of a + * superseded commit cannot spend the budget. The persisted counter is what + * stops a new turn from resetting the cap. + * + * A force-push back to an earlier SHA is rejected the same way; the user + * restarts the automation to work on it. */ async function claimPRAutomationRound({ maxRounds, @@ -125,11 +140,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): state: { $in: CLAIMABLE_STATES }, round: { $lt: maxRounds }, startedAt: { $gte: cutoff }, - lastHeadSha: { $ne: headSha }, + claimedHeads: { $ne: headSha }, }, { $inc: { round: 1 }, $set: { state: 'fixing', lastHeadSha: headSha }, + $push: { claimedHeads: headSha }, $unset: { stopCode: '' }, }, { new: true, select: PROJECTION }, @@ -154,19 +170,50 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: false, error: { code: 'stale_head' } }; } - /** A stopped record only leaves that state through `enablePRAutomation`. */ - async function setPRAutomationState( + /** + * Reports where the round that owns the record ended up. The transition only + * applies while that round is still the current one, so a completion that is + * retried after a later round was claimed cannot move the newer round to + * `waiting` and open the way for an overlapping claim. Returns `null` when + * nothing changed: no record, a stopped record, or a round that is not the + * current one. + */ + async function settlePRAutomationRound({ + round, + state, + ...key + }: t.SettlePRAutomationRoundParams): Promise { + if (!SETTLED_STATES.includes(state)) { + throw new RangeError( + 'A round settles as waiting or needs_user; use stopPRAutomation to stop', + ); + } + const PRAutomation = mongoose.models.PRAutomation; + return PRAutomation.findOneAndUpdate( + { ...keyFilter(key), state: { $in: SETTLEABLE_STATES }, round }, + { $set: { state } }, + { new: true, select: PROJECTION }, + ).lean(); + } + + /** + * Stops the automation whatever round is running, so a user stop is never + * lost to a race. A stop code is required: the client maps it to the reason + * it shows. The first stop wins, and a stopped record leaves that state only + * through `enablePRAutomation`. + */ + async function stopPRAutomation( key: t.PRAutomationKey, - next: { state: PRAutomationState; stopCode?: PRAutomationStopCode }, + stopCode: PRAutomationStopCode, ): Promise { + if (!PR_AUTOMATION_STOP_CODES.includes(stopCode)) { + throw new RangeError('A stopped PR automation needs a known stop code'); + } const PRAutomation = mongoose.models.PRAutomation; - const stopCode = next.state === 'stopped' ? next.stopCode : undefined; return PRAutomation.findOneAndUpdate( { ...keyFilter(key), state: { $ne: 'stopped' } }, - stopCode != null - ? { $set: { state: next.state, stopCode } } - : { $set: { state: next.state }, $unset: { stopCode: '' } }, - { new: true, select: PROJECTION, runValidators: true }, + { $set: { state: 'stopped', stopCode } }, + { new: true, select: PROJECTION }, ).lean(); } @@ -182,17 +229,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ).lean(); } - /** Idempotent by numeric id, and capped. A login is stored for display only. */ + /** + * Idempotent by numeric id. The limit is the configured `maxBots`, resolved by + * the caller, and cannot exceed the schema ceiling. A login is stored for + * display only. + */ async function addPRAutomationBot( key: t.PRAutomationKey, bot: t.IPRAutomationBot, + maxBots: number, ): Promise { + if (!Number.isInteger(maxBots) || maxBots < 1 || maxBots > MAX_PR_AUTOMATION_BOTS) { + throw new RangeError(`maxBots must be an integer from 1 to ${MAX_PR_AUTOMATION_BOTS}`); + } const PRAutomation = mongoose.models.PRAutomation; const updated = await PRAutomation.findOneAndUpdate( { ...keyFilter(key), 'trustedBots.id': { $ne: bot.id }, - [`trustedBots.${MAX_PR_AUTOMATION_BOTS - 1}`]: { $exists: false }, + [`trustedBots.${maxBots - 1}`]: { $exists: false }, }, { $push: { trustedBots: bot } }, { new: true, select: PROJECTION, runValidators: true }, @@ -228,7 +283,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): enablePRAutomation, disablePRAutomation, claimPRAutomationRound, - setPRAutomationState, + settlePRAutomationRound, + stopPRAutomation, setPRAutomationTrust, addPRAutomationBot, removePRAutomationBot, diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index d9318173d1b..b72e48ba2c0 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -8,6 +8,7 @@ import { import type * as t from '~/types'; import { createToolApprovalGrantModel } from '~/models/toolApprovalGrant'; import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user'; +import { createPRAutomationModel } from '~/models/prAutomation'; import balanceSchema from '~/schema/balance'; import userSchema from '~/schema/user'; @@ -51,6 +52,7 @@ beforeAll(async () => { /** Initialize methods */ methods = createUserMethods(mongoose); createToolApprovalGrantModel(mongoose); + createPRAutomationModel(mongoose); }); afterAll(async () => { @@ -2199,6 +2201,22 @@ describe('User Methods - Database Tests', () => { }); }); +describe('PR automation cleanup on account deletion', () => { + it('removes the deleted user records and leaves other users alone', async () => { + const user = await User.create({ email: 'delete-pr@example.com', provider: 'local' }); + const id = user._id.toString(); + const PRAutomation = mongoose.models.PRAutomation; + await PRAutomation.create({ user: id, conversationId: 'chat-a' }); + await PRAutomation.create({ user: id, conversationId: 'chat-b' }); + await PRAutomation.create({ user: 'another-user', conversationId: 'chat-a' }); + + await expect(methods.deleteUserById(id)).resolves.toMatchObject({ deletedCount: 1 }); + + expect(await PRAutomation.countDocuments({ user: id })).toBe(0); + expect(await PRAutomation.countDocuments({ user: 'another-user' })).toBe(1); + }); +}); + describe('personal grant cleanup before account deletion', () => { async function owner() { const user = await User.create({ email: 'delete-grants@example.com', provider: 'local' }); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index cc950283441..80407af37ae 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -643,6 +643,7 @@ export function createUserMethods( try { const User = mongoose.models.User; await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); + await mongoose.models.PRAutomation?.deleteMany({ user: userId }); const result = await User.deleteOne({ _id: userId }); if (result.deletedCount === 0) { return { deletedCount: 0, message: 'No user found with that ID.' }; diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts index d27cc1422ea..e65ee8c97c9 100644 --- a/packages/data-schemas/src/models/index.ts +++ b/packages/data-schemas/src/models/index.ts @@ -12,6 +12,7 @@ import { createSkillSyncStatusModel } from './skillSyncStatus'; import { createConversationTagModel } from './conversationTag'; import { createCodeEnvironmentModel } from './codeEnvironment'; import { createAgentCategoryModel } from './agentCategory'; +import { createPRAutomationModel } from './prAutomation'; import { createResourceLockModel } from './resourceLock'; import { createChatProjectModel } from './chatProject'; import { createAgentApiKeyModel } from './agentApiKey'; @@ -22,7 +23,6 @@ import { createPluginAuthModel } from './pluginAuth'; import { createSharedLinkModel } from './sharedLink'; import { createAccessRoleModel } from './accessRole'; import { createToolFavoriteModel } from './favorite'; -import { createPRAutomationModel } from './prAutomation'; import { createMCPServerModel } from './mcpServer'; import { createAssistantModel } from './assistant'; import { createSkillFileModel } from './skillFile'; diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 584e8cb047e..f9580b6f2a9 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -1,9 +1,9 @@ import { Schema } from 'mongoose'; import { PR_AUTOMATION_STATES } from 'librechat-data-provider'; +import { MAX_PR_AUTOMATION_BOTS } from 'librechat-data-provider'; import { PR_AUTOMATION_STOP_CODES } from 'librechat-data-provider'; import { PR_AUTOMATION_TRUST_LEVELS } from 'librechat-data-provider'; import type { IPRAutomationDocument } from '~/types/prAutomation'; -import { MAX_PR_AUTOMATION_BOTS } from '~/types/prAutomation'; const trustedBotSchema = new Schema( { @@ -16,8 +16,7 @@ const trustedBotSchema = new Schema( const prAutomationSchema: Schema = new Schema( { user: { - type: Schema.Types.ObjectId, - ref: 'User', + type: String, required: true, index: true, }, @@ -56,6 +55,7 @@ const prAutomationSchema: Schema = new Schema( round: { type: Number, default: 0, min: 0, required: true }, startedAt: { type: Date }, lastHeadSha: { type: String, maxlength: 64 }, + claimedHeads: { type: [{ type: String, maxlength: 64 }], default: [] }, }, { timestamps: true }, ); diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index bf1d5d0a4c7..80935e03dee 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -1,9 +1,7 @@ -import type { PRAutomationState } from 'librechat-data-provider'; -import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationTrustLevel } from 'librechat-data-provider'; -import type { Document, Types } from 'mongoose'; - -export const MAX_PR_AUTOMATION_BOTS = 20; +import type { PRAutomationStopCode } from 'librechat-data-provider'; +import type { PRAutomationState } from 'librechat-data-provider'; +import type { Document } from 'mongoose'; /** A bot the user approved for one repository, matched by its numeric account id. */ export interface IPRAutomationBot { @@ -13,7 +11,7 @@ export interface IPRAutomationBot { } export interface IPRAutomation { - user: Types.ObjectId; + user: string; tenantId?: string; conversationId: string; /** `owner/name` of the repository the pull request belongs to. */ @@ -28,7 +26,14 @@ export interface IPRAutomation { round: number; /** First automated round, the start of the wall-clock window. */ startedAt?: Date; + /** The head the latest round works on. */ lastHeadSha?: string; + /** + * Every head a round has been claimed for since the last restart. A delayed + * delivery of an earlier head is rejected against this list. It holds at most + * one entry per round, so the round cap bounds it. + */ + claimedHeads: string[]; createdAt?: Date; updatedAt?: Date; } @@ -36,7 +41,7 @@ export interface IPRAutomation { export interface IPRAutomationDocument extends IPRAutomation, Document {} export interface PRAutomationKey { - userId: string | Types.ObjectId; + userId: string; conversationId: string; } @@ -46,6 +51,12 @@ export interface EnablePRAutomationParams extends PRAutomationKey { pullNumber?: number; } +export interface SettlePRAutomationRoundParams extends PRAutomationKey { + /** The round that is reporting. A completion for any other round is ignored. */ + round: number; + state: 'waiting' | 'needs_user'; +} + export interface ClaimPRAutomationRoundParams extends PRAutomationKey { maxRounds: number; maxMinutes: number; From c9f89d20fef9eda294b5b3a4fc605bccc299f543 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:24:23 +0200 Subject: [PATCH 03/25] fix: Fence PR Automation Rebinding and Deletion Binding a conversation to another pull request starts a fresh run and writes the repository and number in one conditional update, and each claim carries a run id so a completion from a replaced run cannot settle the new one. Conversation and account deletion stop the record first and remove it only after the delete commits, so a failed delete keeps the stored state. --- .../data-provider/src/types/prAutomation.ts | 2 + .../src/methods/conversation.spec.ts | 36 +++++ .../data-schemas/src/methods/conversation.ts | 26 ++-- .../src/methods/prAutomation.spec.ts | 146 ++++++++++++++---- .../data-schemas/src/methods/prAutomation.ts | 118 +++++++++++--- .../src/methods/user.methods.spec.ts | 26 ++++ packages/data-schemas/src/methods/user.ts | 12 +- .../data-schemas/src/schema/prAutomation.ts | 1 + .../data-schemas/src/types/prAutomation.ts | 19 ++- 9 files changed, 318 insertions(+), 68 deletions(-) diff --git a/packages/data-provider/src/types/prAutomation.ts b/packages/data-provider/src/types/prAutomation.ts index 90c2b8d70ac..ecd0f04733e 100644 --- a/packages/data-provider/src/types/prAutomation.ts +++ b/packages/data-provider/src/types/prAutomation.ts @@ -13,6 +13,8 @@ export const PR_AUTOMATION_STOP_CODES = [ 'approval_expired', 'pull_request_closed', 'disabled_by_admin', + 'conversation_deleting', + 'account_deleting', ] as const; /** Largest per-repository bot allowlist a deployment can configure. */ diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index be292eb9b0e..7ebe15b3bea 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -19,6 +19,7 @@ import type { } from '../types'; import { ConversationMethods, createConversationMethods } from './conversation'; import { tenantStorage, runAsSystem } from '~/config/tenantContext'; +import { createPRAutomationMethods } from './prAutomation'; import { createChatProjectMethods } from './chatProject'; import { createModels } from '../models'; @@ -3742,6 +3743,41 @@ describe('Conversation Operations', () => { expect(await PRAutomation.countDocuments({ conversationId: keptConversationId })).toBe(1); }); + describe('when the conversation delete fails', () => { + const user = 'user123'; + const setup = async () => { + const conversationId = uuidv4(); + await Conversation.create({ conversationId, user, endpoint: EModelEndpoint.agents }); + const automation = createPRAutomationMethods(mongoose); + await automation.enablePRAutomation({ userId: user, conversationId }); + jest.spyOn(Conversation, 'deleteMany').mockRejectedValueOnce(new Error('delete failed')); + await expect(deleteConvos(user, { conversationId })).rejects.toThrow('delete failed'); + return { conversationId, automation }; + }; + + it('keeps the PR automation record', async () => { + const { conversationId, automation } = await setup(); + expect(await automation.getPRAutomation({ userId: user, conversationId })).not.toBeNull(); + expect(await Conversation.exists({ conversationId })).not.toBeNull(); + }); + + it('stops the record so a webhook cannot claim it meanwhile', async () => { + const { conversationId, automation } = await setup(); + expect(await automation.getPRAutomation({ userId: user, conversationId })).toMatchObject({ + state: 'stopped', + stopCode: 'conversation_deleting', + }); + const claimed = await automation.claimPRAutomationRound({ + userId: user, + conversationId, + maxRounds: 5, + maxMinutes: 60, + headSha: 'a'.repeat(40), + }); + expect(claimed).toEqual({ ok: false, error: { code: 'not_active' } }); + }); + }); + it('removes the PR automation record of a root that a previous attempt already deleted', async () => { const rootId = uuidv4(); const childId = uuidv4(); diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index 1123dab44cc..6ebff8067fa 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -53,6 +53,7 @@ import { isAgentFadingTier, isAgentFadingTierEntries } from '~/utils/fading'; import { isCompactionSemanticIndexProjection } from '~/types/compaction'; import { withoutMeiliIndexing } from '~/models/plugins/mongoMeili'; import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite'; +import { createPRAutomationMethods } from './prAutomation'; import { isValidObjectIdString } from '~/utils/objectId'; import { decrementTagCounts } from './conversationTag'; import logger from '~/config/winston'; @@ -668,6 +669,7 @@ export function createConversationMethods( deps?: ConversationMethodDeps, ): ConversationMethods { let legacyReceiptExpiryCursor: Types.ObjectId | undefined; + const prAutomation = createPRAutomationMethods(mongoose); /** * Stamps a real assistant reply with a strictly increasing server value. @@ -4140,13 +4142,16 @@ export function createConversationMethods( ); await options?.beforeDelete?.(waveIds); await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds); - /** Removed before the conversation, so a failure here leaves both for a retry and a - * webhook can never claim a record whose controlling conversation is gone. */ - await mongoose.models.PRAutomation?.deleteMany({ - user, - conversationId: { $in: waveIds }, - }); + /** Fenced before the delete so a webhook cannot claim a record whose conversation is + * going away. The record itself is removed only once the delete has committed, so a + * failed delete keeps the stored state and a retry finds it. */ + await prAutomation.stopPRAutomations(user, 'conversation_deleting', waveIds); const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); + try { + await prAutomation.deletePRAutomations(user, waveIds); + } catch { + logger.warn('[deleteConvos] PR automation cleanup failed; the record stays stopped.'); + } if (result.deletedCount > 0) { /** Result erasure is irreversible. Keep receipts intact when a * pre-delete hook or the conversation delete itself fails, so a @@ -4180,10 +4185,11 @@ export function createConversationMethods( ]; if (recoveryConversationIds.length > 0) { - await mongoose.models.PRAutomation?.deleteMany({ - user, - conversationId: { $in: recoveryConversationIds }, - }); + try { + await prAutomation.deletePRAutomations(user, recoveryConversationIds); + } catch { + logger.warn('[deleteConvos] PR automation cleanup failed; the record stays stopped.'); + } await deps?.deleteAgentQueuedTurns?.( user, recoveryConversationIds.map((conversationId) => ({ diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 222a0283107..3e72dcca604 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -16,11 +16,22 @@ const key = { userId, conversationId: 'convo-1' }; const limits = { maxRounds: 3, maxMinutes: 60 }; const maxBots = 3; const head = (n: number) => String(n).repeat(40).slice(0, 40); +const pullOne = { repository: 'acme/one', pullNumber: 1 }; +const pullTwo = { repository: 'acme/one', pullNumber: 2 }; +const otherRepository = { repository: 'acme/two', pullNumber: 2 }; const claim = (n: number, extra: { now?: Date } = {}) => methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(n), ...extra }); -const settle = (round: number, state: 'waiting' | 'needs_user' = 'waiting') => - methods.settlePRAutomationRound({ ...key, round, state }); +/** Claims a round that must succeed and returns the record it started. */ +const startRound = async (n: number) => { + const result = await claim(n); + if (!result.ok) { + throw new Error(`claim ${n} failed: ${result.error.code}`); + } + return result.value; +}; +const settle = (round: number, runId: string, state: 'waiting' | 'needs_user' = 'waiting') => + methods.settlePRAutomationRound({ ...key, round, runId, state }); /** Test setup that does not depend on the settle method under test. */ const toWaiting = () => PRAutomation.updateOne( @@ -105,35 +116,90 @@ describe('enablePRAutomation', () => { }); test('clears approved bots when the conversation is bound to a different repository', async () => { - await methods.enablePRAutomation({ ...key, repository: 'acme/one', pullNumber: 1 }); + await methods.enablePRAutomation({ ...key, binding: pullOne }); await addBot(101); - const rebound = await methods.enablePRAutomation({ - ...key, - repository: 'acme/two', - pullNumber: 2, - }); - expect(rebound).toMatchObject({ repository: 'acme/two', pullNumber: 2, trustedBots: [] }); + const rebound = await methods.enablePRAutomation({ ...key, binding: otherRepository }); + expect(rebound).toMatchObject({ ...otherRepository, trustedBots: [] }); }); test('does not carry a bot approved before any repository was bound', async () => { await methods.enablePRAutomation(key); await addBot(101); - const bound = await methods.enablePRAutomation({ ...key, repository: 'acme/one' }); + const bound = await methods.enablePRAutomation({ ...key, binding: pullOne }); expect(bound.trustedBots).toEqual([]); }); test('keeps approved bots when the same repository is bound to another pull request', async () => { - await methods.enablePRAutomation({ ...key, repository: 'acme/one', pullNumber: 1 }); + await methods.enablePRAutomation({ ...key, binding: pullOne }); await addBot(101); - const rebound = await methods.enablePRAutomation({ - ...key, - repository: 'acme/one', - pullNumber: 2, - }); + const rebound = await methods.enablePRAutomation({ ...key, binding: pullTwo }); expect(rebound).toMatchObject({ pullNumber: 2, trustedBots: [{ id: 101 }] }); }); }); +describe('binding a pull request', () => { + test('starts a fresh run when bound to another pull request in the same repository', async () => { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + await startRound(1); + await toWaiting(); + await startRound(2); + + const rebound = await methods.enablePRAutomation({ ...key, binding: pullTwo }); + expect(rebound).toMatchObject({ pullNumber: 2, state: 'idle', round: 0, claimedHeads: [] }); + expect(rebound.startedAt).toBeUndefined(); + expect(rebound.lastHeadSha).toBeUndefined(); + }); + + test('lets the new pull request claim a head the old one claimed', async () => { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + await startRound(1); + await methods.enablePRAutomation({ ...key, binding: pullTwo }); + expect((await claim(1)).ok).toBe(true); + }); + + test('does not carry the old pull request round count into the new one', async () => { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + for (let round = 1; round <= limits.maxRounds; round++) { + await startRound(round); + await toWaiting(); + } + await methods.enablePRAutomation({ ...key, binding: pullTwo }); + expect((await claim(9)).ok).toBe(true); + }); + + test('ignores a completion from the round of the pull request it replaced', async () => { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + const running = await startRound(1); + await methods.enablePRAutomation({ ...key, binding: pullTwo }); + + expect(await settle(1, running.runId)).toBeNull(); + expect((await methods.getPRAutomation(key))?.state).toBe('idle'); + }); + + test('leaves a run alone when it is bound to the pair it already has', async () => { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + await startRound(1); + const again = await methods.enablePRAutomation({ ...key, binding: pullOne }); + expect(again).toMatchObject({ state: 'fixing', round: 1 }); + }); + + /** Guard, not a proven regression: the interleaving it protects against is timing dependent. */ + test('never leaves a repository and pull request pair nobody asked for', async () => { + await methods.enablePRAutomation(key); + const pairs = [pullOne, otherRepository]; + await Promise.all( + Array.from({ length: 24 }, (_, index) => + methods.enablePRAutomation({ ...key, binding: pairs[index % 2] }), + ), + ); + const record = await methods.getPRAutomation(key); + expect(pairs).toContainEqual({ + repository: record?.repository, + pullNumber: record?.pullNumber, + }); + }); +}); + describe('claimPRAutomationRound', () => { test('starts a round, counts it and records the head', async () => { await methods.enablePRAutomation(key); @@ -215,46 +281,68 @@ describe('claimPRAutomationRound', () => { describe('settlePRAutomationRound', () => { test('settles the round that owns it, including from needs_user', async () => { await methods.enablePRAutomation(key); - await claim(1); - expect(await settle(1, 'needs_user')).toMatchObject({ state: 'needs_user', round: 1 }); - expect(await settle(1, 'waiting')).toMatchObject({ state: 'waiting', round: 1 }); + const { runId } = await startRound(1); + expect(await settle(1, runId, 'needs_user')).toMatchObject({ state: 'needs_user', round: 1 }); + expect(await settle(1, runId, 'waiting')).toMatchObject({ state: 'waiting', round: 1 }); }); test('ignores a completion that belongs to an earlier round', async () => { await methods.enablePRAutomation(key); - await claim(1); - await settle(1); + const { runId } = await startRound(1); + await settle(1, runId); await claim(2); - expect(await settle(1, 'needs_user')).toBeNull(); + expect(await settle(1, runId, 'needs_user')).toBeNull(); expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'fixing', round: 2 }); }); test('does not let a stale completion open the way for an overlapping claim', async () => { await methods.enablePRAutomation(key); - await claim(1); - await settle(1); + const { runId } = await startRound(1); + await settle(1, runId); await claim(2); - await settle(1); + await settle(1, runId); expect(await claim(3)).toEqual({ ok: false, error: { code: 'not_active' } }); }); + test('ignores a completion from a run that was stopped and restarted', async () => { + await methods.enablePRAutomation(key); + const previous = await startRound(1); + await methods.stopPRAutomation(key, 'user_stopped'); + await methods.enablePRAutomation(key); + await startRound(1); + + expect(await settle(1, previous.runId, 'needs_user')).toBeNull(); + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'fixing', round: 1 }); + }); + + test('ignores a completion from a record that was disabled and enabled again', async () => { + await methods.enablePRAutomation(key); + const previous = await startRound(1); + await methods.disablePRAutomation(key); + await methods.enablePRAutomation(key); + await startRound(1); + + expect(await settle(1, previous.runId, 'needs_user')).toBeNull(); + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'fixing', round: 1 }); + }); + test('does not settle a record that never started a round', async () => { await methods.enablePRAutomation(key); - expect(await settle(0)).toBeNull(); + expect(await settle(0, 'no-run')).toBeNull(); expect((await methods.getPRAutomation(key))?.state).toBe('idle'); }); test('does not revive a stopped record', async () => { await methods.enablePRAutomation(key); - await claim(1); + const { runId } = await startRound(1); await methods.stopPRAutomation(key, 'user_stopped'); - expect(await settle(1)).toBeNull(); + expect(await settle(1, runId)).toBeNull(); expect((await methods.getPRAutomation(key))?.state).toBe('stopped'); }); test('returns null for a conversation with no record', async () => { - expect(await settle(1)).toBeNull(); + expect(await settle(1, 'missing-run')).toBeNull(); }); }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index daf0c14207d..2dcf7fd5a7b 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import { PR_AUTOMATION_STOP_CODES, MAX_PR_AUTOMATION_BOTS } from 'librechat-data-provider'; import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; @@ -8,6 +9,11 @@ const PROJECTION = '-_id -__v'; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; +/** Everything that belongs to one run on one pull request. */ +const RUN_RESET = { + $set: { state: 'idle', round: 0, claimedHeads: [] }, + $unset: { stopCode: '', startedAt: '', lastHeadSha: '', runId: '' }, +}; export function createPRAutomationMethods(mongoose: typeof import('mongoose')): { getPRAutomation: (key: t.PRAutomationKey) => Promise; @@ -23,6 +29,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): key: t.PRAutomationKey, stopCode: PRAutomationStopCode, ) => Promise; + stopPRAutomations: ( + userId: string, + stopCode: PRAutomationStopCode, + conversationIds?: string[], + ) => Promise; + deletePRAutomations: (userId: string, conversationIds?: string[]) => Promise; setPRAutomationTrust: ( key: t.PRAutomationKey, trust: PRAutomationTrustLevel, @@ -39,6 +51,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): conversationId, }); + function assertStopCode(stopCode: PRAutomationStopCode): void { + if (!PR_AUTOMATION_STOP_CODES.includes(stopCode)) { + throw new RangeError('A stopped PR automation needs a known stop code'); + } + } + /** Absence is a normal answer here; a query failure throws. */ async function getPRAutomation(key: t.PRAutomationKey): Promise { const PRAutomation = mongoose.models.PRAutomation; @@ -48,17 +66,19 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Idempotent. An existing active record is returned unchanged. A stopped * record is the user explicitly turning the automation back on, so it starts - * a fresh run: the round counter, the time window and the claimed heads reset, - * which they could not do from a webhook-triggered turn. + * a fresh run: the round counter, the time window, the claimed heads and the + * run identity reset, which they could not do from a webhook-triggered turn. * - * The approved bots belong to one repository. Binding the conversation to a - * different repository, or to one for the first time, clears them, so a bot - * approved for repository A is never trusted in repository B. + * `binding` names the pull request. The repository and the number are one + * conditional write, so concurrent requests cannot leave a pair nobody asked + * for. A different repository clears the approved bots, which belong to one + * repository, and starts a fresh run. A different pull request in the same + * repository keeps the bots and starts a fresh run, because the round count, + * the time window and the claimed heads describe the previous pull request. */ async function enablePRAutomation({ trust, - repository, - pullNumber, + binding, ...key }: t.EnablePRAutomationParams): Promise { const PRAutomation = mongoose.models.PRAutomation; @@ -80,19 +100,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): await PRAutomation.updateOne( { ...filter, state: 'stopped' }, { - $set: { state: 'idle', round: 0, claimedHeads: [], ...(trust != null && { trust }) }, - $unset: { stopCode: '', startedAt: '', lastHeadSha: '' }, + $set: { ...RUN_RESET.$set, ...(trust != null && { trust }) }, + $unset: RUN_RESET.$unset, }, ); - if (repository != null) { + if (binding != null) { + await PRAutomation.updateOne( + { ...filter, repository: { $ne: binding.repository } }, + { + $set: { ...RUN_RESET.$set, ...binding, trustedBots: [] }, + $unset: RUN_RESET.$unset, + }, + { runValidators: true }, + ); await PRAutomation.updateOne( - { ...filter, repository: { $ne: repository } }, - { $set: { repository, trustedBots: [] } }, + { ...filter, repository: binding.repository, pullNumber: { $ne: binding.pullNumber } }, + { $set: { ...RUN_RESET.$set, ...binding }, $unset: RUN_RESET.$unset }, + { runValidators: true }, ); } - if (pullNumber != null) { - await PRAutomation.updateOne(filter, { $set: { pullNumber } }); - } const record = await getPRAutomation(key); if (record == null) { throw new Error('PR automation record missing after enable'); @@ -112,7 +138,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * deliveries racing for the same record cannot both pass the cap. A head that * any earlier round already claimed is rejected, so a delayed delivery of a * superseded commit cannot spend the budget. The persisted counter is what - * stops a new turn from resetting the cap. + * stops a new turn from resetting the cap. Each claim gets a `runId`, which + * the round must present to settle. * * A force-push back to an earlier SHA is rejected the same way; the user * restarts the automation to work on it. @@ -127,6 +154,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const PRAutomation = mongoose.models.PRAutomation; const filter = keyFilter(key); const cutoff = new Date(now.getTime() - maxMinutes * 60_000); + const runId = randomUUID(); /** The window opens at the first claim attempt on an active record, once. */ await PRAutomation.updateOne( @@ -144,14 +172,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): }, { $inc: { round: 1 }, - $set: { state: 'fixing', lastHeadSha: headSha }, + $set: { state: 'fixing', lastHeadSha: headSha, runId }, $push: { claimedHeads: headSha }, $unset: { stopCode: '' }, }, { new: true, select: PROJECTION }, ).lean(); if (claimed != null) { - return { ok: true, value: claimed }; + return { ok: true, value: { ...claimed, runId } }; } const current = await getPRAutomation(key); @@ -172,14 +200,16 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Reports where the round that owns the record ended up. The transition only - * applies while that round is still the current one, so a completion that is - * retried after a later round was claimed cannot move the newer round to - * `waiting` and open the way for an overlapping claim. Returns `null` when + * applies while that round is still the current one, identified by its + * `runId`, so a completion that is retried after a later round was claimed, or + * after the run was stopped, restarted or rebound, cannot move the newer run + * to `waiting` and open the way for an overlapping claim. Returns `null` when * nothing changed: no record, a stopped record, or a round that is not the * current one. */ async function settlePRAutomationRound({ round, + runId, state, ...key }: t.SettlePRAutomationRoundParams): Promise { @@ -190,7 +220,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( - { ...keyFilter(key), state: { $in: SETTLEABLE_STATES }, round }, + { ...keyFilter(key), state: { $in: SETTLEABLE_STATES }, round, runId }, { $set: { state } }, { new: true, select: PROJECTION }, ).lean(); @@ -206,9 +236,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): key: t.PRAutomationKey, stopCode: PRAutomationStopCode, ): Promise { - if (!PR_AUTOMATION_STOP_CODES.includes(stopCode)) { - throw new RangeError('A stopped PR automation needs a known stop code'); - } + assertStopCode(stopCode); const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( { ...keyFilter(key), state: { $ne: 'stopped' } }, @@ -217,6 +245,44 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ).lean(); } + /** + * Fences every record of a user, or of the listed conversations, without + * removing it. A deletion fences first so a webhook cannot claim a record + * whose conversation or account is going away, and removes the record only + * after the delete committed, so a failed delete keeps the stored state. + */ + async function stopPRAutomations( + userId: string, + stopCode: PRAutomationStopCode, + conversationIds?: string[], + ): Promise { + assertStopCode(stopCode); + const PRAutomation = mongoose.models.PRAutomation; + if (PRAutomation == null || conversationIds?.length === 0) { + return; + } + await PRAutomation.updateMany( + { + user: userId, + state: { $ne: 'stopped' }, + ...(conversationIds != null && { conversationId: { $in: conversationIds } }), + }, + { $set: { state: 'stopped', stopCode } }, + ); + } + + /** Removes every record of a user, or of the listed conversations. */ + async function deletePRAutomations(userId: string, conversationIds?: string[]): Promise { + const PRAutomation = mongoose.models.PRAutomation; + if (PRAutomation == null || conversationIds?.length === 0) { + return; + } + await PRAutomation.deleteMany({ + user: userId, + ...(conversationIds != null && { conversationId: { $in: conversationIds } }), + }); + } + async function setPRAutomationTrust( key: t.PRAutomationKey, trust: PRAutomationTrustLevel, @@ -285,6 +351,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): claimPRAutomationRound, settlePRAutomationRound, stopPRAutomation, + stopPRAutomations, + deletePRAutomations, setPRAutomationTrust, addPRAutomationBot, removePRAutomationBot, diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index b72e48ba2c0..a52e6a2f55d 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -2202,6 +2202,32 @@ describe('User Methods - Database Tests', () => { }); describe('PR automation cleanup on account deletion', () => { + it('keeps the records, stopped, when the account delete fails', async () => { + const user = await User.create({ email: 'delete-pr-fail@example.com', provider: 'local' }); + const id = user._id.toString(); + const PRAutomation = mongoose.models.PRAutomation; + await PRAutomation.create({ user: id, conversationId: 'chat-a' }); + const deletion = jest.spyOn(User, 'deleteOne').mockRejectedValueOnce(new Error('db down')); + + await expect(methods.deleteUserById(id)).rejects.toThrow('db down'); + deletion.mockRestore(); + + expect(await PRAutomation.findOne({ user: id }).lean()).toMatchObject({ + state: 'stopped', + stopCode: 'account_deleting', + }); + }); + + it('removes the records of an account that no longer exists', async () => { + const id = new mongoose.Types.ObjectId().toString(); + const PRAutomation = mongoose.models.PRAutomation; + await PRAutomation.create({ user: id, conversationId: 'chat-a' }); + + await expect(methods.deleteUserById(id)).resolves.toMatchObject({ deletedCount: 0 }); + + expect(await PRAutomation.countDocuments({ user: id })).toBe(0); + }); + it('removes the deleted user records and leaves other users alone', async () => { const user = await User.create({ email: 'delete-pr@example.com', provider: 'local' }); const id = user._id.toString(); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 80407af37ae..59357ff93ca 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -16,6 +16,7 @@ import type { } from '~/types'; import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types'; import type { CacheStore } from '~/types'; +import { createPRAutomationMethods } from './prAutomation'; import { evictAuthUserDocs } from '~/utils/eviction'; import { escapeRegExp } from '~/utils/string'; import { signPayload } from '~/crypto'; @@ -221,6 +222,8 @@ export function createUserMethods( return normalized; } + const prAutomation = createPRAutomationMethods(mongoose); + /** * Search for a single user based on partial data and return matching user document as plain object. * Email fields in searchCriteria are automatically normalized to lowercase for case-insensitive matching. @@ -643,8 +646,15 @@ export function createUserMethods( try { const User = mongoose.models.User; await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); - await mongoose.models.PRAutomation?.deleteMany({ user: userId }); + /** Fenced first so a webhook cannot claim a record for an account that is going away; + * removed only after the account delete committed. */ + await prAutomation.stopPRAutomations(userId, 'account_deleting'); const result = await User.deleteOne({ _id: userId }); + try { + await prAutomation.deletePRAutomations(userId); + } catch { + logger.warn('[deleteUserById] PR automation cleanup failed; the records stay stopped.'); + } if (result.deletedCount === 0) { return { deletedCount: 0, message: 'No user found with that ID.' }; } diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index f9580b6f2a9..4846dcb371c 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -56,6 +56,7 @@ const prAutomationSchema: Schema = new Schema( startedAt: { type: Date }, lastHeadSha: { type: String, maxlength: 64 }, claimedHeads: { type: [{ type: String, maxlength: 64 }], default: [] }, + runId: { type: String, maxlength: 64 }, }, { timestamps: true }, ); diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 80935e03dee..855c84a11f3 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -28,6 +28,12 @@ export interface IPRAutomation { startedAt?: Date; /** The head the latest round works on. */ lastHeadSha?: string; + /** + * Identifies the round that is running. Only that round may settle the record, so a + * completion from a run that was stopped, restarted or rebound to another pull request + * cannot move the run that replaced it. + */ + runId?: string; /** * Every head a round has been claimed for since the last restart. A delayed * delivery of an earlier head is rejected against this list. It holds at most @@ -45,15 +51,22 @@ export interface PRAutomationKey { conversationId: string; } +/** The pull request a conversation works on. The pair is always written together. */ +export interface PRAutomationBinding { + repository: string; + pullNumber: number; +} + export interface EnablePRAutomationParams extends PRAutomationKey { trust?: PRAutomationTrustLevel; - repository?: string; - pullNumber?: number; + binding?: PRAutomationBinding; } export interface SettlePRAutomationRoundParams extends PRAutomationKey { /** The round that is reporting. A completion for any other round is ignored. */ round: number; + /** The `runId` the claim returned. A completion for any other run is ignored. */ + runId: string; state: 'waiting' | 'needs_user'; } @@ -73,7 +86,7 @@ export type PRAutomationClaimErrorCode = | 'stale_head'; export type ClaimPRAutomationRoundResult = - | { ok: true; value: IPRAutomation } + | { ok: true; value: IPRAutomation & { runId: string } } | { ok: false; error: { code: PRAutomationClaimErrorCode } }; export type PRAutomationBotResult = From 6013eec59c24225636ca56a82d70be5f82747b35 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:56:23 +0200 Subject: [PATCH 04/25] fix: Bind PR Automation Claims to the Pull Request A round claim now names the pull request it is for and applies only while the record is still bound to it, so an event delivered after a rebind cannot spend the new pull request's budget. Enabling a record applies its reset and binding in one conditional write per case, so a stopped record stays stopped when the new binding is rejected. The unique index is ensured before the first write, because automatic index creation can be disabled. --- .../src/methods/conversation.spec.ts | 7 +- .../src/methods/prAutomation.spec.ts | 161 ++++++++++++------ .../data-schemas/src/methods/prAutomation.ts | 96 ++++++++--- .../data-schemas/src/types/prAutomation.ts | 7 + 4 files changed, 196 insertions(+), 75 deletions(-) diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 7ebe15b3bea..552db291893 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -3749,7 +3749,11 @@ describe('Conversation Operations', () => { const conversationId = uuidv4(); await Conversation.create({ conversationId, user, endpoint: EModelEndpoint.agents }); const automation = createPRAutomationMethods(mongoose); - await automation.enablePRAutomation({ userId: user, conversationId }); + await automation.enablePRAutomation({ + userId: user, + conversationId, + binding: { repository: 'acme/one', pullNumber: 1 }, + }); jest.spyOn(Conversation, 'deleteMany').mockRejectedValueOnce(new Error('delete failed')); await expect(deleteConvos(user, { conversationId })).rejects.toThrow('delete failed'); return { conversationId, automation }; @@ -3770,6 +3774,7 @@ describe('Conversation Operations', () => { const claimed = await automation.claimPRAutomationRound({ userId: user, conversationId, + binding: { repository: 'acme/one', pullNumber: 1 }, maxRounds: 5, maxMinutes: 60, headSha: 'a'.repeat(40), diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 3e72dcca604..94be15fb9f5 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -20,8 +20,16 @@ const pullOne = { repository: 'acme/one', pullNumber: 1 }; const pullTwo = { repository: 'acme/one', pullNumber: 2 }; const otherRepository = { repository: 'acme/two', pullNumber: 2 }; -const claim = (n: number, extra: { now?: Date } = {}) => - methods.claimPRAutomationRound({ ...key, ...limits, headSha: head(n), ...extra }); +const mismatch = { ok: false, error: { code: 'binding_mismatch' } }; +const enable = (binding = pullOne) => methods.enablePRAutomation({ ...key, binding }); +const claim = (n: number, extra: { now?: Date; binding?: typeof pullOne } = {}) => + methods.claimPRAutomationRound({ + ...key, + ...limits, + binding: pullOne, + headSha: head(n), + ...extra, + }); /** Claims a round that must succeed and returns the record it started. */ const startRound = async (n: number) => { const result = await claim(n); @@ -67,7 +75,7 @@ describe('getPRAutomation', () => { describe('enablePRAutomation', () => { test('creates an idle record on the narrowest trust level', async () => { - const record = await methods.enablePRAutomation(key); + const record = await enable(); expect(record).toMatchObject({ conversationId: 'convo-1', state: 'idle', @@ -78,24 +86,24 @@ describe('enablePRAutomation', () => { }); test('is idempotent and keeps one record per user and conversation', async () => { - await methods.enablePRAutomation(key); - await methods.enablePRAutomation(key); + await enable(); + await enable(); expect(await PRAutomation.countDocuments({ user: userId })).toBe(1); }); test('does not change an active record that is enabled again', async () => { - await methods.enablePRAutomation(key); + await enable(); await claim(1); - const again = await methods.enablePRAutomation(key); + const again = await enable(); expect(again).toMatchObject({ state: 'fixing', round: 1 }); }); test('restarts a stopped record with a fresh round counter and window', async () => { - await methods.enablePRAutomation(key); + await enable(); await claim(1); await methods.stopPRAutomation(key, 'user_stopped'); - const restarted = await methods.enablePRAutomation(key); + const restarted = await enable(); expect(restarted).toMatchObject({ state: 'idle', round: 0 }); expect(restarted.stopCode).toBeUndefined(); expect(restarted.startedAt).toBeUndefined(); @@ -103,15 +111,15 @@ describe('enablePRAutomation', () => { }); test('lets a restarted record claim a head it claimed before the stop', async () => { - await methods.enablePRAutomation(key); + await enable(); await claim(1); await methods.stopPRAutomation(key, 'user_stopped'); - await methods.enablePRAutomation(key); + await enable(); expect((await claim(1)).ok).toBe(true); }); test('keeps records of different users apart', async () => { - await methods.enablePRAutomation(key); + await enable(); expect(await methods.getPRAutomation({ ...key, userId: otherUserId })).toBeNull(); }); @@ -137,7 +145,40 @@ describe('enablePRAutomation', () => { }); }); +describe('index guarantees', () => { + test('builds the unique index before the first write when automatic indexing is off', async () => { + await PRAutomation.collection.dropIndexes(); + const fresh = createPRAutomationMethods(mongoose); + await fresh.enablePRAutomation({ ...key, binding: pullOne }); + const indexes = await PRAutomation.collection.indexes(); + expect( + indexes.some( + (index) => index.unique === true && index.key.user === 1 && index.key.conversationId === 1, + ), + ).toBe(true); + }); +}); + describe('binding a pull request', () => { + test('keeps a stopped record fenced when the new binding is rejected', async () => { + await enable(pullOne); + await methods.stopPRAutomation(key, 'user_stopped'); + + await expect(enable({ repository: 'acme/one', pullNumber: 0 })).rejects.toThrow(); + + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'stopped', + stopCode: 'user_stopped', + pullNumber: 1, + }); + }); + + test('revives a stopped record onto the pull request it is bound to now', async () => { + await enable(pullOne); + await methods.stopPRAutomation(key, 'user_stopped'); + expect(await enable(pullTwo)).toMatchObject({ state: 'idle', pullNumber: 2, round: 0 }); + }); + test('starts a fresh run when bound to another pull request in the same repository', async () => { await methods.enablePRAutomation({ ...key, binding: pullOne }); await startRound(1); @@ -154,7 +195,7 @@ describe('binding a pull request', () => { await methods.enablePRAutomation({ ...key, binding: pullOne }); await startRound(1); await methods.enablePRAutomation({ ...key, binding: pullTwo }); - expect((await claim(1)).ok).toBe(true); + expect((await claim(1, { binding: pullTwo })).ok).toBe(true); }); test('does not carry the old pull request round count into the new one', async () => { @@ -164,7 +205,7 @@ describe('binding a pull request', () => { await toWaiting(); } await methods.enablePRAutomation({ ...key, binding: pullTwo }); - expect((await claim(9)).ok).toBe(true); + expect((await claim(9, { binding: pullTwo })).ok).toBe(true); }); test('ignores a completion from the round of the pull request it replaced', async () => { @@ -185,7 +226,7 @@ describe('binding a pull request', () => { /** Guard, not a proven regression: the interleaving it protects against is timing dependent. */ test('never leaves a repository and pull request pair nobody asked for', async () => { - await methods.enablePRAutomation(key); + await enable(); const pairs = [pullOne, otherRepository]; await Promise.all( Array.from({ length: 24 }, (_, index) => @@ -202,7 +243,7 @@ describe('binding a pull request', () => { describe('claimPRAutomationRound', () => { test('starts a round, counts it and records the head', async () => { - await methods.enablePRAutomation(key); + await enable(); expect(await claim(1)).toMatchObject({ ok: true, value: { state: 'fixing', round: 1, lastHeadSha: head(1) }, @@ -210,7 +251,7 @@ describe('claimPRAutomationRound', () => { }); test('rejects a second delivery for a head that was already claimed', async () => { - await methods.enablePRAutomation(key); + await enable(); await claim(1); await toWaiting(); expect(await claim(1)).toEqual({ ok: false, error: { code: 'stale_head' } }); @@ -218,7 +259,7 @@ describe('claimPRAutomationRound', () => { }); test('rejects a delayed delivery of an earlier head after a later head was claimed', async () => { - await methods.enablePRAutomation(key); + await enable(); await claim(1); await toWaiting(); await claim(2); @@ -228,13 +269,13 @@ describe('claimPRAutomationRound', () => { }); test('rejects a claim while a round is already running', async () => { - await methods.enablePRAutomation(key); + await enable(); await claim(1); expect(await claim(2)).toEqual({ ok: false, error: { code: 'not_active' } }); }); test('stops at the round cap and never goes past it', async () => { - await methods.enablePRAutomation(key); + await enable(); for (let round = 1; round <= limits.maxRounds; round++) { expect((await claim(round)).ok).toBe(true); await toWaiting(); @@ -244,21 +285,21 @@ describe('claimPRAutomationRound', () => { }); test('lets exactly one of several concurrent deliveries claim the same head', async () => { - await methods.enablePRAutomation(key); + await enable(); const results = await Promise.all(Array.from({ length: 8 }, () => claim(1))); expect(results.filter((result) => result.ok)).toHaveLength(1); expect((await methods.getPRAutomation(key))?.round).toBe(1); }); test('never exceeds the cap under concurrent deliveries for different heads', async () => { - await methods.enablePRAutomation(key); + await enable(); const results = await Promise.all(Array.from({ length: 8 }, (_, index) => claim(index + 1))); expect(results.filter((result) => result.ok)).toHaveLength(1); expect((await methods.getPRAutomation(key))?.round).toBeLessThanOrEqual(limits.maxRounds); }); test('stops once the wall-clock window has passed', async () => { - await methods.enablePRAutomation(key); + await enable(); const start = new Date('2026-01-01T00:00:00Z'); await claim(1, { now: start }); await toWaiting(); @@ -272,22 +313,46 @@ describe('claimPRAutomationRound', () => { }); test('rejects a claim on a stopped record', async () => { - await methods.enablePRAutomation(key); + await enable(); await methods.stopPRAutomation(key, 'user_stopped'); expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_active' } }); }); + + test('rejects a delivery for the pull request the record was rebound away from', async () => { + await enable(pullOne); + await enable(pullTwo); + expect(await claim(1, { binding: pullOne })).toEqual(mismatch); + expect((await methods.getPRAutomation(key))?.round).toBe(0); + }); + + test('rejects a delivery for a repository the record is not bound to', async () => { + await enable(pullOne); + expect(await claim(1, { binding: otherRepository })).toEqual(mismatch); + }); + + test('rejects a claim on a record that is bound to no pull request', async () => { + await methods.enablePRAutomation(key); + expect(await claim(1)).toEqual(mismatch); + }); + + test('does not open the time window for a delivery it rejected', async () => { + await enable(pullOne); + await enable(pullTwo); + await claim(1, { binding: pullOne }); + expect((await methods.getPRAutomation(key))?.startedAt).toBeUndefined(); + }); }); describe('settlePRAutomationRound', () => { test('settles the round that owns it, including from needs_user', async () => { - await methods.enablePRAutomation(key); + await enable(); const { runId } = await startRound(1); expect(await settle(1, runId, 'needs_user')).toMatchObject({ state: 'needs_user', round: 1 }); expect(await settle(1, runId, 'waiting')).toMatchObject({ state: 'waiting', round: 1 }); }); test('ignores a completion that belongs to an earlier round', async () => { - await methods.enablePRAutomation(key); + await enable(); const { runId } = await startRound(1); await settle(1, runId); await claim(2); @@ -297,7 +362,7 @@ describe('settlePRAutomationRound', () => { }); test('does not let a stale completion open the way for an overlapping claim', async () => { - await methods.enablePRAutomation(key); + await enable(); const { runId } = await startRound(1); await settle(1, runId); await claim(2); @@ -306,10 +371,10 @@ describe('settlePRAutomationRound', () => { }); test('ignores a completion from a run that was stopped and restarted', async () => { - await methods.enablePRAutomation(key); + await enable(); const previous = await startRound(1); await methods.stopPRAutomation(key, 'user_stopped'); - await methods.enablePRAutomation(key); + await enable(); await startRound(1); expect(await settle(1, previous.runId, 'needs_user')).toBeNull(); @@ -317,10 +382,10 @@ describe('settlePRAutomationRound', () => { }); test('ignores a completion from a record that was disabled and enabled again', async () => { - await methods.enablePRAutomation(key); + await enable(); const previous = await startRound(1); await methods.disablePRAutomation(key); - await methods.enablePRAutomation(key); + await enable(); await startRound(1); expect(await settle(1, previous.runId, 'needs_user')).toBeNull(); @@ -328,13 +393,13 @@ describe('settlePRAutomationRound', () => { }); test('does not settle a record that never started a round', async () => { - await methods.enablePRAutomation(key); + await enable(); expect(await settle(0, 'no-run')).toBeNull(); expect((await methods.getPRAutomation(key))?.state).toBe('idle'); }); test('does not revive a stopped record', async () => { - await methods.enablePRAutomation(key); + await enable(); const { runId } = await startRound(1); await methods.stopPRAutomation(key, 'user_stopped'); expect(await settle(1, runId)).toBeNull(); @@ -348,7 +413,7 @@ describe('settlePRAutomationRound', () => { describe('stopPRAutomation', () => { test('records the stop code', async () => { - await methods.enablePRAutomation(key); + await enable(); expect(await methods.stopPRAutomation(key, 'round_cap')).toMatchObject({ state: 'stopped', stopCode: 'round_cap', @@ -356,7 +421,7 @@ describe('stopPRAutomation', () => { }); test('stops a running round without waiting for it to settle', async () => { - await methods.enablePRAutomation(key); + await enable(); await claim(1); expect(await methods.stopPRAutomation(key, 'user_stopped')).toMatchObject({ state: 'stopped', @@ -365,14 +430,14 @@ describe('stopPRAutomation', () => { }); test('keeps the first stop code when it is stopped again', async () => { - await methods.enablePRAutomation(key); + await enable(); await methods.stopPRAutomation(key, 'round_cap'); expect(await methods.stopPRAutomation(key, 'user_stopped')).toBeNull(); expect((await methods.getPRAutomation(key))?.stopCode).toBe('round_cap'); }); test('rejects a code outside the stop code list', async () => { - await methods.enablePRAutomation(key); + await enable(); await expect( methods.stopPRAutomation(key, 'because' as unknown as PRAutomationStopCode), ).rejects.toThrow(); @@ -385,13 +450,13 @@ describe('stopPRAutomation', () => { describe('setPRAutomationTrust', () => { test('changes the trust level', async () => { - await methods.enablePRAutomation(key); + await enable(); const updated = await methods.setPRAutomationTrust(key, 'collaborators'); expect(updated?.trust).toBe('collaborators'); }); test('rejects a level outside the enum', async () => { - await methods.enablePRAutomation(key); + await enable(); await expect( methods.setPRAutomationTrust(key, 'everyone' as unknown as 'anyone'), ).rejects.toThrow(); @@ -400,7 +465,7 @@ describe('setPRAutomationTrust', () => { describe('approved bots', () => { test('adds a bot by numeric id', async () => { - await methods.enablePRAutomation(key); + await enable(); expect(await addBot(101, 'review-bot[bot]')).toMatchObject({ ok: true, value: { trustedBots: [{ id: 101, login: 'review-bot[bot]' }] }, @@ -408,7 +473,7 @@ describe('approved bots', () => { }); test('is idempotent by id even when the login was renamed', async () => { - await methods.enablePRAutomation(key); + await enable(); await addBot(101, 'old-name[bot]'); expect((await addBot(101, 'new-name[bot]')).ok).toBe(true); const record = await methods.getPRAutomation(key); @@ -416,7 +481,7 @@ describe('approved bots', () => { }); test('refuses a bot beyond the limit it is given, with a stable code', async () => { - await methods.enablePRAutomation(key); + await enable(); for (let id = 1; id <= maxBots; id++) { expect({ id, ok: (await addBot(id)).ok }).toEqual({ id, ok: true }); } @@ -424,7 +489,7 @@ describe('approved bots', () => { }); test('accepts a bot that is already approved when the list is full', async () => { - await methods.enablePRAutomation(key); + await enable(); for (let id = 1; id <= maxBots; id++) { await addBot(id); } @@ -432,7 +497,7 @@ describe('approved bots', () => { }); test('admits more bots once a higher limit is passed', async () => { - await methods.enablePRAutomation(key); + await enable(); for (let id = 1; id <= maxBots; id++) { await addBot(id); } @@ -440,7 +505,7 @@ describe('approved bots', () => { }); test('rejects a limit below one instead of storing without a cap', async () => { - await methods.enablePRAutomation(key); + await enable(); await expect(addBot(1, undefined, 0)).rejects.toThrow(RangeError); }); @@ -449,7 +514,7 @@ describe('approved bots', () => { }); test('removes a bot by id and leaves the others', async () => { - await methods.enablePRAutomation(key); + await enable(); await addBot(1); await addBot(2); const record = await methods.removePRAutomationBot(key, 1); @@ -457,7 +522,7 @@ describe('approved bots', () => { }); test('keeps each conversation allowlist separate', async () => { - await methods.enablePRAutomation(key); + await enable(); await methods.enablePRAutomation({ userId, conversationId: 'convo-2' }); await addBot(101); const other = await methods.getPRAutomation({ userId, conversationId: 'convo-2' }); @@ -467,7 +532,7 @@ describe('approved bots', () => { describe('disablePRAutomation', () => { test('removes the record and reports it', async () => { - await methods.enablePRAutomation(key); + await enable(); expect(await methods.disablePRAutomation(key)).toEqual({ removed: true }); expect(await methods.getPRAutomation(key)).toBeNull(); }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 2dcf7fd5a7b..3f7d7259d63 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -4,6 +4,7 @@ import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationState } from 'librechat-data-provider'; import type * as t from '~/types/prAutomation'; +import { createIndexesWithRetry } from '~/utils/retry'; const PROJECTION = '-_id -__v'; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; @@ -46,6 +47,24 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ) => Promise; removePRAutomationBot: (key: t.PRAutomationKey, botId: number) => Promise; } { + let indexesPromise: Promise | null = null; + + /** + * The unique `{ user, conversationId }` index is what keeps concurrent first-time enables + * from inserting two records that are then claimed independently and exceed the round cap. + * With `MONGO_AUTO_INDEX=false` the schema declaration alone never builds it, so it is + * ensured before the first write. + */ + function ensureIndexes(): Promise { + if (!indexesPromise) { + indexesPromise = createIndexesWithRetry(mongoose.models.PRAutomation).catch((error) => { + indexesPromise = null; + throw error; + }); + } + return indexesPromise; + } + const keyFilter = ({ userId, conversationId }: t.PRAutomationKey) => ({ user: userId, conversationId, @@ -69,20 +88,28 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * a fresh run: the round counter, the time window, the claimed heads and the * run identity reset, which they could not do from a webhook-triggered turn. * - * `binding` names the pull request. The repository and the number are one - * conditional write, so concurrent requests cannot leave a pair nobody asked - * for. A different repository clears the approved bots, which belong to one - * repository, and starts a fresh run. A different pull request in the same - * repository keeps the bots and starts a fresh run, because the round count, - * the time window and the claimed heads describe the previous pull request. + * `binding` names the pull request. Each case below is one conditional write that + * carries the whole pair and the run reset together, so concurrent requests cannot + * leave a pair nobody asked for, and a stopped record is revived only by the write + * that binds it: a binding that is rejected leaves it stopped. A different repository + * clears the approved bots, which belong to one repository, and starts a fresh run. + * A different pull request in the same repository keeps the bots and starts a fresh + * run, because the round count, the time window and the claimed heads describe the + * previous pull request. */ async function enablePRAutomation({ trust, binding, ...key }: t.EnablePRAutomationParams): Promise { + await ensureIndexes(); const PRAutomation = mongoose.models.PRAutomation; const filter = keyFilter(key); + const reset = (fields: Record) => ({ + $set: { ...RUN_RESET.$set, ...fields }, + $unset: RUN_RESET.$unset, + }); + const options = { runValidators: true }; await PRAutomation.updateOne( filter, { @@ -93,30 +120,41 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): trustedBots: [], claimedHeads: [], trust: trust ?? 'approvedBots', + ...binding, }, }, - { upsert: true, runValidators: true }, - ); - await PRAutomation.updateOne( - { ...filter, state: 'stopped' }, - { - $set: { ...RUN_RESET.$set, ...(trust != null && { trust }) }, - $unset: RUN_RESET.$unset, - }, + { upsert: true, ...options }, ); - if (binding != null) { + const revive = trust != null ? { trust } : {}; + if (binding == null) { + await PRAutomation.updateOne({ ...filter, state: 'stopped' }, reset(revive), options); + } else { + const { repository } = binding; + const otherRepository = { repository: { $ne: repository } }; await PRAutomation.updateOne( - { ...filter, repository: { $ne: binding.repository } }, - { - $set: { ...RUN_RESET.$set, ...binding, trustedBots: [] }, - $unset: RUN_RESET.$unset, - }, - { runValidators: true }, + { ...filter, state: 'stopped', ...otherRepository }, + reset({ ...revive, ...binding, trustedBots: [] }), + options, + ); + await PRAutomation.updateOne( + { ...filter, state: 'stopped', repository }, + reset({ ...revive, ...binding }), + options, ); await PRAutomation.updateOne( - { ...filter, repository: binding.repository, pullNumber: { $ne: binding.pullNumber } }, - { $set: { ...RUN_RESET.$set, ...binding }, $unset: RUN_RESET.$unset }, - { runValidators: true }, + { ...filter, state: { $ne: 'stopped' }, ...otherRepository }, + reset({ ...binding, trustedBots: [] }), + options, + ); + await PRAutomation.updateOne( + { + ...filter, + state: { $ne: 'stopped' }, + repository, + pullNumber: { $ne: binding.pullNumber }, + }, + reset({ ...binding }), + options, ); } const record = await getPRAutomation(key); @@ -139,12 +177,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * any earlier round already claimed is rejected, so a delayed delivery of a * superseded commit cannot spend the budget. The persisted counter is what * stops a new turn from resetting the cap. Each claim gets a `runId`, which - * the round must present to settle. + * the round must present to settle. The delivery names the pull request it is + * for, and the claim applies only while the record is still bound to it, so an + * event that arrives after a rebind cannot spend the new pull request's budget. * * A force-push back to an earlier SHA is rejected the same way; the user * restarts the automation to work on it. */ async function claimPRAutomationRound({ + binding, maxRounds, maxMinutes, headSha, @@ -152,7 +193,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ...key }: t.ClaimPRAutomationRoundParams): Promise { const PRAutomation = mongoose.models.PRAutomation; - const filter = keyFilter(key); + const filter = { ...keyFilter(key), ...binding }; const cutoff = new Date(now.getTime() - maxMinutes * 60_000); const runId = randomUUID(); @@ -186,6 +227,9 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (current == null) { return { ok: false, error: { code: 'not_found' } }; } + if (current.repository !== binding.repository || current.pullNumber !== binding.pullNumber) { + return { ok: false, error: { code: 'binding_mismatch' } }; + } if (!CLAIMABLE_STATES.includes(current.state)) { return { ok: false, error: { code: 'not_active' } }; } diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 855c84a11f3..6f2ff58b79e 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -71,6 +71,12 @@ export interface SettlePRAutomationRoundParams extends PRAutomationKey { } export interface ClaimPRAutomationRoundParams extends PRAutomationKey { + /** + * The pull request the delivery is for. A delivery for a pull request the record is no + * longer bound to cannot claim a round, so an event that arrives after a rebind is rejected + * instead of spending the new pull request's budget. + */ + binding: PRAutomationBinding; maxRounds: number; maxMinutes: number; /** The head the round will work on; recorded so a stale event can be recognized. */ @@ -81,6 +87,7 @@ export interface ClaimPRAutomationRoundParams extends PRAutomationKey { export type PRAutomationClaimErrorCode = | 'not_found' | 'not_active' + | 'binding_mismatch' | 'round_cap' | 'time_cap' | 'stale_head'; From 4bc5a9714c1dc94c545e16cd525490930160c638 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:47:08 +0200 Subject: [PATCH 05/25] fix: Keep PR Automation Fences Through Restart, Rebind and Retention Enabling a record no longer clears a conversation or account deletion fence, and a deletion fence replaces an earlier user stop. A bot approval names the repository it was authorized for and applies only while the record is still bound to it. A round claim removes the record instead of starting work when its conversation was removed by the retention index or is past its retention date. --- .../src/methods/prAutomation.spec.ts | 170 +++++++++++++++++- .../data-schemas/src/methods/prAutomation.ts | 42 ++++- .../data-schemas/src/types/prAutomation.ts | 3 +- 3 files changed, 203 insertions(+), 12 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 94be15fb9f5..5769d77d55e 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -46,8 +46,15 @@ const toWaiting = () => { user: userId, conversationId: key.conversationId }, { $set: { state: 'waiting' } }, ); -const addBot = (id: number, login?: string, limit = maxBots) => - methods.addPRAutomationBot(key, login == null ? { id } : { id, login }, limit); +const addBot = (id: number, login?: string, limit = maxBots, repository = 'acme/one') => + methods.addPRAutomationBot(key, login == null ? { id } : { id, login }, limit, repository); +const seedConversation = (fields: Record = {}) => + mongoose.models.Conversation.create({ + conversationId: key.conversationId, + user: userId, + endpoint: 'agents', + ...fields, + }); beforeAll(async () => { mongoServer = await MongoMemoryServer.create(); @@ -65,6 +72,8 @@ afterAll(async () => { beforeEach(async () => { await PRAutomation.deleteMany({}); + await mongoose.models.Conversation.deleteMany({}); + await seedConversation(); }); describe('getPRAutomation', () => { @@ -130,11 +139,10 @@ describe('enablePRAutomation', () => { expect(rebound).toMatchObject({ ...otherRepository, trustedBots: [] }); }); - test('does not carry a bot approved before any repository was bound', async () => { + test('refuses a bot approved before any repository was bound', async () => { await methods.enablePRAutomation(key); - await addBot(101); - const bound = await methods.enablePRAutomation({ ...key, binding: pullOne }); - expect(bound.trustedBots).toEqual([]); + expect(await addBot(101)).toEqual({ ok: false, error: { code: 'binding_mismatch' } }); + expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([]); }); test('keeps approved bots when the same repository is bound to another pull request', async () => { @@ -145,6 +153,79 @@ describe('enablePRAutomation', () => { }); }); +describe('a record stopped by a deletion fence', () => { + const deletionCodes = ['conversation_deleting', 'account_deleting'] as const; + + test.each(deletionCodes)( + 'is not revived by enabling the same pull request (%s)', + async (code) => { + await enable(pullOne); + await methods.stopPRAutomation(key, code); + expect(await enable(pullOne)).toMatchObject({ state: 'stopped', stopCode: code }); + }, + ); + + test.each(deletionCodes)( + 'is not revived by another pull request in the repository (%s)', + async (code) => { + await enable(pullOne); + await methods.stopPRAutomation(key, code); + expect(await enable(pullTwo)).toMatchObject({ + state: 'stopped', + stopCode: code, + pullNumber: 1, + }); + }, + ); + + test.each(deletionCodes)('is not revived or rebound to another repository (%s)', async (code) => { + await enable(pullOne); + await methods.stopPRAutomation(key, code); + expect(await enable(otherRepository)).toMatchObject({ + state: 'stopped', + stopCode: code, + repository: 'acme/one', + }); + }); + + test.each(deletionCodes)('is not revived without a binding (%s)', async (code) => { + await enable(pullOne); + await methods.stopPRAutomation(key, code); + expect(await methods.enablePRAutomation(key)).toMatchObject({ + state: 'stopped', + stopCode: code, + }); + }); + + test.each(deletionCodes)('still refuses a claim (%s)', async (code) => { + await enable(pullOne); + await methods.stopPRAutomation(key, code); + await enable(pullOne); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_active' } }); + }); + + test.each(deletionCodes)('replaces a user stop when a deletion begins (%s)', async (code) => { + await enable(pullOne); + await methods.stopPRAutomation(key, 'user_stopped'); + await methods.stopPRAutomations(userId, code, [key.conversationId]); + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'stopped', stopCode: code }); + expect(await enable(pullOne)).toMatchObject({ state: 'stopped', stopCode: code }); + }); + + test('keeps a record of another conversation restartable', async () => { + await enable(pullOne); + await methods.stopPRAutomation(key, 'user_stopped'); + await methods.stopPRAutomations(userId, 'conversation_deleting', ['another-convo']); + expect(await methods.getPRAutomation(key)).toMatchObject({ stopCode: 'user_stopped' }); + }); + + test('a user stop is still restartable', async () => { + await enable(pullOne); + await methods.stopPRAutomation(key, 'user_stopped'); + expect(await enable(pullOne)).toMatchObject({ state: 'idle' }); + }); +}); + describe('index guarantees', () => { test('builds the unique index before the first write when automatic indexing is off', async () => { await PRAutomation.collection.dropIndexes(); @@ -343,6 +424,58 @@ describe('claimPRAutomationRound', () => { }); }); +describe('claiming for a conversation that no longer exists', () => { + const gone = { ok: false, error: { code: 'conversation_gone' } }; + + test('rejects a claim once the conversation was removed by the retention index', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + expect(await claim(1)).toEqual(gone); + }); + + test('removes the record so nothing is left to claim later', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + await claim(1); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('rejects a claim for a conversation that expired but is not yet purged', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + await seedConversation({ expiredAt: new Date(Date.now() - 60_000) }); + expect(await claim(1)).toEqual(gone); + }); + + test('accepts a claim while the conversation retention date is still ahead', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + await seedConversation({ expiredAt: new Date(Date.now() + 3_600_000) }); + expect((await claim(1)).ok).toBe(true); + }); + + test('does not spend a round for a conversation that is gone', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + await claim(1); + await enable(); + await seedConversation(); + expect(await startRound(1)).toMatchObject({ round: 1 }); + }); + + test('keeps reporting not_found when there is no record at all', async () => { + await mongoose.models.Conversation.deleteMany({}); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_found' } }); + }); + + test('only looks at the conversation of the same owner', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + await seedConversation({ user: otherUserId }); + expect(await claim(1)).toEqual(gone); + }); +}); + describe('settlePRAutomationRound', () => { test('settles the round that owns it, including from needs_user', async () => { await enable(); @@ -513,6 +646,31 @@ describe('approved bots', () => { expect(await addBot(101)).toEqual({ ok: false, error: { code: 'not_found' } }); }); + test('refuses a bot approved for a repository the record is not bound to', async () => { + await enable(pullOne); + expect(await addBot(101, undefined, maxBots, 'acme/two')).toEqual({ + ok: false, + error: { code: 'binding_mismatch' }, + }); + expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([]); + }); + + test('refuses an approval that was authorized before the record was rebound', async () => { + await enable(pullOne); + await enable(otherRepository); + expect(await addBot(101, undefined, maxBots, 'acme/one')).toEqual({ + ok: false, + error: { code: 'binding_mismatch' }, + }); + expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([]); + }); + + test('still treats an already approved bot as success for the bound repository', async () => { + await enable(pullOne); + await addBot(101); + expect((await addBot(101)).ok).toBe(true); + }); + test('removes a bot by id and leaves the others', async () => { await enable(); await addBot(1); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 3f7d7259d63..bfdd9d96da2 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -4,9 +4,16 @@ import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationState } from 'librechat-data-provider'; import type * as t from '~/types/prAutomation'; +import { activeExpirationFilter } from '~/utils/retention'; import { createIndexesWithRetry } from '~/utils/retry'; const PROJECTION = '-_id -__v'; +/** + * A deletion fence is not a user decision. Enabling must never clear it, or a webhook could + * claim work for a conversation or account that is being removed. + */ +const DELETION_STOP_CODES: PRAutomationStopCode[] = ['conversation_deleting', 'account_deleting']; +const RESTARTABLE = { state: 'stopped', stopCode: { $nin: DELETION_STOP_CODES } }; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; @@ -44,6 +51,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): key: t.PRAutomationKey, bot: t.IPRAutomationBot, maxBots: number, + repository: string, ) => Promise; removePRAutomationBot: (key: t.PRAutomationKey, botId: number) => Promise; } { @@ -127,17 +135,17 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ); const revive = trust != null ? { trust } : {}; if (binding == null) { - await PRAutomation.updateOne({ ...filter, state: 'stopped' }, reset(revive), options); + await PRAutomation.updateOne({ ...filter, ...RESTARTABLE }, reset(revive), options); } else { const { repository } = binding; const otherRepository = { repository: { $ne: repository } }; await PRAutomation.updateOne( - { ...filter, state: 'stopped', ...otherRepository }, + { ...filter, ...RESTARTABLE, ...otherRepository }, reset({ ...revive, ...binding, trustedBots: [] }), options, ); await PRAutomation.updateOne( - { ...filter, state: 'stopped', repository }, + { ...filter, ...RESTARTABLE, repository }, reset({ ...revive, ...binding }), options, ); @@ -197,6 +205,22 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const cutoff = new Date(now.getTime() - maxMinutes * 60_000); const runId = randomUUID(); + /** + * Conversations leave through the retention index without passing `deleteConvos`, so + * the record cannot rely on that cascade. A claim for a conversation that is gone or + * past its retention date removes the record instead of starting work for it. + */ + const conversation = await mongoose.models.Conversation.exists({ + user: key.userId, + conversationId: key.conversationId, + ...activeExpirationFilter(), + }); + if (conversation == null) { + const removed = await PRAutomation.deleteOne(keyFilter(key)); + const code = removed.deletedCount > 0 ? 'conversation_gone' : 'not_found'; + return { ok: false, error: { code } }; + } + /** The window opens at the first claim attempt on an active record, once. */ await PRAutomation.updateOne( { ...filter, state: { $in: CLAIMABLE_STATES }, startedAt: { $exists: false } }, @@ -308,7 +332,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): await PRAutomation.updateMany( { user: userId, - state: { $ne: 'stopped' }, + /** A user stop is overwritten, because only a deletion code resists a restart. */ + stopCode: { $nin: DELETION_STOP_CODES }, ...(conversationIds != null && { conversationId: { $in: conversationIds } }), }, { $set: { state: 'stopped', stopCode } }, @@ -342,12 +367,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Idempotent by numeric id. The limit is the configured `maxBots`, resolved by * the caller, and cannot exceed the schema ceiling. A login is stored for - * display only. + * display only. The approval names the repository it was authorized against and + * applies only while the record is still bound to it, so an approval that + * overlaps a rebind cannot add a bot to a repository nobody approved it for. */ async function addPRAutomationBot( key: t.PRAutomationKey, bot: t.IPRAutomationBot, maxBots: number, + repository: string, ): Promise { if (!Number.isInteger(maxBots) || maxBots < 1 || maxBots > MAX_PR_AUTOMATION_BOTS) { throw new RangeError(`maxBots must be an integer from 1 to ${MAX_PR_AUTOMATION_BOTS}`); @@ -356,6 +384,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const updated = await PRAutomation.findOneAndUpdate( { ...keyFilter(key), + repository, 'trustedBots.id': { $ne: bot.id }, [`trustedBots.${maxBots - 1}`]: { $exists: false }, }, @@ -370,6 +399,9 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (current == null) { return { ok: false, error: { code: 'not_found' } }; } + if (current.repository !== repository) { + return { ok: false, error: { code: 'binding_mismatch' } }; + } if (current.trustedBots.some((existing) => existing.id === bot.id)) { return { ok: true, value: current }; } diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 6f2ff58b79e..c61c5a8f784 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -88,6 +88,7 @@ export type PRAutomationClaimErrorCode = | 'not_found' | 'not_active' | 'binding_mismatch' + | 'conversation_gone' | 'round_cap' | 'time_cap' | 'stale_head'; @@ -98,4 +99,4 @@ export type ClaimPRAutomationRoundResult = export type PRAutomationBotResult = | { ok: true; value: IPRAutomation } - | { ok: false; error: { code: 'not_found' | 'bot_limit' } }; + | { ok: false; error: { code: 'not_found' | 'bot_limit' | 'binding_mismatch' } }; From 08d9a2ac3afad2b660c3ffd9f494f40097163713 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:08:16 +0200 Subject: [PATCH 06/25] fix: Fence PR Automation Stops, Removals and Time Window An event-driven stop names the pull request it is for and applies only while the record is still bound to it, while a user stop stays unconditional. A bot removal names its repository like an approval does. Disabling no longer removes a deletion fence. The time window opens in the same atomic write as the first claimed round. Account deletion removes the records before the account so a failed cleanup is retried instead of being swallowed. --- .../src/methods/prAutomation.spec.ts | 136 ++++++++++++++++-- .../data-schemas/src/methods/prAutomation.ts | 65 ++++++--- .../src/methods/user.methods.spec.ts | 46 ++++-- packages/data-schemas/src/methods/user.ts | 12 +- 4 files changed, 215 insertions(+), 44 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 5769d77d55e..bfa619150ac 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -160,7 +160,7 @@ describe('a record stopped by a deletion fence', () => { 'is not revived by enabling the same pull request (%s)', async (code) => { await enable(pullOne); - await methods.stopPRAutomation(key, code); + await methods.stopPRAutomation(key, code, pullOne); expect(await enable(pullOne)).toMatchObject({ state: 'stopped', stopCode: code }); }, ); @@ -169,7 +169,7 @@ describe('a record stopped by a deletion fence', () => { 'is not revived by another pull request in the repository (%s)', async (code) => { await enable(pullOne); - await methods.stopPRAutomation(key, code); + await methods.stopPRAutomation(key, code, pullOne); expect(await enable(pullTwo)).toMatchObject({ state: 'stopped', stopCode: code, @@ -180,7 +180,7 @@ describe('a record stopped by a deletion fence', () => { test.each(deletionCodes)('is not revived or rebound to another repository (%s)', async (code) => { await enable(pullOne); - await methods.stopPRAutomation(key, code); + await methods.stopPRAutomation(key, code, pullOne); expect(await enable(otherRepository)).toMatchObject({ state: 'stopped', stopCode: code, @@ -190,7 +190,7 @@ describe('a record stopped by a deletion fence', () => { test.each(deletionCodes)('is not revived without a binding (%s)', async (code) => { await enable(pullOne); - await methods.stopPRAutomation(key, code); + await methods.stopPRAutomation(key, code, pullOne); expect(await methods.enablePRAutomation(key)).toMatchObject({ state: 'stopped', stopCode: code, @@ -199,7 +199,7 @@ describe('a record stopped by a deletion fence', () => { test.each(deletionCodes)('still refuses a claim (%s)', async (code) => { await enable(pullOne); - await methods.stopPRAutomation(key, code); + await methods.stopPRAutomation(key, code, pullOne); await enable(pullOne); expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_active' } }); }); @@ -416,6 +416,35 @@ describe('claimPRAutomationRound', () => { expect(await claim(1)).toEqual(mismatch); }); + test('does not open the time window when the claim is refused', async () => { + await enable(pullOne); + const refused = await methods.claimPRAutomationRound({ + ...key, + binding: pullOne, + maxRounds: 0, + maxMinutes: limits.maxMinutes, + headSha: head(1), + }); + expect(refused).toEqual({ ok: false, error: { code: 'round_cap' } }); + expect((await methods.getPRAutomation(key))?.startedAt).toBeUndefined(); + }); + + test('opens the time window with the first round that is claimed', async () => { + await enable(pullOne); + const start = new Date('2026-01-01T00:00:00Z'); + await claim(1, { now: start }); + expect((await methods.getPRAutomation(key))?.startedAt).toEqual(start); + }); + + test('keeps the first start when a later round is claimed', async () => { + await enable(pullOne); + const start = new Date('2026-01-01T00:00:00Z'); + await claim(1, { now: start }); + await toWaiting(); + await claim(2, { now: new Date(start.getTime() + 10 * 60_000) }); + expect((await methods.getPRAutomation(key))?.startedAt).toEqual(start); + }); + test('does not open the time window for a delivery it rejected', async () => { await enable(pullOne); await enable(pullTwo); @@ -547,7 +576,7 @@ describe('settlePRAutomationRound', () => { describe('stopPRAutomation', () => { test('records the stop code', async () => { await enable(); - expect(await methods.stopPRAutomation(key, 'round_cap')).toMatchObject({ + expect(await methods.stopPRAutomation(key, 'round_cap', pullOne)).toMatchObject({ state: 'stopped', stopCode: 'round_cap', }); @@ -564,7 +593,7 @@ describe('stopPRAutomation', () => { test('keeps the first stop code when it is stopped again', async () => { await enable(); - await methods.stopPRAutomation(key, 'round_cap'); + await methods.stopPRAutomation(key, 'round_cap', pullOne); expect(await methods.stopPRAutomation(key, 'user_stopped')).toBeNull(); expect((await methods.getPRAutomation(key))?.stopCode).toBe('round_cap'); }); @@ -572,13 +601,56 @@ describe('stopPRAutomation', () => { test('rejects a code outside the stop code list', async () => { await enable(); await expect( - methods.stopPRAutomation(key, 'because' as unknown as PRAutomationStopCode), + methods.stopPRAutomation(key, 'because' as unknown as PRAutomationStopCode, pullOne), ).rejects.toThrow(); }); test('returns null for a conversation with no record', async () => { expect(await methods.stopPRAutomation(key, 'user_stopped')).toBeNull(); }); + + test('ignores an event-driven stop for a pull request the record was rebound away from', async () => { + await enable(pullOne); + await enable(pullTwo); + expect(await methods.stopPRAutomation(key, 'pull_request_closed', pullOne)).toBeNull(); + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'idle', pullNumber: 2 }); + }); + + test('ignores an event-driven stop for another repository', async () => { + await enable(pullOne); + expect(await methods.stopPRAutomation(key, 'pull_request_closed', otherRepository)).toBeNull(); + expect((await methods.getPRAutomation(key))?.state).toBe('idle'); + }); + + test('applies an event-driven stop for the pull request the record is bound to', async () => { + await enable(pullOne); + expect(await methods.stopPRAutomation(key, 'pull_request_closed', pullOne)).toMatchObject({ + state: 'stopped', + stopCode: 'pull_request_closed', + }); + }); + + test('refuses an event-driven stop that names no pull request', async () => { + await enable(pullOne); + await expect( + ( + methods.stopPRAutomation as ( + target: typeof key, + code: PRAutomationStopCode, + ) => Promise + )(key, 'pull_request_closed'), + ).rejects.toThrow(RangeError); + expect((await methods.getPRAutomation(key))?.state).toBe('idle'); + }); + + test('lets a user stop win even after the record was rebound', async () => { + await enable(pullOne); + await enable(pullTwo); + expect(await methods.stopPRAutomation(key, 'user_stopped')).toMatchObject({ + state: 'stopped', + stopCode: 'user_stopped', + }); + }); }); describe('setPRAutomationTrust', () => { @@ -675,10 +747,27 @@ describe('approved bots', () => { await enable(); await addBot(1); await addBot(2); - const record = await methods.removePRAutomationBot(key, 1); + const record = await methods.removePRAutomationBot(key, 1, 'acme/one'); expect(record?.trustedBots).toEqual([{ id: 2 }]); }); + test('keeps an approval made for another repository when a removal is delayed', async () => { + await enable(pullOne); + await addBot(7); + await enable(otherRepository); + await addBot(7, undefined, maxBots, 'acme/two'); + const record = await methods.removePRAutomationBot(key, 7, 'acme/one'); + expect(record).toBeNull(); + expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([{ id: 7 }]); + }); + + test('removes an approval for the repository the record is bound to', async () => { + await enable(pullOne); + await addBot(7); + const record = await methods.removePRAutomationBot(key, 7, 'acme/one'); + expect(record?.trustedBots).toEqual([]); + }); + test('keeps each conversation allowlist separate', async () => { await enable(); await methods.enablePRAutomation({ userId, conversationId: 'convo-2' }); @@ -698,4 +787,33 @@ describe('disablePRAutomation', () => { test('reports nothing removed when there was no record', async () => { expect(await methods.disablePRAutomation(key)).toEqual({ removed: false }); }); + + test.each(['conversation_deleting', 'account_deleting'] as const)( + 'does not remove a deletion fence (%s)', + async (code) => { + await enable(pullOne); + await methods.stopPRAutomations(userId, code, [key.conversationId]); + expect(await methods.disablePRAutomation(key)).toEqual({ removed: false }); + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'stopped', + stopCode: code, + }); + }, + ); + + test('keeps an overlapping enable from recreating a record under a deletion fence', async () => { + await enable(pullOne); + await methods.stopPRAutomations(userId, 'conversation_deleting', [key.conversationId]); + await methods.disablePRAutomation(key); + expect(await enable(pullOne)).toMatchObject({ + state: 'stopped', + stopCode: 'conversation_deleting', + }); + }); + + test('still removes a record a user stopped', async () => { + await enable(pullOne); + await methods.stopPRAutomation(key, 'user_stopped'); + expect(await methods.disablePRAutomation(key)).toEqual({ removed: true }); + }); }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index bfdd9d96da2..544f97d6633 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -33,10 +33,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): settlePRAutomationRound: ( params: t.SettlePRAutomationRoundParams, ) => Promise; - stopPRAutomation: ( - key: t.PRAutomationKey, - stopCode: PRAutomationStopCode, - ) => Promise; + stopPRAutomation: { + (key: t.PRAutomationKey, stopCode: 'user_stopped'): Promise; + ( + key: t.PRAutomationKey, + stopCode: PRAutomationStopCode, + binding: t.PRAutomationBinding, + ): Promise; + }; stopPRAutomations: ( userId: string, stopCode: PRAutomationStopCode, @@ -53,7 +57,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): maxBots: number, repository: string, ) => Promise; - removePRAutomationBot: (key: t.PRAutomationKey, botId: number) => Promise; + removePRAutomationBot: ( + key: t.PRAutomationKey, + botId: number, + repository: string, + ) => Promise; } { let indexesPromise: Promise | null = null; @@ -172,9 +180,17 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return record; } + /** + * A deletion fence is not removed here. It belongs to the delete that wrote it, and removing + * it early would let an overlapping enable recreate an idle record while that delete is still + * in progress. + */ async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { const PRAutomation = mongoose.models.PRAutomation; - const result = await PRAutomation.deleteOne(keyFilter(key)); + const result = await PRAutomation.deleteOne({ + ...keyFilter(key), + stopCode: { $nin: DELETION_STOP_CODES }, + }); return { removed: result.deletedCount > 0 }; } @@ -221,21 +237,22 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: false, error: { code } }; } - /** The window opens at the first claim attempt on an active record, once. */ - await PRAutomation.updateOne( - { ...filter, state: { $in: CLAIMABLE_STATES }, startedAt: { $exists: false } }, - { $set: { startedAt: now } }, - ); - + /** + * The window opens with the first round that is actually claimed. `$min` sets `startedAt` + * only when it is missing and keeps the earlier time otherwise, so the window is part of the + * same atomic write: a claim that fails or is abandoned leaves nothing behind to count + * against `maxMinutes`. + */ const claimed = await PRAutomation.findOneAndUpdate( { ...filter, state: { $in: CLAIMABLE_STATES }, round: { $lt: maxRounds }, - startedAt: { $gte: cutoff }, + $or: [{ startedAt: { $exists: false } }, { startedAt: { $gte: cutoff } }], claimedHeads: { $ne: headSha }, }, { + $min: { startedAt: now }, $inc: { round: 1 }, $set: { state: 'fixing', lastHeadSha: headSha, runId }, $push: { claimedHeads: headSha }, @@ -295,19 +312,27 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } /** - * Stops the automation whatever round is running, so a user stop is never - * lost to a race. A stop code is required: the client maps it to the reason - * it shows. The first stop wins, and a stopped record leaves that state only - * through `enablePRAutomation`. + * Stops the automation whatever round is running. A stop code is required: the client + * maps it to the reason it shows. The first stop wins, and a stopped record leaves that + * state only through `enablePRAutomation`. + * + * A user stop is unconditional, so it is never lost to a race. Every other code comes from + * an event about one pull request, and a delayed one must not stop the run that replaced + * it, so it names the pull request it is for and applies only while the record is still + * bound to it. */ async function stopPRAutomation( key: t.PRAutomationKey, stopCode: PRAutomationStopCode, + binding?: t.PRAutomationBinding, ): Promise { assertStopCode(stopCode); + if (stopCode !== 'user_stopped' && binding == null) { + throw new RangeError('An event-driven stop must name the pull request it is for'); + } const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( - { ...keyFilter(key), state: { $ne: 'stopped' } }, + { ...keyFilter(key), ...binding, state: { $ne: 'stopped' } }, { $set: { state: 'stopped', stopCode } }, { new: true, select: PROJECTION }, ).lean(); @@ -408,13 +433,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: false, error: { code: 'bot_limit' } }; } + /** Names the repository like an approval does, so a delayed removal cannot delete a newer approval. */ async function removePRAutomationBot( key: t.PRAutomationKey, botId: number, + repository: string, ): Promise { const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( - keyFilter(key), + { ...keyFilter(key), repository }, { $pull: { trustedBots: { id: botId } } }, { new: true, select: PROJECTION }, ).lean(); diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index a52e6a2f55d..1cce10bc9ad 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -2202,20 +2202,48 @@ describe('User Methods - Database Tests', () => { }); describe('PR automation cleanup on account deletion', () => { - it('keeps the records, stopped, when the account delete fails', async () => { + it('keeps the account and the records, stopped, when the cleanup fails, so deletion is retried', async () => { const user = await User.create({ email: 'delete-pr-fail@example.com', provider: 'local' }); const id = user._id.toString(); const PRAutomation = mongoose.models.PRAutomation; await PRAutomation.create({ user: id, conversationId: 'chat-a' }); - const deletion = jest.spyOn(User, 'deleteOne').mockRejectedValueOnce(new Error('db down')); - - await expect(methods.deleteUserById(id)).rejects.toThrow('db down'); - deletion.mockRestore(); + const cleanup = jest + .spyOn(PRAutomation, 'deleteMany') + .mockRejectedValueOnce(new Error('synthetic cleanup failure')); + const deletion = jest.spyOn(User, 'deleteOne'); + try { + await expect(methods.deleteUserById(id)).rejects.toThrow('synthetic cleanup failure'); + expect(deletion).not.toHaveBeenCalled(); + expect(await User.exists({ _id: user._id })).not.toBeNull(); + expect(await PRAutomation.findOne({ user: id }).lean()).toMatchObject({ + state: 'stopped', + stopCode: 'account_deleting', + }); + await expect(methods.deleteUserById(id)).resolves.toMatchObject({ deletedCount: 1 }); + expect(await User.exists({ _id: user._id })).toBeNull(); + expect(await PRAutomation.countDocuments({ user: id })).toBe(0); + } finally { + cleanup.mockRestore(); + deletion.mockRestore(); + } + }); - expect(await PRAutomation.findOne({ user: id }).lean()).toMatchObject({ - state: 'stopped', - stopCode: 'account_deleting', - }); + it('removes the records before it deletes the account', async () => { + const user = await User.create({ email: 'delete-pr-order@example.com', provider: 'local' }); + const id = user._id.toString(); + const PRAutomation = mongoose.models.PRAutomation; + await PRAutomation.create({ user: id, conversationId: 'chat-a' }); + const cleanup = jest.spyOn(PRAutomation, 'deleteMany'); + const deletion = jest.spyOn(User, 'deleteOne'); + try { + await methods.deleteUserById(id); + expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan( + deletion.mock.invocationCallOrder[0], + ); + } finally { + cleanup.mockRestore(); + deletion.mockRestore(); + } }); it('removes the records of an account that no longer exists', async () => { diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 59357ff93ca..23fab5cddc2 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -646,15 +646,13 @@ export function createUserMethods( try { const User = mongoose.models.User; await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); - /** Fenced first so a webhook cannot claim a record for an account that is going away; - * removed only after the account delete committed. */ + /** Fenced first so a webhook cannot claim a record for an account that is going away, + * then removed before the account, like the grants above. A failed cleanup throws with the + * account intact, so the deletion is retried instead of reporting success and leaving the + * repository and bot metadata behind with nothing that would ever remove it. */ await prAutomation.stopPRAutomations(userId, 'account_deleting'); + await prAutomation.deletePRAutomations(userId); const result = await User.deleteOne({ _id: userId }); - try { - await prAutomation.deletePRAutomations(userId); - } catch { - logger.warn('[deleteUserById] PR automation cleanup failed; the records stay stopped.'); - } if (result.deletedCount === 0) { return { deletedCount: 0, message: 'No user found with that ID.' }; } From bf8d19d39e7e5b880cf5665abebd6878f4c71152 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:39:21 +0200 Subject: [PATCH 07/25] fix: Give PR Automation Deletion Fences a Way Out and Cover First-Time Enables A deletion now writes a tombstone for a conversation that has no record yet, so an enable already in flight cannot create an idle record under it. When the delete settles, the fence is released: a record whose conversation is gone is removed, and one whose conversation survived becomes a restartable stop with the deletion_aborted code. Account deletion removes the records only after the user delete committed. A claim for a subagent thread also requires its parent and root conversations to be active. The schema rejects a fractional pull request number. --- .../data-provider/src/types/prAutomation.ts | 2 + .../src/methods/conversation.spec.ts | 15 +- .../data-schemas/src/methods/conversation.ts | 23 ++- .../src/methods/prAutomation.spec.ts | 192 ++++++++++++++++++ .../data-schemas/src/methods/prAutomation.ts | 103 +++++++++- .../src/methods/user.methods.spec.ts | 45 ++-- packages/data-schemas/src/methods/user.ts | 20 +- .../data-schemas/src/schema/prAutomation.ts | 9 +- 8 files changed, 380 insertions(+), 29 deletions(-) diff --git a/packages/data-provider/src/types/prAutomation.ts b/packages/data-provider/src/types/prAutomation.ts index ecd0f04733e..e7892884ddd 100644 --- a/packages/data-provider/src/types/prAutomation.ts +++ b/packages/data-provider/src/types/prAutomation.ts @@ -15,6 +15,8 @@ export const PR_AUTOMATION_STOP_CODES = [ 'disabled_by_admin', 'conversation_deleting', 'account_deleting', + /** A deletion that did not commit released its fence; the user can turn the automation back on. */ + 'deletion_aborted', ] as const; /** Largest per-repository bot allowlist a deployment can configure. */ diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 552db291893..113ff89c92c 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -3765,11 +3765,11 @@ describe('Conversation Operations', () => { expect(await Conversation.exists({ conversationId })).not.toBeNull(); }); - it('stops the record so a webhook cannot claim it meanwhile', async () => { + it('leaves the record stopped, so a webhook cannot claim it, but not fenced for good', async () => { const { conversationId, automation } = await setup(); expect(await automation.getPRAutomation({ userId: user, conversationId })).toMatchObject({ state: 'stopped', - stopCode: 'conversation_deleting', + stopCode: 'deletion_aborted', }); const claimed = await automation.claimPRAutomationRound({ userId: user, @@ -3781,6 +3781,17 @@ describe('Conversation Operations', () => { }); expect(claimed).toEqual({ ok: false, error: { code: 'not_active' } }); }); + + it('lets the user turn the automation back on for the conversation they kept', async () => { + const { conversationId, automation } = await setup(); + expect( + await automation.enablePRAutomation({ + userId: user, + conversationId, + binding: { repository: 'acme/one', pullNumber: 1 }, + }), + ).toMatchObject({ state: 'idle' }); + }); }); it('removes the PR automation record of a root that a previous attempt already deleted', async () => { diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index 6ebff8067fa..4b42b7e856c 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -4143,14 +4143,27 @@ export function createConversationMethods( await options?.beforeDelete?.(waveIds); await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds); /** Fenced before the delete so a webhook cannot claim a record whose conversation is - * going away. The record itself is removed only once the delete has committed, so a - * failed delete keeps the stored state and a retry finds it. */ + * going away, with a tombstone when none exists yet. Once the delete settles the fence + * is released: a record whose conversation is gone is removed, and one whose + * conversation survived becomes an ordinary stop the user can restart. */ await prAutomation.stopPRAutomations(user, 'conversation_deleting', waveIds); - const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); + let result: Awaited>; try { - await prAutomation.deletePRAutomations(user, waveIds); + result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); + } catch (error) { + await prAutomation.releasePRAutomationFences(user, waveIds).catch(() => { + logger.warn( + '[deleteConvos] PR automation fence release failed; the record stays stopped.', + ); + }); + throw error; + } + try { + await prAutomation.releasePRAutomationFences(user, waveIds); } catch { - logger.warn('[deleteConvos] PR automation cleanup failed; the record stays stopped.'); + logger.warn( + '[deleteConvos] PR automation fence release failed; the record stays stopped.', + ); } if (result.deletedCount > 0) { /** Result erasure is irreversible. Keep receipts intact when a diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index bfa619150ac..f0f2202444e 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -226,6 +226,198 @@ describe('a record stopped by a deletion fence', () => { }); }); +describe('a deletion fence written before any record exists', () => { + const fenceConversation = () => + methods.stopPRAutomations(userId, 'conversation_deleting', [key.conversationId]); + + test('is not replaced by an enable that was already in flight', async () => { + await fenceConversation(); + expect(await enable(pullOne)).toMatchObject({ + state: 'stopped', + stopCode: 'conversation_deleting', + }); + }); + + test('refuses a claim', async () => { + await fenceConversation(); + await enable(pullOne); + expect((await claim(1)).ok).toBe(false); + }); + + test('is written once when the fence is written twice', async () => { + await fenceConversation(); + await fenceConversation(); + expect(await PRAutomation.countDocuments({ user: userId })).toBe(1); + }); + + test('does not touch another conversation', async () => { + await fenceConversation(); + expect(await methods.getPRAutomation({ userId, conversationId: 'other-convo' })).toBeNull(); + }); +}); + +describe('releasing a deletion fence after an aborted deletion', () => { + const fence = () => + methods.stopPRAutomations(userId, 'conversation_deleting', [key.conversationId]); + + test('keeps a fence that only existed for the deletion restartable if the conversation survived', async () => { + await fence(); + await methods.releasePRAutomationFences(userId, [key.conversationId]); + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'stopped', + stopCode: 'deletion_aborted', + }); + }); + + test('removes a fence whose conversation was removed', async () => { + await fence(); + await mongoose.models.Conversation.deleteMany({}); + await methods.releasePRAutomationFences(userId, [key.conversationId]); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('keeps the record of a conversation that survived, stopped and restartable', async () => { + await enable(pullOne); + await fence(); + await methods.releasePRAutomationFences(userId, [key.conversationId]); + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'stopped', + stopCode: 'deletion_aborted', + repository: 'acme/one', + }); + }); + + test('lets the user turn a released record back on', async () => { + await enable(pullOne); + await fence(); + await methods.releasePRAutomationFences(userId, [key.conversationId]); + expect(await enable(pullOne)).toMatchObject({ state: 'idle' }); + }); + + test('removes the record of a conversation the deletion did remove', async () => { + await enable(pullOne); + await fence(); + await mongoose.models.Conversation.deleteMany({}); + await methods.releasePRAutomationFences(userId, [key.conversationId]); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('leaves a record that is not under a deletion fence alone', async () => { + await enable(pullOne); + await methods.releasePRAutomationFences(userId, [key.conversationId]); + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'idle' }); + }); + + test('releases every fenced record of an account that survived', async () => { + await enable(pullOne); + await methods.stopPRAutomations(userId, 'account_deleting'); + await methods.releasePRAutomationFences(userId); + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'stopped', + stopCode: 'deletion_aborted', + }); + }); +}); + +describe('claiming for a subagent thread whose root conversation is gone', () => { + const childKey = { userId, conversationId: 'child-1' }; + const seedChild = () => + mongoose.models.Conversation.create({ + conversationId: childKey.conversationId, + user: userId, + endpoint: 'agents', + subagentThread: { + rootConversationId: key.conversationId, + parentConversationId: key.conversationId, + parentMessageId: 'message-1', + parentToolCallId: 'call-1', + subagentType: 'agent-child', + subagentKind: 'agent', + depth: 1, + }, + }); + const claimChild = () => + methods.claimPRAutomationRound({ + ...childKey, + ...limits, + binding: pullOne, + headSha: head(1), + }); + + test('rejects a claim for a child once its root was deleted', async () => { + await seedChild(); + await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + await mongoose.models.Conversation.deleteOne({ conversationId: key.conversationId }); + expect(await claimChild()).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + }); + + test('removes the child record so nothing is left to claim later', async () => { + await seedChild(); + await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + await mongoose.models.Conversation.deleteOne({ conversationId: key.conversationId }); + await claimChild(); + expect(await methods.getPRAutomation(childKey)).toBeNull(); + }); + + test('rejects a claim for a child whose root passed its retention date', async () => { + await seedChild(); + await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + await mongoose.models.Conversation.updateOne( + { conversationId: key.conversationId }, + { $set: { expiredAt: new Date(Date.now() - 60_000) } }, + ); + expect(await claimChild()).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + }); + + test('rejects a claim for a grandchild once its parent was deleted but its root remains', async () => { + await seedChild(); + await mongoose.models.Conversation.create({ + conversationId: 'grandchild-1', + user: userId, + endpoint: 'agents', + subagentThread: { + rootConversationId: key.conversationId, + parentConversationId: childKey.conversationId, + parentMessageId: 'message-2', + parentToolCallId: 'call-2', + subagentType: 'agent-child', + subagentKind: 'agent', + depth: 2, + }, + }); + const grandKey = { userId, conversationId: 'grandchild-1' }; + await methods.enablePRAutomation({ ...grandKey, binding: pullOne }); + await mongoose.models.Conversation.deleteOne({ conversationId: childKey.conversationId }); + expect( + await methods.claimPRAutomationRound({ + ...grandKey, + ...limits, + binding: pullOne, + headSha: head(1), + }), + ).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + }); + + test('accepts a claim for a child while its root is active', async () => { + await seedChild(); + await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + expect((await claimChild()).ok).toBe(true); + }); +}); + +describe('pull request numbers', () => { + test.each([1.5, 0.5, 2.0000001])('rejects %s as a pull request number', async (pullNumber) => { + await expect(enable({ repository: 'acme/one', pullNumber })).rejects.toThrow(); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('still accepts a whole pull request number', async () => { + expect(await enable({ repository: 'acme/one', pullNumber: 42 })).toMatchObject({ + pullNumber: 42, + }); + }); +}); + describe('index guarantees', () => { test('builds the unique index before the first write when automatic indexing is off', async () => { await PRAutomation.collection.dropIndexes(); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 544f97d6633..a577b7fdb10 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -14,6 +14,8 @@ const PROJECTION = '-_id -__v'; */ const DELETION_STOP_CODES: PRAutomationStopCode[] = ['conversation_deleting', 'account_deleting']; const RESTARTABLE = { state: 'stopped', stopCode: { $nin: DELETION_STOP_CODES } }; +/** Conversations looked up per query when a fence is released. */ +const RELEASE_BATCH = 500; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; @@ -46,6 +48,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): stopCode: PRAutomationStopCode, conversationIds?: string[], ) => Promise; + releasePRAutomationFences: (userId: string, conversationIds?: string[]) => Promise; deletePRAutomations: (userId: string, conversationIds?: string[]) => Promise; setPRAutomationTrust: ( key: t.PRAutomationKey, @@ -226,12 +229,31 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * the record cannot rely on that cascade. A claim for a conversation that is gone or * past its retention date removes the record instead of starting work for it. */ - const conversation = await mongoose.models.Conversation.exists({ + const conversation = await mongoose.models.Conversation.findOne({ user: key.userId, conversationId: key.conversationId, ...activeExpirationFilter(), - }); - if (conversation == null) { + }) + .select('subagentThread') + .lean<{ subagentThread?: { rootConversationId: string; parentConversationId: string } }>(); + /** + * A subagent thread is controlled by its parent and its root. A deletion removes a root + * before it discovers the descendants, so for a moment the child still exists while the + * conversation that controls it does not; both ancestors must be active to claim. + */ + const thread = conversation?.subagentThread; + const ancestors = + thread == null ? [] : [thread.rootConversationId, thread.parentConversationId]; + const required = [...new Set(ancestors)]; + const activeAncestors = + required.length === 0 + ? 0 + : await mongoose.models.Conversation.countDocuments({ + user: key.userId, + conversationId: { $in: required }, + ...activeExpirationFilter(), + }); + if (conversation == null || activeAncestors !== required.length) { const removed = await PRAutomation.deleteOne(keyFilter(key)); const code = removed.deletedCount > 0 ? 'conversation_gone' : 'not_found'; return { ok: false, error: { code } }; @@ -363,6 +385,80 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): }, { $set: { state: 'stopped', stopCode } }, ); + /** + * A conversation without a record gets a tombstone, so an enable that was already in + * flight finds the fence instead of creating an idle record. An existing record, fenced + * or not, makes this a no-op, and the unique index keeps it to one per conversation. + */ + if (conversationIds == null || !DELETION_STOP_CODES.includes(stopCode)) { + return; + } + await ensureIndexes(); + await Promise.all( + conversationIds.map((conversationId) => + PRAutomation.updateOne( + { user: userId, conversationId }, + { + $setOnInsert: { + user: userId, + conversationId, + state: 'stopped', + stopCode, + round: 0, + trustedBots: [], + claimedHeads: [], + trust: 'approvedBots', + }, + }, + { upsert: true, runValidators: true }, + ), + ), + ); + } + + /** + * Gives a deletion fence its way out. A deletion that did not commit, or only partly did, + * must not leave a conversation that survived with an automation nobody can turn back on: + * its record becomes an ordinary stop the user can restart. A record whose conversation is + * gone is removed. Records that are not under a deletion fence are left alone. + */ + async function releasePRAutomationFences( + userId: string, + conversationIds?: string[], + ): Promise { + const PRAutomation = mongoose.models.PRAutomation; + if (PRAutomation == null || conversationIds?.length === 0) { + return; + } + const fenced = { + user: userId, + stopCode: { $in: DELETION_STOP_CODES }, + ...(conversationIds != null && { conversationId: { $in: conversationIds } }), + }; + const records = await PRAutomation.find(fenced) + .select('conversationId') + .lean<{ conversationId: string }[]>(); + const ids = records.map(({ conversationId }) => conversationId); + for (let start = 0; start < ids.length; start += RELEASE_BATCH) { + const batch = ids.slice(start, start + RELEASE_BATCH); + const surviving: string[] = await mongoose.models.Conversation.distinct('conversationId', { + user: userId, + conversationId: { $in: batch }, + ...activeExpirationFilter(), + }); + const kept = new Set(surviving); + const gone = batch.filter((id) => !kept.has(id)); + const survivors = batch.filter((id) => kept.has(id)); + if (survivors.length > 0) { + await PRAutomation.updateMany( + { ...fenced, conversationId: { $in: survivors } }, + { $set: { stopCode: 'deletion_aborted' } }, + ); + } + if (gone.length > 0) { + await PRAutomation.deleteMany({ ...fenced, conversationId: { $in: gone } }); + } + } } /** Removes every record of a user, or of the listed conversations. */ @@ -455,6 +551,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): settlePRAutomationRound, stopPRAutomation, stopPRAutomations, + releasePRAutomationFences, deletePRAutomations, setPRAutomationTrust, addPRAutomationBot, diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index 1cce10bc9ad..f34295133c1 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -9,6 +9,7 @@ import type * as t from '~/types'; import { createToolApprovalGrantModel } from '~/models/toolApprovalGrant'; import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user'; import { createPRAutomationModel } from '~/models/prAutomation'; +import { createConversationModel } from '~/models/convo'; import balanceSchema from '~/schema/balance'; import userSchema from '~/schema/user'; @@ -53,6 +54,7 @@ beforeAll(async () => { methods = createUserMethods(mongoose); createToolApprovalGrantModel(mongoose); createPRAutomationModel(mongoose); + createConversationModel(mongoose); }); afterAll(async () => { @@ -2202,33 +2204,50 @@ describe('User Methods - Database Tests', () => { }); describe('PR automation cleanup on account deletion', () => { - it('keeps the account and the records, stopped, when the cleanup fails, so deletion is retried', async () => { + it('keeps the records, restartable, when the account delete fails', async () => { const user = await User.create({ email: 'delete-pr-fail@example.com', provider: 'local' }); const id = user._id.toString(); const PRAutomation = mongoose.models.PRAutomation; + await mongoose.models.Conversation.create({ + conversationId: 'chat-a', + user: id, + endpoint: 'agents', + }); + await PRAutomation.create({ user: id, conversationId: 'chat-a', repository: 'acme/one' }); + const deletion = jest.spyOn(User, 'deleteOne').mockRejectedValueOnce(new Error('db down')); + try { + await expect(methods.deleteUserById(id)).rejects.toThrow('db down'); + expect(await User.exists({ _id: user._id })).not.toBeNull(); + expect(await PRAutomation.findOne({ user: id }).lean()).toMatchObject({ + state: 'stopped', + stopCode: 'deletion_aborted', + repository: 'acme/one', + }); + } finally { + deletion.mockRestore(); + } + }); + + it('converges on a retry when the cleanup fails after the account was deleted', async () => { + const user = await User.create({ email: 'delete-pr-retry@example.com', provider: 'local' }); + const id = user._id.toString(); + const PRAutomation = mongoose.models.PRAutomation; await PRAutomation.create({ user: id, conversationId: 'chat-a' }); const cleanup = jest .spyOn(PRAutomation, 'deleteMany') .mockRejectedValueOnce(new Error('synthetic cleanup failure')); - const deletion = jest.spyOn(User, 'deleteOne'); try { await expect(methods.deleteUserById(id)).rejects.toThrow('synthetic cleanup failure'); - expect(deletion).not.toHaveBeenCalled(); - expect(await User.exists({ _id: user._id })).not.toBeNull(); - expect(await PRAutomation.findOne({ user: id }).lean()).toMatchObject({ - state: 'stopped', - stopCode: 'account_deleting', - }); - await expect(methods.deleteUserById(id)).resolves.toMatchObject({ deletedCount: 1 }); expect(await User.exists({ _id: user._id })).toBeNull(); + expect(await PRAutomation.countDocuments({ user: id })).toBe(1); + await expect(methods.deleteUserById(id)).resolves.toMatchObject({ deletedCount: 0 }); expect(await PRAutomation.countDocuments({ user: id })).toBe(0); } finally { cleanup.mockRestore(); - deletion.mockRestore(); } }); - it('removes the records before it deletes the account', async () => { + it('removes the records after the account delete committed, not before', async () => { const user = await User.create({ email: 'delete-pr-order@example.com', provider: 'local' }); const id = user._id.toString(); const PRAutomation = mongoose.models.PRAutomation; @@ -2237,8 +2256,8 @@ describe('PR automation cleanup on account deletion', () => { const deletion = jest.spyOn(User, 'deleteOne'); try { await methods.deleteUserById(id); - expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan( - deletion.mock.invocationCallOrder[0], + expect(deletion.mock.invocationCallOrder[0]).toBeLessThan( + cleanup.mock.invocationCallOrder[0], ); } finally { cleanup.mockRestore(); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 23fab5cddc2..4fb9e76ed2b 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -646,13 +646,23 @@ export function createUserMethods( try { const User = mongoose.models.User; await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); - /** Fenced first so a webhook cannot claim a record for an account that is going away, - * then removed before the account, like the grants above. A failed cleanup throws with the - * account intact, so the deletion is retried instead of reporting success and leaving the - * repository and bot metadata behind with nothing that would ever remove it. */ + /** Fenced first so a webhook cannot claim a record for an account that is going away. + * The records are removed only once the account delete committed, so a delete that fails + * keeps the user's automation: the fence is released into an ordinary stop they can + * restart. A cleanup that fails after the commit throws, and the retry removes them. */ await prAutomation.stopPRAutomations(userId, 'account_deleting'); + let result: Awaited>; + try { + result = await User.deleteOne({ _id: userId }); + } catch (error) { + await prAutomation.releasePRAutomationFences(userId).catch(() => { + logger.warn( + '[deleteUserById] PR automation fence release failed; the records stay stopped.', + ); + }); + throw error; + } await prAutomation.deletePRAutomations(userId); - const result = await User.deleteOne({ _id: userId }); if (result.deletedCount === 0) { return { deletedCount: 0, message: 'No user found with that ID.' }; } diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 4846dcb371c..62f80bb22ca 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -30,7 +30,14 @@ const prAutomationSchema: Schema = new Schema( maxlength: 256, }, repository: { type: String, maxlength: 256 }, - pullNumber: { type: Number, min: 1 }, + pullNumber: { + type: Number, + min: 1, + validate: { + validator: Number.isInteger, + message: 'A pull request number is a whole number', + }, + }, state: { type: String, enum: PR_AUTOMATION_STATES, From ff56699dc7408c47805f094a7ff99773671c3103 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:42:06 +0200 Subject: [PATCH 08/25] refactor: Check PR Automation Liveness at Claim Instead of Fencing Deletions Conversation and account deletion no longer write fences or tombstones. They remove records after the delete commits, in bounded batches, so a deletion of a conversation that never used the feature writes nothing. A round claim now checks, before and after its atomic write, that the owner has no account deletion in progress and that the conversation and its ancestors are active; a claim that loses that race hands its round back and never starts work. The auth cache is invalidated before the fallible cleanup, and the redundant user index is dropped. --- .../data-provider/src/types/prAutomation.ts | 4 - .../src/methods/conversation.spec.ts | 61 ++-- .../data-schemas/src/methods/conversation.ts | 41 +-- .../src/methods/prAutomation.spec.ts | 272 +++++++----------- .../data-schemas/src/methods/prAutomation.ts | 249 ++++++---------- .../src/methods/user.methods.spec.ts | 33 ++- packages/data-schemas/src/methods/user.ts | 23 +- .../data-schemas/src/schema/prAutomation.ts | 3 +- .../data-schemas/src/types/prAutomation.ts | 1 + 9 files changed, 289 insertions(+), 398 deletions(-) diff --git a/packages/data-provider/src/types/prAutomation.ts b/packages/data-provider/src/types/prAutomation.ts index e7892884ddd..90c2b8d70ac 100644 --- a/packages/data-provider/src/types/prAutomation.ts +++ b/packages/data-provider/src/types/prAutomation.ts @@ -13,10 +13,6 @@ export const PR_AUTOMATION_STOP_CODES = [ 'approval_expired', 'pull_request_closed', 'disabled_by_admin', - 'conversation_deleting', - 'account_deleting', - /** A deletion that did not commit released its fence; the user can turn the automation back on. */ - 'deletion_aborted', ] as const; /** Largest per-repository bot allowlist a deployment can configure. */ diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 113ff89c92c..f57d45a66f8 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -3765,33 +3765,50 @@ describe('Conversation Operations', () => { expect(await Conversation.exists({ conversationId })).not.toBeNull(); }); - it('leaves the record stopped, so a webhook cannot claim it, but not fenced for good', async () => { + it('leaves the record exactly as it was, so the kept conversation keeps working', async () => { const { conversationId, automation } = await setup(); expect(await automation.getPRAutomation({ userId: user, conversationId })).toMatchObject({ - state: 'stopped', - stopCode: 'deletion_aborted', + state: 'idle', + repository: 'acme/one', + pullNumber: 1, }); - const claimed = await automation.claimPRAutomationRound({ - userId: user, - conversationId, - binding: { repository: 'acme/one', pullNumber: 1 }, - maxRounds: 5, - maxMinutes: 60, - headSha: 'a'.repeat(40), - }); - expect(claimed).toEqual({ ok: false, error: { code: 'not_active' } }); }); + }); - it('lets the user turn the automation back on for the conversation they kept', async () => { - const { conversationId, automation } = await setup(); - expect( - await automation.enablePRAutomation({ - userId: user, - conversationId, - binding: { repository: 'acme/one', pullNumber: 1 }, - }), - ).toMatchObject({ state: 'idle' }); - }); + it('writes nothing for a conversation that never had an automation', async () => { + const conversationId = uuidv4(); + await Conversation.create({ conversationId, user: 'user123', endpoint: 'agents' }); + const PRAutomation = mongoose.models.PRAutomation; + const writes = ['updateOne', 'updateMany', 'bulkWrite', 'insertMany', 'create'].map( + (method) => jest.spyOn(PRAutomation, method as 'updateOne'), + ); + try { + await deleteConvos('user123', { conversationId }); + for (const write of writes) { + expect(write).not.toHaveBeenCalled(); + } + expect(await PRAutomation.countDocuments({ user: 'user123', conversationId })).toBe(0); + } finally { + writes.forEach((write) => write.mockRestore()); + } + }); + + it('still reports a deletion that committed when the record cleanup keeps failing', async () => { + const conversationId = uuidv4(); + await Conversation.create({ conversationId, user: 'user123', endpoint: 'agents' }); + const PRAutomation = mongoose.models.PRAutomation; + await PRAutomation.create({ user: 'user123', conversationId }); + const cleanup = jest + .spyOn(PRAutomation, 'deleteMany') + .mockRejectedValue(new Error('cleanup unavailable')); + try { + await expect(deleteConvos('user123', { conversationId })).resolves.toMatchObject({ + deletedCount: 1, + }); + expect(cleanup.mock.calls.length).toBeGreaterThan(1); + } finally { + cleanup.mockRestore(); + } }); it('removes the PR automation record of a root that a previous attempt already deleted', async () => { diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index 4b42b7e856c..af1a55e2785 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -4142,29 +4142,14 @@ export function createConversationMethods( ); await options?.beforeDelete?.(waveIds); await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds); - /** Fenced before the delete so a webhook cannot claim a record whose conversation is - * going away, with a tombstone when none exists yet. Once the delete settles the fence - * is released: a record whose conversation is gone is removed, and one whose - * conversation survived becomes an ordinary stop the user can restart. */ - await prAutomation.stopPRAutomations(user, 'conversation_deleting', waveIds); - let result: Awaited>; - try { - result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); - } catch (error) { - await prAutomation.releasePRAutomationFences(user, waveIds).catch(() => { - logger.warn( - '[deleteConvos] PR automation fence release failed; the record stays stopped.', - ); - }); - throw error; - } - try { - await prAutomation.releasePRAutomationFences(user, waveIds); - } catch { - logger.warn( - '[deleteConvos] PR automation fence release failed; the record stays stopped.', - ); - } + const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); + /** Only after the delete committed, so a failed delete keeps the user's automation. + * A claim refuses any record whose conversation is gone and removes it, so a cleanup + * that still fails after its retry converges on the next claim. */ + await retryCascadeOperation(() => prAutomation.deletePRAutomations(user, waveIds)).catch( + (error) => + logger.warn('[deleteConvos] PR automation cleanup deferred to its next claim', error), + ); if (result.deletedCount > 0) { /** Result erasure is irreversible. Keep receipts intact when a * pre-delete hook or the conversation delete itself fails, so a @@ -4198,11 +4183,11 @@ export function createConversationMethods( ]; if (recoveryConversationIds.length > 0) { - try { - await prAutomation.deletePRAutomations(user, recoveryConversationIds); - } catch { - logger.warn('[deleteConvos] PR automation cleanup failed; the record stays stopped.'); - } + await retryCascadeOperation(() => + prAutomation.deletePRAutomations(user, recoveryConversationIds), + ).catch((error) => + logger.warn('[deleteConvos] PR automation cleanup deferred to its next claim', error), + ); await deps?.deleteAgentQueuedTurns?.( user, recoveryConversationIds.map((conversationId) => ({ diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index f0f2202444e..a4d15423294 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -73,6 +73,8 @@ afterAll(async () => { beforeEach(async () => { await PRAutomation.deleteMany({}); await mongoose.models.Conversation.deleteMany({}); + await mongoose.models.User.deleteMany({}); + await mongoose.models.User.create({ _id: userId, email: 'owner@example.com', provider: 'local' }); await seedConversation(); }); @@ -153,169 +155,138 @@ describe('enablePRAutomation', () => { }); }); -describe('a record stopped by a deletion fence', () => { - const deletionCodes = ['conversation_deleting', 'account_deleting'] as const; - - test.each(deletionCodes)( - 'is not revived by enabling the same pull request (%s)', - async (code) => { - await enable(pullOne); - await methods.stopPRAutomation(key, code, pullOne); - expect(await enable(pullOne)).toMatchObject({ state: 'stopped', stopCode: code }); - }, - ); - - test.each(deletionCodes)( - 'is not revived by another pull request in the repository (%s)', - async (code) => { - await enable(pullOne); - await methods.stopPRAutomation(key, code, pullOne); - expect(await enable(pullTwo)).toMatchObject({ - state: 'stopped', - stopCode: code, - pullNumber: 1, - }); - }, - ); - - test.each(deletionCodes)('is not revived or rebound to another repository (%s)', async (code) => { - await enable(pullOne); - await methods.stopPRAutomation(key, code, pullOne); - expect(await enable(otherRepository)).toMatchObject({ - state: 'stopped', - stopCode: code, - repository: 'acme/one', - }); - }); - - test.each(deletionCodes)('is not revived without a binding (%s)', async (code) => { - await enable(pullOne); - await methods.stopPRAutomation(key, code, pullOne); - expect(await methods.enablePRAutomation(key)).toMatchObject({ - state: 'stopped', - stopCode: code, - }); - }); - - test.each(deletionCodes)('still refuses a claim (%s)', async (code) => { - await enable(pullOne); - await methods.stopPRAutomation(key, code, pullOne); - await enable(pullOne); - expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_active' } }); - }); +describe('claiming while the owner is being deleted', () => { + const startAccountDeletion = () => + mongoose.models.User.updateOne( + { _id: userId }, + { $set: { agentTriggerDeletionStartedAt: new Date() } }, + ); - test.each(deletionCodes)('replaces a user stop when a deletion begins (%s)', async (code) => { + test('refuses a claim once account deletion has started', async () => { await enable(pullOne); - await methods.stopPRAutomation(key, 'user_stopped'); - await methods.stopPRAutomations(userId, code, [key.conversationId]); - expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'stopped', stopCode: code }); - expect(await enable(pullOne)).toMatchObject({ state: 'stopped', stopCode: code }); + await startAccountDeletion(); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'owner_inactive' } }); }); - test('keeps a record of another conversation restartable', async () => { + test('keeps the record and spends no round, so a cancelled deletion leaves it as it was', async () => { await enable(pullOne); - await methods.stopPRAutomation(key, 'user_stopped'); - await methods.stopPRAutomations(userId, 'conversation_deleting', ['another-convo']); - expect(await methods.getPRAutomation(key)).toMatchObject({ stopCode: 'user_stopped' }); + await startAccountDeletion(); + await claim(1); + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'idle', round: 0 }); }); - test('a user stop is still restartable', async () => { + test('refuses a claim for an owner that no longer exists', async () => { await enable(pullOne); - await methods.stopPRAutomation(key, 'user_stopped'); - expect(await enable(pullOne)).toMatchObject({ state: 'idle' }); + await mongoose.models.User.deleteMany({}); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'owner_inactive' } }); }); -}); - -describe('a deletion fence written before any record exists', () => { - const fenceConversation = () => - methods.stopPRAutomations(userId, 'conversation_deleting', [key.conversationId]); - test('is not replaced by an enable that was already in flight', async () => { - await fenceConversation(); - expect(await enable(pullOne)).toMatchObject({ - state: 'stopped', - stopCode: 'conversation_deleting', - }); - }); - - test('refuses a claim', async () => { - await fenceConversation(); + test('claims again once the deletion was cancelled', async () => { await enable(pullOne); - expect((await claim(1)).ok).toBe(false); - }); - - test('is written once when the fence is written twice', async () => { - await fenceConversation(); - await fenceConversation(); - expect(await PRAutomation.countDocuments({ user: userId })).toBe(1); - }); - - test('does not touch another conversation', async () => { - await fenceConversation(); - expect(await methods.getPRAutomation({ userId, conversationId: 'other-convo' })).toBeNull(); + await startAccountDeletion(); + await claim(1); + await mongoose.models.User.updateOne( + { _id: userId }, + { $unset: { agentTriggerDeletionStartedAt: 1 } }, + ); + expect((await claim(1)).ok).toBe(true); }); }); -describe('releasing a deletion fence after an aborted deletion', () => { - const fence = () => - methods.stopPRAutomations(userId, 'conversation_deleting', [key.conversationId]); - - test('keeps a fence that only existed for the deletion restartable if the conversation survived', async () => { - await fence(); - await methods.releasePRAutomationFences(userId, [key.conversationId]); - expect(await methods.getPRAutomation(key)).toMatchObject({ - state: 'stopped', - stopCode: 'deletion_aborted', - }); - }); - - test('removes a fence whose conversation was removed', async () => { - await fence(); - await mongoose.models.Conversation.deleteMany({}); - await methods.releasePRAutomationFences(userId, [key.conversationId]); - expect(await methods.getPRAutomation(key)).toBeNull(); - }); - - test('keeps the record of a conversation that survived, stopped and restartable', async () => { - await enable(pullOne); - await fence(); - await methods.releasePRAutomationFences(userId, [key.conversationId]); - expect(await methods.getPRAutomation(key)).toMatchObject({ - state: 'stopped', - stopCode: 'deletion_aborted', - repository: 'acme/one', - }); +describe('a deletion that lands while a claim is in flight', () => { + /** Runs `during` right after the claim's atomic write, before the claim checks again. */ + const interleave = (during: () => Promise) => { + const original = PRAutomation.findOneAndUpdate.bind(PRAutomation); + return jest.spyOn(PRAutomation, 'findOneAndUpdate').mockImplementationOnce((( + ...args: Parameters + ) => { + const query = original(...args); + const lean = query.lean.bind(query); + query.lean = ((...leanArgs: Parameters) => { + const result = lean(...leanArgs); + return { + ...result, + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + result + .then(async (value: unknown) => { + await during(); + return value; + }) + .then(resolve, reject), + }; + }) as typeof query.lean; + return query; + }) as typeof PRAutomation.findOneAndUpdate); + }; + + test('does not hand out a round for a conversation deleted after the first check', async () => { + await enable(pullOne); + const spy = interleave(() => mongoose.models.Conversation.deleteMany({})); + try { + expect(await claim(1)).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + } finally { + spy.mockRestore(); + } }); - test('lets the user turn a released record back on', async () => { + test('does not hand out a round once account deletion started after the first check', async () => { await enable(pullOne); - await fence(); - await methods.releasePRAutomationFences(userId, [key.conversationId]); - expect(await enable(pullOne)).toMatchObject({ state: 'idle' }); + const spy = interleave(() => + mongoose.models.User.updateOne( + { _id: userId }, + { $set: { agentTriggerDeletionStartedAt: new Date() } }, + ), + ); + try { + expect(await claim(1)).toEqual({ ok: false, error: { code: 'owner_inactive' } }); + } finally { + spy.mockRestore(); + } }); - test('removes the record of a conversation the deletion did remove', async () => { + test('leaves no round running when the owner deletion started mid-claim', async () => { await enable(pullOne); - await fence(); - await mongoose.models.Conversation.deleteMany({}); - await methods.releasePRAutomationFences(userId, [key.conversationId]); - expect(await methods.getPRAutomation(key)).toBeNull(); + const spy = interleave(() => + mongoose.models.User.updateOne( + { _id: userId }, + { $set: { agentTriggerDeletionStartedAt: new Date() } }, + ), + ); + try { + await claim(1); + } finally { + spy.mockRestore(); + } + expect((await methods.getPRAutomation(key))?.state).toBe('waiting'); }); +}); - test('leaves a record that is not under a deletion fence alone', async () => { - await enable(pullOne); - await methods.releasePRAutomationFences(userId, [key.conversationId]); - expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'idle' }); +describe('removing records in bulk', () => { + test('removes every listed conversation, across more than one batch', async () => { + const ids = Array.from({ length: 2500 }, (_, index) => `convo-${index}`); + await PRAutomation.insertMany(ids.map((conversationId) => ({ user: userId, conversationId }))); + await methods.deletePRAutomations(userId, ids); + expect(await PRAutomation.countDocuments({ user: userId })).toBe(0); + }); + + test('bounds each write instead of sending one unbounded list', async () => { + const ids = Array.from({ length: 2500 }, (_, index) => `convo-${index}`); + const spy = jest.spyOn(PRAutomation, 'deleteMany'); + try { + await methods.deletePRAutomations(userId, ids); + const sizes = spy.mock.calls.map( + ([filter]) => (filter as { conversationId: { $in: string[] } }).conversationId.$in.length, + ); + expect(Math.max(...sizes)).toBeLessThanOrEqual(1000); + } finally { + spy.mockRestore(); + } }); - test('releases every fenced record of an account that survived', async () => { - await enable(pullOne); - await methods.stopPRAutomations(userId, 'account_deleting'); - await methods.releasePRAutomationFences(userId); - expect(await methods.getPRAutomation(key)).toMatchObject({ - state: 'stopped', - stopCode: 'deletion_aborted', - }); + test('leaves other users alone', async () => { + await PRAutomation.create({ user: 'someone-else', conversationId: 'convo-1' }); + await methods.deletePRAutomations(userId, ['convo-1']); + expect(await PRAutomation.countDocuments({ user: 'someone-else' })).toBe(1); }); }); @@ -980,29 +951,6 @@ describe('disablePRAutomation', () => { expect(await methods.disablePRAutomation(key)).toEqual({ removed: false }); }); - test.each(['conversation_deleting', 'account_deleting'] as const)( - 'does not remove a deletion fence (%s)', - async (code) => { - await enable(pullOne); - await methods.stopPRAutomations(userId, code, [key.conversationId]); - expect(await methods.disablePRAutomation(key)).toEqual({ removed: false }); - expect(await methods.getPRAutomation(key)).toMatchObject({ - state: 'stopped', - stopCode: code, - }); - }, - ); - - test('keeps an overlapping enable from recreating a record under a deletion fence', async () => { - await enable(pullOne); - await methods.stopPRAutomations(userId, 'conversation_deleting', [key.conversationId]); - await methods.disablePRAutomation(key); - expect(await enable(pullOne)).toMatchObject({ - state: 'stopped', - stopCode: 'conversation_deleting', - }); - }); - test('still removes a record a user stopped', async () => { await enable(pullOne); await methods.stopPRAutomation(key, 'user_stopped'); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index a577b7fdb10..db45ca8c739 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -5,17 +5,15 @@ import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationState } from 'librechat-data-provider'; import type * as t from '~/types/prAutomation'; import { activeExpirationFilter } from '~/utils/retention'; +import { isValidObjectIdString } from '~/utils/objectId'; import { createIndexesWithRetry } from '~/utils/retry'; const PROJECTION = '-_id -__v'; -/** - * A deletion fence is not a user decision. Enabling must never clear it, or a webhook could - * claim work for a conversation or account that is being removed. - */ -const DELETION_STOP_CODES: PRAutomationStopCode[] = ['conversation_deleting', 'account_deleting']; -const RESTARTABLE = { state: 'stopped', stopCode: { $nin: DELETION_STOP_CODES } }; -/** Conversations looked up per query when a fence is released. */ -const RELEASE_BATCH = 500; +const RESTARTABLE = { state: 'stopped' }; +/** Records removed per write, so a cleanup never holds one unbounded `$in`. */ +const DELETE_BATCH = 1000; + +type Liveness = 'live' | 'conversation_gone' | 'owner_inactive'; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; @@ -43,12 +41,6 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): binding: t.PRAutomationBinding, ): Promise; }; - stopPRAutomations: ( - userId: string, - stopCode: PRAutomationStopCode, - conversationIds?: string[], - ) => Promise; - releasePRAutomationFences: (userId: string, conversationIds?: string[]) => Promise; deletePRAutomations: (userId: string, conversationIds?: string[]) => Promise; setPRAutomationTrust: ( key: t.PRAutomationKey, @@ -89,6 +81,63 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): conversationId, }); + /** + * Whether a round may run for this record right now: the owner exists and no account + * deletion has started (the same durable marker agent triggers use), and the conversation + * and every conversation that controls it still exist and are inside their retention. A + * subagent thread is controlled by its parent and its root, and a deletion removes the + * root before it discovers the descendants. Deletions do not fence the record; the claim + * asks this on both sides of its write instead. + */ + async function checkLiveness(key: t.PRAutomationKey): Promise { + if (!isValidObjectIdString(key.userId)) { + return 'owner_inactive'; + } + const Conversation = mongoose.models.Conversation; + const active = activeExpirationFilter(); + const [owner, conversation] = await Promise.all([ + mongoose.models.User.exists({ + _id: key.userId, + agentTriggerDeletionStartedAt: { $exists: false }, + }), + Conversation.findOne({ user: key.userId, conversationId: key.conversationId, ...active }) + .select('subagentThread') + .lean<{ subagentThread?: { rootConversationId: string; parentConversationId: string } }>(), + ]); + if (owner == null) { + return 'owner_inactive'; + } + if (conversation == null) { + return 'conversation_gone'; + } + const thread = conversation.subagentThread; + if (thread == null) { + return 'live'; + } + const ancestors = [...new Set([thread.rootConversationId, thread.parentConversationId])]; + const activeAncestors = await Conversation.countDocuments({ + user: key.userId, + conversationId: { $in: ancestors }, + ...active, + }); + return activeAncestors === ancestors.length ? 'live' : 'conversation_gone'; + } + + /** A record whose conversation is gone can never run again, so it is removed. */ + async function refuseClaim( + key: t.PRAutomationKey, + liveness: Exclude, + ): Promise { + if (liveness === 'owner_inactive') { + return { ok: false, error: { code: 'owner_inactive' } }; + } + const removed = await mongoose.models.PRAutomation.deleteOne(keyFilter(key)); + return { + ok: false, + error: { code: removed.deletedCount > 0 ? 'conversation_gone' : 'not_found' }, + }; + } + function assertStopCode(stopCode: PRAutomationStopCode): void { if (!PR_AUTOMATION_STOP_CODES.includes(stopCode)) { throw new RangeError('A stopped PR automation needs a known stop code'); @@ -183,17 +232,9 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return record; } - /** - * A deletion fence is not removed here. It belongs to the delete that wrote it, and removing - * it early would let an overlapping enable recreate an idle record while that delete is still - * in progress. - */ async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { const PRAutomation = mongoose.models.PRAutomation; - const result = await PRAutomation.deleteOne({ - ...keyFilter(key), - stopCode: { $nin: DELETION_STOP_CODES }, - }); + const result = await PRAutomation.deleteOne(keyFilter(key)); return { removed: result.deletedCount > 0 }; } @@ -225,38 +266,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const runId = randomUUID(); /** - * Conversations leave through the retention index without passing `deleteConvos`, so - * the record cannot rely on that cascade. A claim for a conversation that is gone or - * past its retention date removes the record instead of starting work for it. - */ - const conversation = await mongoose.models.Conversation.findOne({ - user: key.userId, - conversationId: key.conversationId, - ...activeExpirationFilter(), - }) - .select('subagentThread') - .lean<{ subagentThread?: { rootConversationId: string; parentConversationId: string } }>(); - /** - * A subagent thread is controlled by its parent and its root. A deletion removes a root - * before it discovers the descendants, so for a moment the child still exists while the - * conversation that controls it does not; both ancestors must be active to claim. + * Conversations also leave through the retention index, which never runs + * `deleteConvos`, so liveness is checked here rather than trusted to a cascade. */ - const thread = conversation?.subagentThread; - const ancestors = - thread == null ? [] : [thread.rootConversationId, thread.parentConversationId]; - const required = [...new Set(ancestors)]; - const activeAncestors = - required.length === 0 - ? 0 - : await mongoose.models.Conversation.countDocuments({ - user: key.userId, - conversationId: { $in: required }, - ...activeExpirationFilter(), - }); - if (conversation == null || activeAncestors !== required.length) { - const removed = await PRAutomation.deleteOne(keyFilter(key)); - const code = removed.deletedCount > 0 ? 'conversation_gone' : 'not_found'; - return { ok: false, error: { code } }; + const before = await checkLiveness(key); + if (before !== 'live') { + return refuseClaim(key, before); } /** @@ -283,7 +298,20 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): { new: true, select: PROJECTION }, ).lean(); if (claimed != null) { - return { ok: true, value: { ...claimed, runId } }; + /** + * A deletion that started between the first check and the write is seen here. The + * claimed round is handed back to `waiting` under its own run id, so it never starts + * work, and a record whose conversation is gone is removed. + */ + const after = await checkLiveness(key); + if (after === 'live') { + return { ok: true, value: { ...claimed, runId } }; + } + await PRAutomation.updateOne( + { ...keyFilter(key), runId, state: 'fixing' }, + { $set: { state: 'waiting' } }, + ); + return refuseClaim(key, after); } const current = await getPRAutomation(key); @@ -361,116 +389,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } /** - * Fences every record of a user, or of the listed conversations, without - * removing it. A deletion fences first so a webhook cannot claim a record - * whose conversation or account is going away, and removes the record only - * after the delete committed, so a failed delete keeps the stored state. + * Removes every record of a user, or of the listed conversations, in bounded batches. + * Cleanup only: it creates nothing, so a deployment that never used the feature pays one + * indexed delete that matches no document. */ - async function stopPRAutomations( - userId: string, - stopCode: PRAutomationStopCode, - conversationIds?: string[], - ): Promise { - assertStopCode(stopCode); + async function deletePRAutomations(userId: string, conversationIds?: string[]): Promise { const PRAutomation = mongoose.models.PRAutomation; if (PRAutomation == null || conversationIds?.length === 0) { return; } - await PRAutomation.updateMany( - { - user: userId, - /** A user stop is overwritten, because only a deletion code resists a restart. */ - stopCode: { $nin: DELETION_STOP_CODES }, - ...(conversationIds != null && { conversationId: { $in: conversationIds } }), - }, - { $set: { state: 'stopped', stopCode } }, - ); - /** - * A conversation without a record gets a tombstone, so an enable that was already in - * flight finds the fence instead of creating an idle record. An existing record, fenced - * or not, makes this a no-op, and the unique index keeps it to one per conversation. - */ - if (conversationIds == null || !DELETION_STOP_CODES.includes(stopCode)) { - return; - } - await ensureIndexes(); - await Promise.all( - conversationIds.map((conversationId) => - PRAutomation.updateOne( - { user: userId, conversationId }, - { - $setOnInsert: { - user: userId, - conversationId, - state: 'stopped', - stopCode, - round: 0, - trustedBots: [], - claimedHeads: [], - trust: 'approvedBots', - }, - }, - { upsert: true, runValidators: true }, - ), - ), - ); - } - - /** - * Gives a deletion fence its way out. A deletion that did not commit, or only partly did, - * must not leave a conversation that survived with an automation nobody can turn back on: - * its record becomes an ordinary stop the user can restart. A record whose conversation is - * gone is removed. Records that are not under a deletion fence are left alone. - */ - async function releasePRAutomationFences( - userId: string, - conversationIds?: string[], - ): Promise { - const PRAutomation = mongoose.models.PRAutomation; - if (PRAutomation == null || conversationIds?.length === 0) { + if (conversationIds == null) { + await PRAutomation.deleteMany({ user: userId }); return; } - const fenced = { - user: userId, - stopCode: { $in: DELETION_STOP_CODES }, - ...(conversationIds != null && { conversationId: { $in: conversationIds } }), - }; - const records = await PRAutomation.find(fenced) - .select('conversationId') - .lean<{ conversationId: string }[]>(); - const ids = records.map(({ conversationId }) => conversationId); - for (let start = 0; start < ids.length; start += RELEASE_BATCH) { - const batch = ids.slice(start, start + RELEASE_BATCH); - const surviving: string[] = await mongoose.models.Conversation.distinct('conversationId', { + for (let start = 0; start < conversationIds.length; start += DELETE_BATCH) { + await PRAutomation.deleteMany({ user: userId, - conversationId: { $in: batch }, - ...activeExpirationFilter(), + conversationId: { $in: conversationIds.slice(start, start + DELETE_BATCH) }, }); - const kept = new Set(surviving); - const gone = batch.filter((id) => !kept.has(id)); - const survivors = batch.filter((id) => kept.has(id)); - if (survivors.length > 0) { - await PRAutomation.updateMany( - { ...fenced, conversationId: { $in: survivors } }, - { $set: { stopCode: 'deletion_aborted' } }, - ); - } - if (gone.length > 0) { - await PRAutomation.deleteMany({ ...fenced, conversationId: { $in: gone } }); - } - } - } - - /** Removes every record of a user, or of the listed conversations. */ - async function deletePRAutomations(userId: string, conversationIds?: string[]): Promise { - const PRAutomation = mongoose.models.PRAutomation; - if (PRAutomation == null || conversationIds?.length === 0) { - return; } - await PRAutomation.deleteMany({ - user: userId, - ...(conversationIds != null && { conversationId: { $in: conversationIds } }), - }); } async function setPRAutomationTrust( @@ -550,8 +487,6 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): claimPRAutomationRound, settlePRAutomationRound, stopPRAutomation, - stopPRAutomations, - releasePRAutomationFences, deletePRAutomations, setPRAutomationTrust, addPRAutomationBot, diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index f34295133c1..c1fabf08100 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -2204,23 +2204,17 @@ describe('User Methods - Database Tests', () => { }); describe('PR automation cleanup on account deletion', () => { - it('keeps the records, restartable, when the account delete fails', async () => { + it('keeps the records exactly as they were when the account delete fails', async () => { const user = await User.create({ email: 'delete-pr-fail@example.com', provider: 'local' }); const id = user._id.toString(); const PRAutomation = mongoose.models.PRAutomation; - await mongoose.models.Conversation.create({ - conversationId: 'chat-a', - user: id, - endpoint: 'agents', - }); await PRAutomation.create({ user: id, conversationId: 'chat-a', repository: 'acme/one' }); const deletion = jest.spyOn(User, 'deleteOne').mockRejectedValueOnce(new Error('db down')); try { await expect(methods.deleteUserById(id)).rejects.toThrow('db down'); expect(await User.exists({ _id: user._id })).not.toBeNull(); expect(await PRAutomation.findOne({ user: id }).lean()).toMatchObject({ - state: 'stopped', - stopCode: 'deletion_aborted', + state: 'idle', repository: 'acme/one', }); } finally { @@ -2228,6 +2222,29 @@ describe('PR automation cleanup on account deletion', () => { } }); + it('invalidates the auth cache before a cleanup that fails', async () => { + enableAuthUserDocCache(); + const user = await User.create({ email: 'delete-pr-cache@example.com', provider: 'local' }); + const id = user._id.toString(); + const cache = { + get: jest.fn().mockResolvedValue(['auth-cache-key-a']), + set: jest.fn().mockResolvedValue(true), + delete: jest.fn().mockResolvedValue(true), + }; + const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache }); + const cleanup = jest + .spyOn(mongoose.models.PRAutomation, 'deleteMany') + .mockRejectedValueOnce(new Error('synthetic cleanup failure')); + try { + await expect(methodsWithCache.deleteUserById(id)).rejects.toThrow( + 'synthetic cleanup failure', + ); + expect(cache.delete).toHaveBeenCalledWith(`${AUTH_USER_DOC_BY_ID_PREFIX}:${id}`); + } finally { + cleanup.mockRestore(); + } + }); + it('converges on a retry when the cleanup fails after the account was deleted', async () => { const user = await User.create({ email: 'delete-pr-retry@example.com', provider: 'local' }); const id = user._id.toString(); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 4fb9e76ed2b..28dbb4efe04 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -646,27 +646,18 @@ export function createUserMethods( try { const User = mongoose.models.User; await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); - /** Fenced first so a webhook cannot claim a record for an account that is going away. - * The records are removed only once the account delete committed, so a delete that fails - * keeps the user's automation: the fence is released into an ordinary stop they can - * restart. A cleanup that fails after the commit throws, and the retry removes them. */ - await prAutomation.stopPRAutomations(userId, 'account_deleting'); - let result: Awaited>; - try { - result = await User.deleteOne({ _id: userId }); - } catch (error) { - await prAutomation.releasePRAutomationFences(userId).catch(() => { - logger.warn( - '[deleteUserById] PR automation fence release failed; the records stay stopped.', - ); - }); - throw error; + const result = await User.deleteOne({ _id: userId }); + if (result.deletedCount > 0) { + /** Before any fallible cleanup, so a stale cached `req.user` cannot outlive the account. */ + await invalidateAuthUserDocCache(userId); } + /** Removed only after the account delete committed, so a failed delete keeps the user's + * automation. A claim refuses a deleted owner, and a failed cleanup throws so the caller + * retries; a retry on a deleted account still reaches this line. */ await prAutomation.deletePRAutomations(userId); if (result.deletedCount === 0) { return { deletedCount: 0, message: 'No user found with that ID.' }; } - await invalidateAuthUserDocCache(userId); return { deletedCount: result.deletedCount, message: 'User was deleted successfully.' }; } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 62f80bb22ca..2505a9a9f5b 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -15,10 +15,11 @@ const trustedBotSchema = new Schema( const prAutomationSchema: Schema = new Schema( { + /** Served by the unique `{ user, conversationId }` index below; a second index only adds + * write cost. */ user: { type: String, required: true, - index: true, }, tenantId: { type: String, diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index c61c5a8f784..8eac84f7a69 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -89,6 +89,7 @@ export type PRAutomationClaimErrorCode = | 'not_active' | 'binding_mismatch' | 'conversation_gone' + | 'owner_inactive' | 'round_cap' | 'time_cap' | 'stale_head'; From 3c0650de80feebe0571749191ba07a447553f667 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:28:16 +0200 Subject: [PATCH 09/25] fix: Undo Rejected PR Automation Claims and Fence Event Stops by Run --- .../src/methods/conversation.spec.ts | 7 +- .../src/methods/prAutomation.spec.ts | 215 ++++++++++++++---- .../data-schemas/src/methods/prAutomation.ts | 128 ++++++++--- .../src/methods/user.methods.spec.ts | 21 +- packages/data-schemas/src/methods/user.ts | 14 +- .../data-schemas/src/schema/prAutomation.ts | 2 + .../data-schemas/src/types/prAutomation.ts | 21 +- 7 files changed, 322 insertions(+), 86 deletions(-) diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index f57d45a66f8..9bd94db2f55 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -3749,10 +3749,11 @@ describe('Conversation Operations', () => { const conversationId = uuidv4(); await Conversation.create({ conversationId, user, endpoint: EModelEndpoint.agents }); const automation = createPRAutomationMethods(mongoose); - await automation.enablePRAutomation({ - userId: user, + await mongoose.models.PRAutomation.create({ + user, conversationId, - binding: { repository: 'acme/one', pullNumber: 1 }, + repository: 'acme/one', + pullNumber: 1, }); jest.spyOn(Conversation, 'deleteMany').mockRejectedValueOnce(new Error('delete failed')); await expect(deleteConvos(user, { conversationId })).rejects.toThrow('delete failed'); diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index a4d15423294..815e2fc85a4 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -2,6 +2,7 @@ import mongoose from 'mongoose'; import { logger, createModels } from '..'; import { MongoMemoryServer } from 'mongodb-memory-server'; import type { PRAutomationStopCode } from 'librechat-data-provider'; +import type { PRAutomationEventStopCode } from '~/types/prAutomation'; import { createPRAutomationMethods } from './prAutomation'; logger.silent = true; @@ -21,7 +22,15 @@ const pullTwo = { repository: 'acme/one', pullNumber: 2 }; const otherRepository = { repository: 'acme/two', pullNumber: 2 }; const mismatch = { ok: false, error: { code: 'binding_mismatch' } }; -const enable = (binding = pullOne) => methods.enablePRAutomation({ ...key, binding }); +/** Enables and returns the record, for setups that expect the enable to succeed. */ +const enableWith = async (params: Parameters[0]) => { + const result = await methods.enablePRAutomation(params); + if (!result.ok) { + throw new Error(`enable failed: ${result.error.code}`); + } + return result.value; +}; +const enable = (binding = pullOne) => enableWith({ ...key, binding }); const claim = (n: number, extra: { now?: Date; binding?: typeof pullOne } = {}) => methods.claimPRAutomationRound({ ...key, @@ -56,6 +65,43 @@ const seedConversation = (fields: Record = {}) => ...fields, }); +/** Runs `during` right after the claim's atomic write, before the claim checks again. */ +function interleave(during: () => Promise) { + const original = PRAutomation.findOneAndUpdate.bind(PRAutomation); + return jest.spyOn(PRAutomation, 'findOneAndUpdate').mockImplementationOnce((( + ...args: Parameters + ) => { + const query = original(...args); + const lean = query.lean.bind(query); + query.lean = ((...leanArgs: Parameters) => { + const result = lean(...leanArgs); + return { + ...result, + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + result + .then(async (value: unknown) => { + await during(); + return value; + }) + .then(resolve, reject), + }; + }) as typeof query.lean; + return query; + }) as typeof PRAutomation.findOneAndUpdate); +} +/** What an event handler captured about the run it acts on. */ +const fenceFor = async (binding = pullOne) => ({ + ...binding, + epoch: (await methods.getPRAutomation(key))?.epoch as string, +}); +const startAccountDeletionNow = () => + mongoose.models.User.updateOne( + { _id: userId }, + { $set: { agentTriggerDeletionStartedAt: new Date() } }, + ); +const cancelAccountDeletion = () => + mongoose.models.User.updateOne({ _id: userId }, { $unset: { agentTriggerDeletionStartedAt: 1 } }); + beforeAll(async () => { mongoServer = await MongoMemoryServer.create(); await mongoose.connect(mongoServer.getUri()); @@ -137,7 +183,7 @@ describe('enablePRAutomation', () => { test('clears approved bots when the conversation is bound to a different repository', async () => { await methods.enablePRAutomation({ ...key, binding: pullOne }); await addBot(101); - const rebound = await methods.enablePRAutomation({ ...key, binding: otherRepository }); + const rebound = await enableWith({ ...key, binding: otherRepository }); expect(rebound).toMatchObject({ ...otherRepository, trustedBots: [] }); }); @@ -150,7 +196,7 @@ describe('enablePRAutomation', () => { test('keeps approved bots when the same repository is bound to another pull request', async () => { await methods.enablePRAutomation({ ...key, binding: pullOne }); await addBot(101); - const rebound = await methods.enablePRAutomation({ ...key, binding: pullTwo }); + const rebound = await enableWith({ ...key, binding: pullTwo }); expect(rebound).toMatchObject({ pullNumber: 2, trustedBots: [{ id: 101 }] }); }); }); @@ -181,6 +227,13 @@ describe('claiming while the owner is being deleted', () => { expect(await claim(1)).toEqual({ ok: false, error: { code: 'owner_inactive' } }); }); + test('removes the record of an owner that no longer exists', async () => { + await enable(pullOne); + await mongoose.models.User.deleteMany({}); + await claim(1); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + test('claims again once the deletion was cancelled', async () => { await enable(pullOne); await startAccountDeletion(); @@ -194,31 +247,6 @@ describe('claiming while the owner is being deleted', () => { }); describe('a deletion that lands while a claim is in flight', () => { - /** Runs `during` right after the claim's atomic write, before the claim checks again. */ - const interleave = (during: () => Promise) => { - const original = PRAutomation.findOneAndUpdate.bind(PRAutomation); - return jest.spyOn(PRAutomation, 'findOneAndUpdate').mockImplementationOnce((( - ...args: Parameters - ) => { - const query = original(...args); - const lean = query.lean.bind(query); - query.lean = ((...leanArgs: Parameters) => { - const result = lean(...leanArgs); - return { - ...result, - then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => - result - .then(async (value: unknown) => { - await during(); - return value; - }) - .then(resolve, reject), - }; - }) as typeof query.lean; - return query; - }) as typeof PRAutomation.findOneAndUpdate); - }; - test('does not hand out a round for a conversation deleted after the first check', async () => { await enable(pullOne); const spy = interleave(() => mongoose.models.Conversation.deleteMany({})); @@ -257,7 +285,71 @@ describe('a deletion that lands while a claim is in flight', () => { } finally { spy.mockRestore(); } - expect((await methods.getPRAutomation(key))?.state).toBe('waiting'); + expect((await methods.getPRAutomation(key))?.state).not.toBe('fixing'); + }); + + /** Runs one claim whose owner deletion starts between its write and its second check. */ + const claimDuringDeletion = async (n: number) => { + const spy = interleave(startAccountDeletionNow); + try { + return await claim(n); + } finally { + spy.mockRestore(); + } + }; + + test('lets the same head claim again once a deletion that started mid-claim is cancelled', async () => { + await enable(pullOne); + await claimDuringDeletion(1); + await cancelAccountDeletion(); + expect((await claim(1)).ok).toBe(true); + }); + + test('does not spend a round on a claim it handed back', async () => { + await enable(pullOne); + await claimDuringDeletion(1); + expect((await methods.getPRAutomation(key))?.round).toBe(0); + }); + + test('does not open the time window for a claim it handed back', async () => { + await enable(pullOne); + await claimDuringDeletion(1); + expect((await methods.getPRAutomation(key))?.startedAt).toBeUndefined(); + }); + + test('hands a later claim back to exactly where the previous round left the record', async () => { + await enable(pullOne); + await startRound(1); + await toWaiting(); + await claimDuringDeletion(2); + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'waiting', + round: 1, + lastHeadSha: head(1), + claimedHeads: [head(1)], + }); + }); +}); + +describe('enabling for a conversation or owner that no longer exists', () => { + test('does not leave a record behind for a deleted conversation', async () => { + await mongoose.models.Conversation.deleteMany({}); + await methods.enablePRAutomation({ ...key, binding: pullOne }); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('reports that the conversation is gone', async () => { + await mongoose.models.Conversation.deleteMany({}); + expect(await methods.enablePRAutomation({ ...key, binding: pullOne })).toEqual({ + ok: false, + error: { code: 'conversation_gone' }, + }); + }); + + test('does not leave a record behind for an owner that no longer exists', async () => { + await mongoose.models.User.deleteMany({}); + await methods.enablePRAutomation({ ...key, binding: pullOne }); + expect(await methods.getPRAutomation(key)).toBeNull(); }); }); @@ -429,7 +521,7 @@ describe('binding a pull request', () => { await toWaiting(); await startRound(2); - const rebound = await methods.enablePRAutomation({ ...key, binding: pullTwo }); + const rebound = await enableWith({ ...key, binding: pullTwo }); expect(rebound).toMatchObject({ pullNumber: 2, state: 'idle', round: 0, claimedHeads: [] }); expect(rebound.startedAt).toBeUndefined(); expect(rebound.lastHeadSha).toBeUndefined(); @@ -464,7 +556,7 @@ describe('binding a pull request', () => { test('leaves a run alone when it is bound to the pair it already has', async () => { await methods.enablePRAutomation({ ...key, binding: pullOne }); await startRound(1); - const again = await methods.enablePRAutomation({ ...key, binding: pullOne }); + const again = await enableWith({ ...key, binding: pullOne }); expect(again).toMatchObject({ state: 'fixing', round: 1 }); }); @@ -512,6 +604,17 @@ describe('claimPRAutomationRound', () => { expect((await methods.getPRAutomation(key))?.round).toBe(2); }); + test('reports a retryable conflict when the running round settles before the fallback read', async () => { + await enable(); + await claim(1); + const spy = interleave(toWaiting); + try { + expect(await claim(2)).toEqual({ ok: false, error: { code: 'conflict' } }); + } finally { + spy.mockRestore(); + } + }); + test('rejects a claim while a round is already running', async () => { await enable(); await claim(1); @@ -650,8 +753,8 @@ describe('claiming for a conversation that no longer exists', () => { await enable(); await mongoose.models.Conversation.deleteMany({}); await claim(1); - await enable(); await seedConversation(); + await enable(); expect(await startRound(1)).toMatchObject({ round: 1 }); }); @@ -739,7 +842,7 @@ describe('settlePRAutomationRound', () => { describe('stopPRAutomation', () => { test('records the stop code', async () => { await enable(); - expect(await methods.stopPRAutomation(key, 'round_cap', pullOne)).toMatchObject({ + expect(await methods.stopPRAutomation(key, 'round_cap', await fenceFor())).toMatchObject({ state: 'stopped', stopCode: 'round_cap', }); @@ -756,7 +859,7 @@ describe('stopPRAutomation', () => { test('keeps the first stop code when it is stopped again', async () => { await enable(); - await methods.stopPRAutomation(key, 'round_cap', pullOne); + await methods.stopPRAutomation(key, 'round_cap', await fenceFor()); expect(await methods.stopPRAutomation(key, 'user_stopped')).toBeNull(); expect((await methods.getPRAutomation(key))?.stopCode).toBe('round_cap'); }); @@ -764,7 +867,11 @@ describe('stopPRAutomation', () => { test('rejects a code outside the stop code list', async () => { await enable(); await expect( - methods.stopPRAutomation(key, 'because' as unknown as PRAutomationStopCode, pullOne), + methods.stopPRAutomation( + key, + 'because' as unknown as PRAutomationEventStopCode, + await fenceFor(), + ), ).rejects.toThrow(); }); @@ -775,19 +882,23 @@ describe('stopPRAutomation', () => { test('ignores an event-driven stop for a pull request the record was rebound away from', async () => { await enable(pullOne); await enable(pullTwo); - expect(await methods.stopPRAutomation(key, 'pull_request_closed', pullOne)).toBeNull(); + expect(await methods.stopPRAutomation(key, 'pull_request_closed', await fenceFor())).toBeNull(); expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'idle', pullNumber: 2 }); }); test('ignores an event-driven stop for another repository', async () => { await enable(pullOne); - expect(await methods.stopPRAutomation(key, 'pull_request_closed', otherRepository)).toBeNull(); + expect( + await methods.stopPRAutomation(key, 'pull_request_closed', await fenceFor(otherRepository)), + ).toBeNull(); expect((await methods.getPRAutomation(key))?.state).toBe('idle'); }); test('applies an event-driven stop for the pull request the record is bound to', async () => { await enable(pullOne); - expect(await methods.stopPRAutomation(key, 'pull_request_closed', pullOne)).toMatchObject({ + expect( + await methods.stopPRAutomation(key, 'pull_request_closed', await fenceFor()), + ).toMatchObject({ state: 'stopped', stopCode: 'pull_request_closed', }); @@ -806,6 +917,33 @@ describe('stopPRAutomation', () => { expect((await methods.getPRAutomation(key))?.state).toBe('idle'); }); + test('applies a user stop that names a pull request the record was rebound away from', async () => { + await enable(pullOne); + await enable(pullTwo); + const stop = methods.stopPRAutomation as ( + target: typeof key, + code: PRAutomationStopCode, + binding: typeof pullOne, + ) => Promise; + expect(await stop(key, 'user_stopped', pullOne)).toMatchObject({ state: 'stopped' }); + }); + + test('ignores an event stop from a run that was stopped and restarted on the same pull request', async () => { + await enable(pullOne); + const previous = await fenceFor(); + await methods.stopPRAutomation(key, 'user_stopped'); + await enable(pullOne); + expect(await methods.stopPRAutomation(key, 'pull_request_closed', previous)).toBeNull(); + }); + + test('ignores an event stop from a record that was disabled and enabled again', async () => { + await enable(pullOne); + const previous = await fenceFor(); + await methods.disablePRAutomation(key); + await enable(pullOne); + expect(await methods.stopPRAutomation(key, 'pull_request_closed', previous)).toBeNull(); + }); + test('lets a user stop win even after the record was rebound', async () => { await enable(pullOne); await enable(pullTwo); @@ -933,6 +1071,7 @@ describe('approved bots', () => { test('keeps each conversation allowlist separate', async () => { await enable(); + await seedConversation({ conversationId: 'convo-2' }); await methods.enablePRAutomation({ userId, conversationId: 'convo-2' }); await addBot(101); const other = await methods.getPRAutomation({ userId, conversationId: 'convo-2' }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index db45ca8c739..fa21a2b9883 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -13,19 +13,26 @@ const RESTARTABLE = { state: 'stopped' }; /** Records removed per write, so a cleanup never holds one unbounded `$in`. */ const DELETE_BATCH = 1000; -type Liveness = 'live' | 'conversation_gone' | 'owner_inactive'; +/** + * `owner_inactive` is an account deletion in progress, which can still be cancelled, so the + * record is kept. `owner_gone` and `conversation_gone` can never run again. + */ +type Liveness = 'live' | 'conversation_gone' | 'owner_gone' | 'owner_inactive'; +type Gone = Extract; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; -/** Everything that belongs to one run on one pull request. */ -const RUN_RESET = { - $set: { state: 'idle', round: 0, claimedHeads: [] }, +/** Everything that belongs to one run on one pull request, under a new epoch. */ +const runReset = (fields: Record = {}) => ({ + $set: { state: 'idle', round: 0, claimedHeads: [], epoch: randomUUID(), ...fields }, $unset: { stopCode: '', startedAt: '', lastHeadSha: '', runId: '' }, -}; +}); +const isGone = (liveness: Liveness): liveness is Gone => + liveness === 'conversation_gone' || liveness === 'owner_gone'; export function createPRAutomationMethods(mongoose: typeof import('mongoose')): { getPRAutomation: (key: t.PRAutomationKey) => Promise; - enablePRAutomation: (params: t.EnablePRAutomationParams) => Promise; + enablePRAutomation: (params: t.EnablePRAutomationParams) => Promise; disablePRAutomation: (key: t.PRAutomationKey) => Promise<{ removed: boolean }>; claimPRAutomationRound: ( params: t.ClaimPRAutomationRoundParams, @@ -37,8 +44,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): (key: t.PRAutomationKey, stopCode: 'user_stopped'): Promise; ( key: t.PRAutomationKey, - stopCode: PRAutomationStopCode, - binding: t.PRAutomationBinding, + stopCode: t.PRAutomationEventStopCode, + fence: t.PRAutomationRunFence, ): Promise; }; deletePRAutomations: (userId: string, conversationIds?: string[]) => Promise; @@ -86,8 +93,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * deletion has started (the same durable marker agent triggers use), and the conversation * and every conversation that controls it still exist and are inside their retention. A * subagent thread is controlled by its parent and its root, and a deletion removes the - * root before it discovers the descendants. Deletions do not fence the record; the claim - * asks this on both sides of its write instead. + * root before it discovers the descendants. Deletions do not fence the record; enable and + * claim ask this on both sides of their write instead. */ async function checkLiveness(key: t.PRAutomationKey): Promise { if (!isValidObjectIdString(key.userId)) { @@ -96,15 +103,17 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const Conversation = mongoose.models.Conversation; const active = activeExpirationFilter(); const [owner, conversation] = await Promise.all([ - mongoose.models.User.exists({ - _id: key.userId, - agentTriggerDeletionStartedAt: { $exists: false }, - }), + mongoose.models.User.findById(key.userId) + .select('agentTriggerDeletionStartedAt') + .lean<{ agentTriggerDeletionStartedAt?: Date }>(), Conversation.findOne({ user: key.userId, conversationId: key.conversationId, ...active }) .select('subagentThread') .lean<{ subagentThread?: { rootConversationId: string; parentConversationId: string } }>(), ]); if (owner == null) { + return 'owner_gone'; + } + if (owner.agentTriggerDeletionStartedAt != null) { return 'owner_inactive'; } if (conversation == null) { @@ -123,18 +132,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return activeAncestors === ancestors.length ? 'live' : 'conversation_gone'; } - /** A record whose conversation is gone can never run again, so it is removed. */ + /** A missing owner reads as inactive to callers; the distinction only decides removal. */ + const goneCode = (liveness: Gone) => + liveness === 'owner_gone' ? 'owner_inactive' : 'conversation_gone'; + + /** + * A record whose conversation or owner is gone can never run again, so it is removed. This + * also converges a cleanup that failed after the delete committed. + */ async function refuseClaim( key: t.PRAutomationKey, liveness: Exclude, ): Promise { - if (liveness === 'owner_inactive') { - return { ok: false, error: { code: 'owner_inactive' } }; + if (!isGone(liveness)) { + return { ok: false, error: { code: liveness } }; } const removed = await mongoose.models.PRAutomation.deleteOne(keyFilter(key)); return { ok: false, - error: { code: removed.deletedCount > 0 ? 'conversation_gone' : 'not_found' }, + error: { code: removed.deletedCount > 0 ? goneCode(liveness) : 'not_found' }, }; } @@ -164,19 +180,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * A different pull request in the same repository keeps the bots and starts a fresh * run, because the round count, the time window and the claimed heads describe the * previous pull request. + * + * Liveness is checked on both sides of the writes, like a claim. An enable for a conversation + * or owner that is already gone writes nothing. One that passed the first check but lost a + * race with the deletion and its cleanup removes the record it wrote, so a late enable cannot + * recreate a record nobody will clean up. */ async function enablePRAutomation({ trust, binding, ...key - }: t.EnablePRAutomationParams): Promise { + }: t.EnablePRAutomationParams): Promise { + const before = await checkLiveness(key); + if (before !== 'live') { + return { ok: false, error: { code: isGone(before) ? goneCode(before) : before } }; + } await ensureIndexes(); const PRAutomation = mongoose.models.PRAutomation; const filter = keyFilter(key); - const reset = (fields: Record) => ({ - $set: { ...RUN_RESET.$set, ...fields }, - $unset: RUN_RESET.$unset, - }); + const reset = runReset; const options = { runValidators: true }; await PRAutomation.updateOne( filter, @@ -187,6 +209,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): round: 0, trustedBots: [], claimedHeads: [], + epoch: randomUUID(), trust: trust ?? 'approvedBots', ...binding, }, @@ -225,11 +248,16 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): options, ); } + const after = await checkLiveness(key); + if (isGone(after)) { + await PRAutomation.deleteOne(filter); + return { ok: false, error: { code: goneCode(after) } }; + } const record = await getPRAutomation(key); if (record == null) { throw new Error('PR automation record missing after enable'); } - return record; + return { ok: true, value: record }; } async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { @@ -300,16 +328,32 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (claimed != null) { /** * A deletion that started between the first check and the write is seen here. The - * claimed round is handed back to `waiting` under its own run id, so it never starts - * work, and a record whose conversation is gone is removed. + * whole claim is undone under its own run id, so it never starts work and a cancelled + * deletion leaves the record as it was: the round, the window and the head are not + * spent. Rounds and heads are claimed one for one and the window opens with the first + * round, so the state before the claim follows from the claimed record. A record whose + * conversation or owner is gone is removed. */ const after = await checkLiveness(key); if (after === 'live') { return { ok: true, value: { ...claimed, runId } }; } + const previousHeads = claimed.claimedHeads.slice(0, -1); + const lastHeadSha = previousHeads[previousHeads.length - 1]; await PRAutomation.updateOne( - { ...keyFilter(key), runId, state: 'fixing' }, - { $set: { state: 'waiting' } }, + { ...keyFilter(key), runId, state: 'fixing', round: claimed.round }, + { + $set: { + state: lastHeadSha == null ? 'idle' : 'waiting', + round: claimed.round - 1, + claimedHeads: previousHeads, + ...(lastHeadSha != null && { lastHeadSha }), + }, + $unset: { + runId: '', + ...(lastHeadSha == null && { startedAt: '', lastHeadSha: '' }), + }, + }, ); return refuseClaim(key, after); } @@ -330,7 +374,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (current.startedAt != null && current.startedAt < cutoff) { return { ok: false, error: { code: 'time_cap' } }; } - return { ok: false, error: { code: 'stale_head' } }; + if (current.claimedHeads.includes(headSha)) { + return { ok: false, error: { code: 'stale_head' } }; + } + /** Every condition passes now, so the record changed between the write and this read. */ + return { ok: false, error: { code: 'conflict' } }; } /** @@ -366,23 +414,29 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * maps it to the reason it shows. The first stop wins, and a stopped record leaves that * state only through `enablePRAutomation`. * - * A user stop is unconditional, so it is never lost to a race. Every other code comes from - * an event about one pull request, and a delayed one must not stop the run that replaced - * it, so it names the pull request it is for and applies only while the record is still - * bound to it. + * A user stop is unconditional, so it is never lost to a race; any fence passed with it is + * ignored. Every other code comes from an event about one run, and a delayed one must not + * stop the run that replaced it, so it names the pull request and the epoch it saw and + * applies only while both are current. A restart on the same pull request starts a new + * epoch, so a stop meant for the previous run cannot end the new one. */ async function stopPRAutomation( key: t.PRAutomationKey, stopCode: PRAutomationStopCode, - binding?: t.PRAutomationBinding, + fence?: t.PRAutomationRunFence, ): Promise { assertStopCode(stopCode); - if (stopCode !== 'user_stopped' && binding == null) { - throw new RangeError('An event-driven stop must name the pull request it is for'); + const user = stopCode === 'user_stopped'; + if (!user && (fence?.repository == null || fence.pullNumber == null || !fence.epoch)) { + throw new RangeError('An event-driven stop must name the pull request and run it is for'); } + const condition = + user || fence == null + ? {} + : { repository: fence.repository, pullNumber: fence.pullNumber, epoch: fence.epoch }; const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( - { ...keyFilter(key), ...binding, state: { $ne: 'stopped' } }, + { ...keyFilter(key), ...condition, state: { $ne: 'stopped' } }, { $set: { state: 'stopped', stopCode } }, { new: true, select: PROJECTION }, ).lean(); diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index c1fabf08100..9acc6d3d060 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -2236,15 +2236,28 @@ describe('PR automation cleanup on account deletion', () => { .spyOn(mongoose.models.PRAutomation, 'deleteMany') .mockRejectedValueOnce(new Error('synthetic cleanup failure')); try { - await expect(methodsWithCache.deleteUserById(id)).rejects.toThrow( - 'synthetic cleanup failure', - ); + await methodsWithCache.deleteUserById(id); expect(cache.delete).toHaveBeenCalledWith(`${AUTH_USER_DOC_BY_ID_PREFIX}:${id}`); } finally { cleanup.mockRestore(); } }); + it('reports the account deleted when only the cleanup after the delete fails', async () => { + const user = await User.create({ email: 'delete-pr-committed@example.com', provider: 'local' }); + const id = user._id.toString(); + const PRAutomation = mongoose.models.PRAutomation; + await PRAutomation.create({ user: id, conversationId: 'chat-a' }); + const cleanup = jest + .spyOn(PRAutomation, 'deleteMany') + .mockRejectedValueOnce(new Error('synthetic cleanup failure')); + try { + await expect(methods.deleteUserById(id)).resolves.toMatchObject({ deletedCount: 1 }); + } finally { + cleanup.mockRestore(); + } + }); + it('converges on a retry when the cleanup fails after the account was deleted', async () => { const user = await User.create({ email: 'delete-pr-retry@example.com', provider: 'local' }); const id = user._id.toString(); @@ -2254,7 +2267,7 @@ describe('PR automation cleanup on account deletion', () => { .spyOn(PRAutomation, 'deleteMany') .mockRejectedValueOnce(new Error('synthetic cleanup failure')); try { - await expect(methods.deleteUserById(id)).rejects.toThrow('synthetic cleanup failure'); + await methods.deleteUserById(id); expect(await User.exists({ _id: user._id })).toBeNull(); expect(await PRAutomation.countDocuments({ user: id })).toBe(1); await expect(methods.deleteUserById(id)).resolves.toMatchObject({ deletedCount: 0 }); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 28dbb4efe04..b9fadcbc109 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -652,9 +652,17 @@ export function createUserMethods( await invalidateAuthUserDocCache(userId); } /** Removed only after the account delete committed, so a failed delete keeps the user's - * automation. A claim refuses a deleted owner, and a failed cleanup throws so the caller - * retries; a retry on a deleted account still reaches this line. */ - await prAutomation.deletePRAutomations(userId); + * automation. Once the account is gone a cleanup failure must not report the deletion as + * failed: the next claim removes any record whose owner no longer exists. A retry on an + * account that was already gone has committed nothing, so its failure still throws. */ + try { + await prAutomation.deletePRAutomations(userId); + } catch (error) { + if (result.deletedCount === 0) { + throw error; + } + logger.warn('[deleteUserById] PR automation cleanup deferred to its next claim', error); + } if (result.deletedCount === 0) { return { deletedCount: 0, message: 'No user found with that ID.' }; } diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 2505a9a9f5b..b7b4d732552 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -1,4 +1,5 @@ import { Schema } from 'mongoose'; +import { randomUUID } from 'node:crypto'; import { PR_AUTOMATION_STATES } from 'librechat-data-provider'; import { MAX_PR_AUTOMATION_BOTS } from 'librechat-data-provider'; import { PR_AUTOMATION_STOP_CODES } from 'librechat-data-provider'; @@ -65,6 +66,7 @@ const prAutomationSchema: Schema = new Schema( lastHeadSha: { type: String, maxlength: 64 }, claimedHeads: { type: [{ type: String, maxlength: 64 }], default: [] }, runId: { type: String, maxlength: 64 }, + epoch: { type: String, maxlength: 64, required: true, default: () => randomUUID() }, }, { timestamps: true }, ); diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 8eac84f7a69..fb8f54bfc43 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -34,6 +34,12 @@ export interface IPRAutomation { * cannot move the run that replaced it. */ runId?: string; + /** + * Identifies the run: a new value on every enable that starts one, every restart and every + * rebind. An event-driven stop names it, so a stop meant for an earlier run on the same pull + * request cannot end the current one. + */ + epoch: string; /** * Every head a round has been claimed for since the last restart. A delayed * delivery of an earlier head is rejected against this list. It holds at most @@ -57,6 +63,17 @@ export interface PRAutomationBinding { pullNumber: number; } +/** What an event handler saw about the run it acts on. */ +export interface PRAutomationRunFence extends PRAutomationBinding { + epoch: string; +} + +export type PRAutomationEventStopCode = Exclude; + +export type EnablePRAutomationResult = + | { ok: true; value: IPRAutomation } + | { ok: false; error: { code: 'conversation_gone' | 'owner_inactive' } }; + export interface EnablePRAutomationParams extends PRAutomationKey { trust?: PRAutomationTrustLevel; binding?: PRAutomationBinding; @@ -92,7 +109,9 @@ export type PRAutomationClaimErrorCode = | 'owner_inactive' | 'round_cap' | 'time_cap' - | 'stale_head'; + | 'stale_head' + /** The record changed while the claim was being decided; the delivery may be retried. */ + | 'conflict'; export type ClaimPRAutomationRoundResult = | { ok: true; value: IPRAutomation & { runId: string } } From 1e141932376307405fcf3fc14fcd59345765d304 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:49:11 +0200 Subject: [PATCH 10/25] fix: Walk the Full Subagent Lineage and Fence PR Automation Enables and Approvals --- .../src/methods/prAutomation.spec.ts | 123 +++++++++++++++++- .../data-schemas/src/methods/prAutomation.ts | 98 ++++++++++---- .../data-schemas/src/types/prAutomation.ts | 6 + 3 files changed, 202 insertions(+), 25 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 815e2fc85a4..80e5bf0c116 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -55,8 +55,11 @@ const toWaiting = () => { user: userId, conversationId: key.conversationId }, { $set: { state: 'waiting' } }, ); -const addBot = (id: number, login?: string, limit = maxBots, repository = 'acme/one') => - methods.addPRAutomationBot(key, login == null ? { id } : { id, login }, limit, repository); +const addBot = async (id: number, login?: string, limit = maxBots, repository = 'acme/one') => + methods.addPRAutomationBot(key, login == null ? { id } : { id, login }, limit, { + repository, + epoch: (await methods.getPRAutomation(key))?.epoch ?? 'none', + }); const seedConversation = (fields: Record = {}) => mongoose.models.Conversation.create({ conversationId: key.conversationId, @@ -346,6 +349,58 @@ describe('enabling for a conversation or owner that no longer exists', () => { }); }); + /** Runs `during` right after the enable's first write, before it checks again. */ + const interleaveEnable = (during: () => Promise) => { + const original = PRAutomation.updateOne.bind(PRAutomation); + return jest.spyOn(PRAutomation, 'updateOne').mockImplementationOnce((( + ...args: Parameters + ) => { + const query = original(...args); + const then = query.then.bind(query); + query.then = ((resolve, reject) => + then(async (value) => { + await during(); + return value; + }).then(resolve, reject)) as typeof query.then; + return query; + }) as typeof PRAutomation.updateOne); + }; + + test('rejects an enable once account deletion started mid-write', async () => { + const spy = interleaveEnable(startAccountDeletionNow); + try { + expect(await methods.enablePRAutomation({ ...key, binding: pullOne })).toEqual({ + ok: false, + error: { code: 'owner_inactive' }, + }); + } finally { + spy.mockRestore(); + } + }); + + test('leaves nothing claimable after a cancelled deletion that an enable crossed', async () => { + const spy = interleaveEnable(startAccountDeletionNow); + try { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + } finally { + spy.mockRestore(); + } + await cancelAccountDeletion(); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_found' } }); + }); + + test('keeps a record the crossing enable did not change', async () => { + await enable(pullOne); + await startRound(1); + const spy = interleaveEnable(startAccountDeletionNow); + try { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + } finally { + spy.mockRestore(); + } + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'fixing', round: 1 }); + }); + test('does not leave a record behind for an owner that no longer exists', async () => { await mongoose.models.User.deleteMany({}); await methods.enablePRAutomation({ ...key, binding: pullOne }); @@ -461,6 +516,38 @@ describe('claiming for a subagent thread whose root conversation is gone', () => ).toEqual({ ok: false, error: { code: 'conversation_gone' } }); }); + test('rejects a claim for a descendant once an ancestor between its parent and root was deleted', async () => { + await seedChild(); + const thread = (conversationId: string, parentConversationId: string, depth: number) => + mongoose.models.Conversation.create({ + conversationId, + user: userId, + endpoint: 'agents', + subagentThread: { + rootConversationId: key.conversationId, + parentConversationId, + parentMessageId: `message-${depth}`, + parentToolCallId: `call-${depth}`, + subagentType: 'agent-child', + subagentKind: 'agent', + depth, + }, + }); + await thread('grandchild-1', childKey.conversationId, 2); + await thread('great-1', 'grandchild-1', 3); + const greatKey = { userId, conversationId: 'great-1' }; + await methods.enablePRAutomation({ ...greatKey, binding: pullOne }); + await mongoose.models.Conversation.deleteOne({ conversationId: childKey.conversationId }); + expect( + await methods.claimPRAutomationRound({ + ...greatKey, + ...limits, + binding: pullOne, + headSha: head(1), + }), + ).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + }); + test('accepts a claim for a child while its root is active', async () => { await seedChild(); await methods.enablePRAutomation({ ...childKey, binding: pullOne }); @@ -615,6 +702,30 @@ describe('claimPRAutomationRound', () => { } }); + test('refuses a head that is not a commit SHA before writing anything', async () => { + await enable(); + await expect( + methods.claimPRAutomationRound({ + ...key, + ...limits, + binding: pullOne, + headSha: 'a'.repeat(65), + }), + ).rejects.toThrow(RangeError); + expect((await methods.getPRAutomation(key))?.claimedHeads).toEqual([]); + }); + + test('accepts a SHA-256 head', async () => { + await enable(); + const result = await methods.claimPRAutomationRound({ + ...key, + ...limits, + binding: pullOne, + headSha: 'b'.repeat(64), + }); + expect(result.ok).toBe(true); + }); + test('rejects a claim while a round is already running', async () => { await enable(); await claim(1); @@ -970,6 +1081,14 @@ describe('setPRAutomationTrust', () => { }); describe('approved bots', () => { + test('refuses an approval from before the record was rebound away and back', async () => { + await enable(pullOne); + const stale = { repository: 'acme/one', epoch: (await methods.getPRAutomation(key))!.epoch }; + await enable(otherRepository); + await enable(pullOne); + expect(await methods.addPRAutomationBot(key, { id: 7 }, maxBots, stale)).toEqual(mismatch); + }); + test('adds a bot by numeric id', async () => { await enable(); expect(await addBot(101, 'review-bot[bot]')).toMatchObject({ diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index fa21a2b9883..41dfa1e95a1 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -1,5 +1,9 @@ import { randomUUID } from 'node:crypto'; -import { PR_AUTOMATION_STOP_CODES, MAX_PR_AUTOMATION_BOTS } from 'librechat-data-provider'; +import { + MAX_SUBAGENT_DEPTH, + MAX_PR_AUTOMATION_BOTS, + PR_AUTOMATION_STOP_CODES, +} from 'librechat-data-provider'; import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationState } from 'librechat-data-provider'; @@ -12,6 +16,8 @@ const PROJECTION = '-_id -__v'; const RESTARTABLE = { state: 'stopped' }; /** Records removed per write, so a cleanup never holds one unbounded `$in`. */ const DELETE_BATCH = 1000; +/** A full SHA-1 or SHA-256 commit id; anything else is refused before it is stored. */ +const COMMIT_SHA = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i; /** * `owner_inactive` is an account deletion in progress, which can still be cancelled, so the @@ -57,7 +63,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): key: t.PRAutomationKey, bot: t.IPRAutomationBot, maxBots: number, - repository: string, + fence: t.PRAutomationBotFence, ) => Promise; removePRAutomationBot: ( key: t.PRAutomationKey, @@ -92,9 +98,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * Whether a round may run for this record right now: the owner exists and no account * deletion has started (the same durable marker agent triggers use), and the conversation * and every conversation that controls it still exist and are inside their retention. A - * subagent thread is controlled by its parent and its root, and a deletion removes the - * root before it discovers the descendants. Deletions do not fence the record; enable and - * claim ask this on both sides of their write instead. + * subagent thread is controlled by every ancestor up to its root, and a deletion removes + * one generation before it discovers the next, so the whole parent chain is walked. Depth + * is capped by `MAX_SUBAGENT_DEPTH`, which bounds the walk; a longer chain fails closed. + * Deletions do not fence the record; enable and claim ask this on both sides of their write + * instead. */ async function checkLiveness(key: t.PRAutomationKey): Promise { if (!isValidObjectIdString(key.userId)) { @@ -102,13 +110,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } const Conversation = mongoose.models.Conversation; const active = activeExpirationFilter(); + type Thread = { rootConversationId: string; parentConversationId: string }; + type Lineage = { conversationId: string; subagentThread?: Thread }; const [owner, conversation] = await Promise.all([ mongoose.models.User.findById(key.userId) .select('agentTriggerDeletionStartedAt') .lean<{ agentTriggerDeletionStartedAt?: Date }>(), Conversation.findOne({ user: key.userId, conversationId: key.conversationId, ...active }) - .select('subagentThread') - .lean<{ subagentThread?: { rootConversationId: string; parentConversationId: string } }>(), + .select('conversationId subagentThread') + .lean(), ]); if (owner == null) { return 'owner_gone'; @@ -119,17 +129,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (conversation == null) { return 'conversation_gone'; } - const thread = conversation.subagentThread; - if (thread == null) { - return 'live'; - } - const ancestors = [...new Set([thread.rootConversationId, thread.parentConversationId])]; - const activeAncestors = await Conversation.countDocuments({ - user: key.userId, - conversationId: { $in: ancestors }, - ...active, - }); - return activeAncestors === ancestors.length ? 'live' : 'conversation_gone'; + const rootConversationId = conversation.subagentThread?.rootConversationId; + let current: Lineage | null = conversation; + for (let hops = 0; current?.subagentThread != null; hops++) { + if (hops >= MAX_SUBAGENT_DEPTH) { + return 'conversation_gone'; + } + current = await Conversation.findOne({ + user: key.userId, + conversationId: current.subagentThread.parentConversationId, + ...active, + }) + .select('conversationId subagentThread') + .lean(); + } + if (current == null) { + return 'conversation_gone'; + } + const reachedRoot = rootConversationId == null || current.conversationId === rootConversationId; + return reachedRoot ? 'live' : 'conversation_gone'; } /** A missing owner reads as inactive to callers; the distinction only decides removal. */ @@ -199,6 +217,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const PRAutomation = mongoose.models.PRAutomation; const filter = keyFilter(key); const reset = runReset; + /** What the record was before this attempt, so a crossed deletion can undo only its write. */ + const previous = await PRAutomation.findOne(filter).lean(); const options = { runValidators: true }; await PRAutomation.updateOne( filter, @@ -253,6 +273,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): await PRAutomation.deleteOne(filter); return { ok: false, error: { code: goneCode(after) } }; } + if (after !== 'live') { + await undoEnable(filter, previous); + return { ok: false, error: { code: after } }; + } const record = await getPRAutomation(key); if (record == null) { throw new Error('PR automation record missing after enable'); @@ -260,6 +284,29 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: true, value: record }; } + /** + * Puts the record back as this enable found it when an account deletion started during the + * write, which can still be cancelled. The undo applies only while the record still carries + * the epoch the attempt left, so it never reverts a later enable. An attempt that changed + * nothing leaves the epoch as it was and is not touched. + */ + async function undoEnable( + filter: ReturnType, + previous: t.IPRAutomation | null, + ): Promise { + const PRAutomation = mongoose.models.PRAutomation; + const written = await PRAutomation.findOne(filter).select('epoch').lean<{ epoch: string }>(); + if (written == null || written.epoch === previous?.epoch) { + return; + } + const guard = { ...filter, epoch: written.epoch }; + if (previous == null) { + await PRAutomation.deleteOne(guard); + return; + } + await PRAutomation.replaceOne(guard, previous); + } + async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { const PRAutomation = mongoose.models.PRAutomation; const result = await PRAutomation.deleteOne(keyFilter(key)); @@ -288,6 +335,9 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): now = new Date(), ...key }: t.ClaimPRAutomationRoundParams): Promise { + if (!COMMIT_SHA.test(headSha)) { + throw new RangeError('headSha must be a full commit SHA'); + } const PRAutomation = mongoose.models.PRAutomation; const filter = { ...keyFilter(key), ...binding }; const cutoff = new Date(now.getTime() - maxMinutes * 60_000); @@ -479,15 +529,16 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Idempotent by numeric id. The limit is the configured `maxBots`, resolved by * the caller, and cannot exceed the schema ceiling. A login is stored for - * display only. The approval names the repository it was authorized against and - * applies only while the record is still bound to it, so an approval that - * overlaps a rebind cannot add a bot to a repository nobody approved it for. + * display only. The approval names the repository and the epoch it was authorized + * against and applies only while both are current, so an approval that overlaps a + * rebind, including one away and back to the same repository that cleared the list, + * cannot add a bot nobody approved for the current binding. */ async function addPRAutomationBot( key: t.PRAutomationKey, bot: t.IPRAutomationBot, maxBots: number, - repository: string, + { repository, epoch }: t.PRAutomationBotFence, ): Promise { if (!Number.isInteger(maxBots) || maxBots < 1 || maxBots > MAX_PR_AUTOMATION_BOTS) { throw new RangeError(`maxBots must be an integer from 1 to ${MAX_PR_AUTOMATION_BOTS}`); @@ -497,6 +548,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): { ...keyFilter(key), repository, + epoch, 'trustedBots.id': { $ne: bot.id }, [`trustedBots.${maxBots - 1}`]: { $exists: false }, }, @@ -511,7 +563,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (current == null) { return { ok: false, error: { code: 'not_found' } }; } - if (current.repository !== repository) { + if (current.repository !== repository || current.epoch !== epoch) { return { ok: false, error: { code: 'binding_mismatch' } }; } if (current.trustedBots.some((existing) => existing.id === bot.id)) { diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index fb8f54bfc43..2951b29b601 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -68,6 +68,12 @@ export interface PRAutomationRunFence extends PRAutomationBinding { epoch: string; } +/** What an approval was authorized against: the bound repository and its run epoch. */ +export interface PRAutomationBotFence { + repository: string; + epoch: string; +} + export type PRAutomationEventStopCode = Exclude; export type EnablePRAutomationResult = From 678cf91d2ca04d9b1c1d2eb82d036f011fa1e2d7 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:13:33 +0200 Subject: [PATCH 11/25] fix: Fence PR Automation Claims and Bot Removals by Epoch and Read Lineage in One Query --- .../src/methods/prAutomation.spec.ts | 107 ++++++++++++-- .../data-schemas/src/methods/prAutomation.ts | 137 +++++++++++------- .../data-schemas/src/types/prAutomation.ts | 5 +- 3 files changed, 181 insertions(+), 68 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 80e5bf0c116..76e6f29334f 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -31,13 +31,19 @@ const enableWith = async (params: Parameters[ return result.value; }; const enable = (binding = pullOne) => enableWith({ ...key, binding }); -const claim = (n: number, extra: { now?: Date; binding?: typeof pullOne } = {}) => +/** The current run of a record, as a webhook handler would read it before claiming. */ +const currentEpoch = async (target = key) => + (await methods.getPRAutomation(target))?.epoch ?? 'none'; +const claim = async ( + n: number, + { binding = pullOne, ...extra }: { now?: Date; binding?: typeof pullOne; epoch?: string } = {}, +) => methods.claimPRAutomationRound({ ...key, ...limits, - binding: pullOne, headSha: head(n), ...extra, + binding: { ...binding, epoch: extra.epoch ?? (await currentEpoch()) }, }); /** Claims a round that must succeed and returns the record it started. */ const startRound = async (n: number) => { @@ -68,10 +74,17 @@ const seedConversation = (fields: Record = {}) => ...fields, }); -/** Runs `during` right after the claim's atomic write, before the claim checks again. */ -function interleave(during: () => Promise) { - const original = PRAutomation.findOneAndUpdate.bind(PRAutomation); - return jest.spyOn(PRAutomation, 'findOneAndUpdate').mockImplementationOnce((( +/** + * Runs `during` right after the next `findOneAndUpdate` of `model` resolves, which for the + * default is the claim's atomic write, before the claim checks again. + */ +function interleave( + during: () => Promise, + model: mongoose.Model = PRAutomation, + method: 'findOneAndUpdate' | 'findById' = 'findOneAndUpdate', +) { + const original = (model[method] as typeof PRAutomation.findOneAndUpdate).bind(model); + return jest.spyOn(model, method as 'findOneAndUpdate').mockImplementationOnce((( ...args: Parameters ) => { const query = original(...args); @@ -389,6 +402,29 @@ describe('enabling for a conversation or owner that no longer exists', () => { expect(await claim(1)).toEqual({ ok: false, error: { code: 'not_found' } }); }); + test('does not roll back an enable that landed after the crossing enable was rejected', async () => { + await enable(pullOne); + const spy = interleaveEnable(async () => { + await startAccountDeletionNow(); + /** After the rejected enable reads the owner, the deletion is cancelled and a newer + * enable rebinds the record before the rejected one undoes its write. */ + interleave( + async () => { + await cancelAccountDeletion(); + await enable(pullTwo); + }, + mongoose.models.User as mongoose.Model, + 'findById', + ); + }); + try { + await methods.enablePRAutomation({ ...key, binding: pullOne }); + } finally { + spy.mockRestore(); + } + expect((await methods.getPRAutomation(key))?.pullNumber).toBe(2); + }); + test('keeps a record the crossing enable did not change', async () => { await enable(pullOne); await startRound(1); @@ -454,11 +490,11 @@ describe('claiming for a subagent thread whose root conversation is gone', () => depth: 1, }, }); - const claimChild = () => + const claimChild = async () => methods.claimPRAutomationRound({ ...childKey, ...limits, - binding: pullOne, + binding: { ...pullOne, epoch: await currentEpoch(childKey) }, headSha: head(1), }); @@ -510,7 +546,7 @@ describe('claiming for a subagent thread whose root conversation is gone', () => await methods.claimPRAutomationRound({ ...grandKey, ...limits, - binding: pullOne, + binding: { ...pullOne, epoch: await currentEpoch(grandKey) }, headSha: head(1), }), ).toEqual({ ok: false, error: { code: 'conversation_gone' } }); @@ -542,7 +578,7 @@ describe('claiming for a subagent thread whose root conversation is gone', () => await methods.claimPRAutomationRound({ ...greatKey, ...limits, - binding: pullOne, + binding: { ...pullOne, epoch: await currentEpoch(greatKey) }, headSha: head(1), }), ).toEqual({ ok: false, error: { code: 'conversation_gone' } }); @@ -708,7 +744,7 @@ describe('claimPRAutomationRound', () => { methods.claimPRAutomationRound({ ...key, ...limits, - binding: pullOne, + binding: { ...pullOne, epoch: await currentEpoch() }, headSha: 'a'.repeat(65), }), ).rejects.toThrow(RangeError); @@ -720,12 +756,20 @@ describe('claimPRAutomationRound', () => { const result = await methods.claimPRAutomationRound({ ...key, ...limits, - binding: pullOne, + binding: { ...pullOne, epoch: await currentEpoch() }, headSha: 'b'.repeat(64), }); expect(result.ok).toBe(true); }); + test('rejects a delivery from a run that was stopped and restarted on the same pull request', async () => { + await enable(pullOne); + const stale = await currentEpoch(); + await methods.stopPRAutomation(key, 'user_stopped'); + await enable(pullOne); + expect(await claim(1, { epoch: stale })).toEqual(mismatch); + }); + test('rejects a claim while a round is already running', async () => { await enable(); await claim(1); @@ -797,7 +841,7 @@ describe('claimPRAutomationRound', () => { await enable(pullOne); const refused = await methods.claimPRAutomationRound({ ...key, - binding: pullOne, + binding: { ...pullOne, epoch: await currentEpoch() }, maxRounds: 0, maxMinutes: limits.maxMinutes, headSha: head(1), @@ -1081,6 +1125,28 @@ describe('setPRAutomationTrust', () => { }); describe('approved bots', () => { + test('keeps a bot reapproved after the record was rebound away and back', async () => { + await enable(pullOne); + const stale = { repository: 'acme/one', epoch: await currentEpoch() }; + await enable(otherRepository); + await enable(pullOne); + await addBot(7); + await methods.removePRAutomationBot(key, 7, stale); + expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([{ id: 7 }]); + }); + + test('keeps approved bots when the repository is named with different casing', async () => { + await enable(pullOne); + await addBot(7); + await enable({ repository: 'Acme/One', pullNumber: 1 }); + expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([{ id: 7 }]); + }); + + test('claims for the bound repository named with different casing', async () => { + await enable(pullOne); + expect((await claim(1, { binding: { repository: 'ACME/one', pullNumber: 1 } })).ok).toBe(true); + }); + test('refuses an approval from before the record was rebound away and back', async () => { await enable(pullOne); const stale = { repository: 'acme/one', epoch: (await methods.getPRAutomation(key))!.epoch }; @@ -1167,7 +1233,10 @@ describe('approved bots', () => { await enable(); await addBot(1); await addBot(2); - const record = await methods.removePRAutomationBot(key, 1, 'acme/one'); + const record = await methods.removePRAutomationBot(key, 1, { + repository: 'acme/one', + epoch: await currentEpoch(), + }); expect(record?.trustedBots).toEqual([{ id: 2 }]); }); @@ -1176,7 +1245,10 @@ describe('approved bots', () => { await addBot(7); await enable(otherRepository); await addBot(7, undefined, maxBots, 'acme/two'); - const record = await methods.removePRAutomationBot(key, 7, 'acme/one'); + const record = await methods.removePRAutomationBot(key, 7, { + repository: 'acme/one', + epoch: await currentEpoch(), + }); expect(record).toBeNull(); expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([{ id: 7 }]); }); @@ -1184,7 +1256,10 @@ describe('approved bots', () => { test('removes an approval for the repository the record is bound to', async () => { await enable(pullOne); await addBot(7); - const record = await methods.removePRAutomationBot(key, 7, 'acme/one'); + const record = await methods.removePRAutomationBot(key, 7, { + repository: 'acme/one', + epoch: await currentEpoch(), + }); expect(record?.trustedBots).toEqual([]); }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 41dfa1e95a1..e23393641a9 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -28,11 +28,13 @@ type Gone = Extract; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; -/** Everything that belongs to one run on one pull request, under a new epoch. */ -const runReset = (fields: Record = {}) => ({ - $set: { state: 'idle', round: 0, claimedHeads: [], epoch: randomUUID(), ...fields }, +/** Everything that belongs to one run on one pull request, under the given new epoch. */ +const runReset = (epoch: string, fields: Record = {}) => ({ + $set: { state: 'idle', round: 0, claimedHeads: [], epoch, ...fields }, $unset: { stopCode: '', startedAt: '', lastHeadSha: '', runId: '' }, }); +/** GitHub compares `owner/name` without case, so records store and match the lower case form. */ +const canonicalRepository = (repository: string) => repository.toLowerCase(); const isGone = (liveness: Liveness): liveness is Gone => liveness === 'conversation_gone' || liveness === 'owner_gone'; @@ -68,7 +70,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): removePRAutomationBot: ( key: t.PRAutomationKey, botId: number, - repository: string, + fence: t.PRAutomationBotFence, ) => Promise; } { let indexesPromise: Promise | null = null; @@ -99,8 +101,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * deletion has started (the same durable marker agent triggers use), and the conversation * and every conversation that controls it still exist and are inside their retention. A * subagent thread is controlled by every ancestor up to its root, and a deletion removes - * one generation before it discovers the next, so the whole parent chain is walked. Depth - * is capped by `MAX_SUBAGENT_DEPTH`, which bounds the walk; a longer chain fails closed. + * one generation before it discovers the next, so the whole parent chain is checked. One + * `$graphLookup` follows the chain through active conversations only, so it stops at the + * first missing or expired ancestor and reaches the root only when every link is live. The + * walk is bounded by `MAX_SUBAGENT_DEPTH`; a longer chain fails closed. * Deletions do not fence the record; enable and claim ask this on both sides of their write * instead. */ @@ -110,15 +114,37 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } const Conversation = mongoose.models.Conversation; const active = activeExpirationFilter(); - type Thread = { rootConversationId: string; parentConversationId: string }; - type Lineage = { conversationId: string; subagentThread?: Thread }; - const [owner, conversation] = await Promise.all([ + const scope = { user: key.userId, ...active }; + type Lineage = { + subagentThread?: { rootConversationId: string }; + ancestors: { conversationId: string }[]; + }; + const [owner, lineage] = await Promise.all([ mongoose.models.User.findById(key.userId) .select('agentTriggerDeletionStartedAt') .lean<{ agentTriggerDeletionStartedAt?: Date }>(), - Conversation.findOne({ user: key.userId, conversationId: key.conversationId, ...active }) - .select('conversationId subagentThread') - .lean(), + Conversation.aggregate([ + { $match: { ...scope, conversationId: key.conversationId } }, + { $limit: 1 }, + { + $graphLookup: { + from: 'conversations', + startWith: '$subagentThread.parentConversationId', + connectFromField: 'subagentThread.parentConversationId', + connectToField: 'conversationId', + as: 'ancestors', + maxDepth: MAX_SUBAGENT_DEPTH - 1, + restrictSearchWithMatch: scope, + }, + }, + { + $project: { + _id: 0, + 'subagentThread.rootConversationId': 1, + 'ancestors.conversationId': 1, + }, + }, + ]), ]); if (owner == null) { return 'owner_gone'; @@ -126,27 +152,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (owner.agentTriggerDeletionStartedAt != null) { return 'owner_inactive'; } + const [conversation] = lineage; if (conversation == null) { return 'conversation_gone'; } - const rootConversationId = conversation.subagentThread?.rootConversationId; - let current: Lineage | null = conversation; - for (let hops = 0; current?.subagentThread != null; hops++) { - if (hops >= MAX_SUBAGENT_DEPTH) { - return 'conversation_gone'; - } - current = await Conversation.findOne({ - user: key.userId, - conversationId: current.subagentThread.parentConversationId, - ...active, - }) - .select('conversationId subagentThread') - .lean(); - } - if (current == null) { - return 'conversation_gone'; + const root = conversation.subagentThread?.rootConversationId; + if (root == null) { + return 'live'; } - const reachedRoot = rootConversationId == null || current.conversationId === rootConversationId; + const reachedRoot = conversation.ancestors.some((ancestor) => ancestor.conversationId === root); return reachedRoot ? 'live' : 'conversation_gone'; } @@ -206,9 +220,13 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): */ async function enablePRAutomation({ trust, - binding, + binding: requested, ...key }: t.EnablePRAutomationParams): Promise { + const binding = + requested == null + ? undefined + : { ...requested, repository: canonicalRepository(requested.repository) }; const before = await checkLiveness(key); if (before !== 'live') { return { ok: false, error: { code: isGone(before) ? goneCode(before) : before } }; @@ -216,8 +234,13 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): await ensureIndexes(); const PRAutomation = mongoose.models.PRAutomation; const filter = keyFilter(key); - const reset = runReset; - /** What the record was before this attempt, so a crossed deletion can undo only its write. */ + /** + * Every write of this attempt stamps the same new epoch, so the record carries it only when + * this attempt changed it. A crossed deletion undoes exactly that, and nothing a later + * enable wrote. + */ + const attempt = randomUUID(); + const reset = (fields: Record) => runReset(attempt, fields); const previous = await PRAutomation.findOne(filter).lean(); const options = { runValidators: true }; await PRAutomation.updateOne( @@ -229,7 +252,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): round: 0, trustedBots: [], claimedHeads: [], - epoch: randomUUID(), + epoch: attempt, trust: trust ?? 'approvedBots', ...binding, }, @@ -274,7 +297,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: false, error: { code: goneCode(after) } }; } if (after !== 'live') { - await undoEnable(filter, previous); + await undoEnable({ ...filter, epoch: attempt }, previous); return { ok: false, error: { code: after } }; } const record = await getPRAutomation(key); @@ -286,20 +309,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Puts the record back as this enable found it when an account deletion started during the - * write, which can still be cancelled. The undo applies only while the record still carries - * the epoch the attempt left, so it never reverts a later enable. An attempt that changed - * nothing leaves the epoch as it was and is not touched. + * write, which can still be cancelled. `guard` names the epoch only this attempt wrote, so + * an attempt that changed nothing, or a record a later enable already replaced, is left + * alone. */ async function undoEnable( - filter: ReturnType, + guard: ReturnType & { epoch: string }, previous: t.IPRAutomation | null, ): Promise { const PRAutomation = mongoose.models.PRAutomation; - const written = await PRAutomation.findOne(filter).select('epoch').lean<{ epoch: string }>(); - if (written == null || written.epoch === previous?.epoch) { - return; - } - const guard = { ...filter, epoch: written.epoch }; if (previous == null) { await PRAutomation.deleteOne(guard); return; @@ -339,7 +357,13 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): throw new RangeError('headSha must be a full commit SHA'); } const PRAutomation = mongoose.models.PRAutomation; - const filter = { ...keyFilter(key), ...binding }; + const repository = canonicalRepository(binding.repository); + const filter = { + ...keyFilter(key), + repository, + pullNumber: binding.pullNumber, + epoch: binding.epoch, + }; const cutoff = new Date(now.getTime() - maxMinutes * 60_000); const runId = randomUUID(); @@ -412,7 +436,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (current == null) { return { ok: false, error: { code: 'not_found' } }; } - if (current.repository !== binding.repository || current.pullNumber !== binding.pullNumber) { + if ( + current.repository !== repository || + current.pullNumber !== binding.pullNumber || + current.epoch !== binding.epoch + ) { return { ok: false, error: { code: 'binding_mismatch' } }; } if (!CLAIMABLE_STATES.includes(current.state)) { @@ -483,7 +511,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const condition = user || fence == null ? {} - : { repository: fence.repository, pullNumber: fence.pullNumber, epoch: fence.epoch }; + : { + repository: canonicalRepository(fence.repository), + pullNumber: fence.pullNumber, + epoch: fence.epoch, + }; const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( { ...keyFilter(key), ...condition, state: { $ne: 'stopped' } }, @@ -538,8 +570,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): key: t.PRAutomationKey, bot: t.IPRAutomationBot, maxBots: number, - { repository, epoch }: t.PRAutomationBotFence, + fence: t.PRAutomationBotFence, ): Promise { + const repository = canonicalRepository(fence.repository); + const { epoch } = fence; if (!Number.isInteger(maxBots) || maxBots < 1 || maxBots > MAX_PR_AUTOMATION_BOTS) { throw new RangeError(`maxBots must be an integer from 1 to ${MAX_PR_AUTOMATION_BOTS}`); } @@ -572,15 +606,18 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: false, error: { code: 'bot_limit' } }; } - /** Names the repository like an approval does, so a delayed removal cannot delete a newer approval. */ + /** + * Names the repository and epoch like an approval does, so a delayed removal cannot delete + * an approval made for a later binding, including one away and back to the same repository. + */ async function removePRAutomationBot( key: t.PRAutomationKey, botId: number, - repository: string, + { repository, epoch }: t.PRAutomationBotFence, ): Promise { const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( - { ...keyFilter(key), repository }, + { ...keyFilter(key), repository: canonicalRepository(repository), epoch }, { $pull: { trustedBots: { id: botId } } }, { new: true, select: PROJECTION }, ).lean(); diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 2951b29b601..90a26e92f94 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -97,9 +97,10 @@ export interface ClaimPRAutomationRoundParams extends PRAutomationKey { /** * The pull request the delivery is for. A delivery for a pull request the record is no * longer bound to cannot claim a round, so an event that arrives after a rebind is rejected - * instead of spending the new pull request's budget. + * instead of spending the new pull request's budget. The epoch the delivery was admitted + * under is part of it, so a delivery for a run that was restarted is rejected too. */ - binding: PRAutomationBinding; + binding: PRAutomationRunFence; maxRounds: number; maxMinutes: number; /** The head the round will work on; recorded so a stale event can be recognized. */ From ad743b3930116cf851edc887f25f0de233b1424a Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:30:43 +0200 Subject: [PATCH 12/25] fix: Read PR Automation Lineage Without graphLookup and Restore the True Enable Preimage --- .../src/methods/prAutomation.spec.ts | 66 ++++-- .../data-schemas/src/methods/prAutomation.ts | 197 +++++++++++------- .../data-schemas/src/schema/prAutomation.ts | 10 +- 3 files changed, 183 insertions(+), 90 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 76e6f29334f..af3bd41b3e8 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -16,7 +16,7 @@ const otherUserId = new mongoose.Types.ObjectId().toString(); const key = { userId, conversationId: 'convo-1' }; const limits = { maxRounds: 3, maxMinutes: 60 }; const maxBots = 3; -const head = (n: number) => String(n).repeat(40).slice(0, 40); +const head = (n: number) => (n === 1 ? 'a' : String(n)).repeat(40).slice(0, 40); const pullOne = { repository: 'acme/one', pullNumber: 1 }; const pullTwo = { repository: 'acme/one', pullNumber: 2 }; const otherRepository = { repository: 'acme/two', pullNumber: 2 }; @@ -191,6 +191,18 @@ describe('enablePRAutomation', () => { expect((await claim(1)).ok).toBe(true); }); + test('applies a narrower trust level passed with a rebind of a running record', async () => { + await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); + const rebound = await enableWith({ ...key, binding: otherRepository, trust: 'approvedBots' }); + expect(rebound.trust).toBe('approvedBots'); + }); + + test('applies a trust level passed with a rebind to another pull request', async () => { + await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); + const rebound = await enableWith({ ...key, binding: pullTwo, trust: 'collaborators' }); + expect(rebound.trust).toBe('collaborators'); + }); + test('keeps records of different users apart', async () => { await enable(); expect(await methods.getPRAutomation({ ...key, userId: otherUserId })).toBeNull(); @@ -363,21 +375,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { }); /** Runs `during` right after the enable's first write, before it checks again. */ - const interleaveEnable = (during: () => Promise) => { - const original = PRAutomation.updateOne.bind(PRAutomation); - return jest.spyOn(PRAutomation, 'updateOne').mockImplementationOnce((( - ...args: Parameters - ) => { - const query = original(...args); - const then = query.then.bind(query); - query.then = ((resolve, reject) => - then(async (value) => { - await during(); - return value; - }).then(resolve, reject)) as typeof query.then; - return query; - }) as typeof PRAutomation.updateOne); - }; + const interleaveEnable = (during: () => Promise) => interleave(during); test('rejects an enable once account deletion started mid-write', async () => { const spy = interleaveEnable(startAccountDeletionNow); @@ -425,6 +423,23 @@ describe('enabling for a conversation or owner that no longer exists', () => { expect((await methods.getPRAutomation(key))?.pullNumber).toBe(2); }); + test('restores the enable that landed between the crossing enable writes, not an older one', async () => { + await enable(pullOne); + /** A newer enable lands after the crossing enable's first write and before its rebind, + * then account deletion starts before the crossing enable checks again. */ + const spy = interleave(async () => { + await enable(otherRepository); + await startAccountDeletionNow(); + }); + try { + await methods.enablePRAutomation({ ...key, binding: pullTwo }); + } finally { + spy.mockRestore(); + } + await cancelAccountDeletion(); + expect(await methods.getPRAutomation(key)).toMatchObject(otherRepository); + }); + test('keeps a record the crossing enable did not change', async () => { await enable(pullOne); await startRound(1); @@ -751,6 +766,19 @@ describe('claimPRAutomationRound', () => { expect((await methods.getPRAutomation(key))?.claimedHeads).toEqual([]); }); + test('treats a head in upper case as the head it already claimed', async () => { + await enable(); + await claim(1); + await toWaiting(); + const upper = await methods.claimPRAutomationRound({ + ...key, + ...limits, + binding: { ...pullOne, epoch: await currentEpoch() }, + headSha: 'A'.repeat(40), + }); + expect(upper).toEqual({ ok: false, error: { code: 'stale_head' } }); + }); + test('accepts a SHA-256 head', async () => { await enable(); const result = await methods.claimPRAutomationRound({ @@ -1155,6 +1183,12 @@ describe('approved bots', () => { expect(await methods.addPRAutomationBot(key, { id: 7 }, maxBots, stale)).toEqual(mismatch); }); + test('refuses a bot id that is not a whole number', async () => { + await enable(); + await expect(addBot(7.5)).rejects.toThrow(); + expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([]); + }); + test('adds a bot by numeric id', async () => { await enable(); expect(await addBot(101, 'review-bot[bot]')).toMatchObject({ diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index e23393641a9..170f61de770 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -7,8 +7,8 @@ import { import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationState } from 'librechat-data-provider'; +import type { PipelineStage } from 'mongoose'; import type * as t from '~/types/prAutomation'; -import { activeExpirationFilter } from '~/utils/retention'; import { isValidObjectIdString } from '~/utils/objectId'; import { createIndexesWithRetry } from '~/utils/retry'; @@ -35,6 +35,55 @@ const runReset = (epoch: string, fields: Record = {}) => ({ }); /** GitHub compares `owner/name` without case, so records store and match the lower case form. */ const canonicalRepository = (repository: string) => repository.toLowerCase(); +const LINK_FIELDS = [ + 'conversationId', + 'user', + 'expiredAt', + 'subagentThread.rootConversationId', + 'subagentThread.parentConversationId', +]; + +/** + * One aggregation that reads a conversation and up to `MAX_SUBAGENT_DEPTH` ancestors, one + * `$lookup` per generation, so a liveness check is one round trip at any depth. `$graphLookup` + * would be shorter but Amazon DocumentDB rejects it. Each lookup is served by the unique + * `{ conversationId, user, tenantId }` index; ownership and retention are judged on the result. + */ +function lineagePipeline({ userId, conversationId }: t.PRAutomationKey): PipelineStage[] { + const stages: PipelineStage[] = [ + { $match: { user: userId, conversationId } }, + { $limit: 1 }, + { + $project: { + _id: 0, + l0: Object.fromEntries(LINK_FIELDS.map((field) => [field, `$${field}`])), + }, + }, + ]; + for (let level = 1; level <= MAX_SUBAGENT_DEPTH; level++) { + const as = `l${level}`; + stages.push( + { + $lookup: { + from: 'conversations', + localField: `l${level - 1}.subagentThread.parentConversationId`, + foreignField: 'conversationId', + as, + }, + }, + { $unwind: { path: `$${as}`, preserveNullAndEmptyArrays: true } }, + ); + } + const keep: Record = {}; + for (let level = 0; level <= MAX_SUBAGENT_DEPTH; level++) { + for (const field of LINK_FIELDS) { + keep[`l${level}.${field}`] = 1; + } + } + stages.push({ $project: { _id: 0, ...keep } }); + return stages; +} + const isGone = (liveness: Liveness): liveness is Gone => liveness === 'conversation_gone' || liveness === 'owner_gone'; @@ -101,10 +150,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * deletion has started (the same durable marker agent triggers use), and the conversation * and every conversation that controls it still exist and are inside their retention. A * subagent thread is controlled by every ancestor up to its root, and a deletion removes - * one generation before it discovers the next, so the whole parent chain is checked. One - * `$graphLookup` follows the chain through active conversations only, so it stops at the - * first missing or expired ancestor and reaches the root only when every link is live. The - * walk is bounded by `MAX_SUBAGENT_DEPTH`; a longer chain fails closed. + * one generation before it discovers the next, so the whole parent chain is checked, read in + * one aggregation. The chain is live only when every link belongs to the owner, is inside + * its retention and the walk ends at the recorded root. The walk is bounded by + * `MAX_SUBAGENT_DEPTH`; a longer chain fails closed. * Deletions do not fence the record; enable and claim ask this on both sides of their write * instead. */ @@ -113,38 +162,17 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return 'owner_inactive'; } const Conversation = mongoose.models.Conversation; - const active = activeExpirationFilter(); - const scope = { user: key.userId, ...active }; - type Lineage = { - subagentThread?: { rootConversationId: string }; - ancestors: { conversationId: string }[]; + type Link = { + conversationId: string; + user: string; + expiredAt?: Date | null; + subagentThread?: { rootConversationId?: string; parentConversationId?: string }; }; - const [owner, lineage] = await Promise.all([ + const [owner, rows] = await Promise.all([ mongoose.models.User.findById(key.userId) .select('agentTriggerDeletionStartedAt') .lean<{ agentTriggerDeletionStartedAt?: Date }>(), - Conversation.aggregate([ - { $match: { ...scope, conversationId: key.conversationId } }, - { $limit: 1 }, - { - $graphLookup: { - from: 'conversations', - startWith: '$subagentThread.parentConversationId', - connectFromField: 'subagentThread.parentConversationId', - connectToField: 'conversationId', - as: 'ancestors', - maxDepth: MAX_SUBAGENT_DEPTH - 1, - restrictSearchWithMatch: scope, - }, - }, - { - $project: { - _id: 0, - 'subagentThread.rootConversationId': 1, - 'ancestors.conversationId': 1, - }, - }, - ]), + Conversation.aggregate>(lineagePipeline(key)), ]); if (owner == null) { return 'owner_gone'; @@ -152,16 +180,33 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (owner.agentTriggerDeletionStartedAt != null) { return 'owner_inactive'; } - const [conversation] = lineage; + /** Links of other users never count: a lookup by `conversationId` is not scoped by owner. */ + const now = Date.now(); + const links = new Map(); + for (const row of rows) { + for (const link of Object.values(row)) { + const active = link?.expiredAt == null || new Date(link.expiredAt).getTime() > now; + if (link != null && String(link.user) === key.userId && active) { + links.set(link.conversationId, link); + } + } + } + const conversation = links.get(key.conversationId); if (conversation == null) { return 'conversation_gone'; } + let current: Link | undefined = conversation; + for (let hops = 0; current?.subagentThread?.parentConversationId != null; hops++) { + if (hops >= MAX_SUBAGENT_DEPTH) { + return 'conversation_gone'; + } + current = links.get(current.subagentThread.parentConversationId); + } const root = conversation.subagentThread?.rootConversationId; - if (root == null) { - return 'live'; + if (current == null || (root != null && current.conversationId !== root)) { + return 'conversation_gone'; } - const reachedRoot = conversation.ancestors.some((ancestor) => ancestor.conversationId === root); - return reachedRoot ? 'live' : 'conversation_gone'; + return 'live'; } /** A missing owner reads as inactive to callers; the distinction only decides removal. */ @@ -236,14 +281,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const filter = keyFilter(key); /** * Every write of this attempt stamps the same new epoch, so the record carries it only when - * this attempt changed it. A crossed deletion undoes exactly that, and nothing a later - * enable wrote. + * this attempt changed it. Each write returns the record as it was just before that write, + * atomically with it, and the last one this attempt did not already own is what a crossed + * deletion restores: a later enable that landed between this attempt's read and write is + * kept, and nothing a later enable wrote afterwards is reverted. */ const attempt = randomUUID(); const reset = (fields: Record) => runReset(attempt, fields); - const previous = await PRAutomation.findOne(filter).lean(); const options = { runValidators: true }; - await PRAutomation.updateOne( + let undo: { previous: t.IPRAutomation | null } | undefined; + const write = async (condition: Record, update: Record) => { + const before = await PRAutomation.findOneAndUpdate({ ...filter, ...condition }, update, { + ...options, + new: false, + }).lean(); + if (before != null && before.epoch !== attempt) { + undo = { previous: before }; + } + }; + const existing = await PRAutomation.findOneAndUpdate( filter, { $setOnInsert: { @@ -257,38 +313,30 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ...binding, }, }, - { upsert: true, ...options }, - ); + { upsert: true, new: false, ...options }, + ).lean(); + if (existing == null) { + undo = { previous: null }; + } + /** A trust level passed with the request applies to every write that starts a run. */ const revive = trust != null ? { trust } : {}; if (binding == null) { - await PRAutomation.updateOne({ ...filter, ...RESTARTABLE }, reset(revive), options); + await write(RESTARTABLE, reset(revive)); } else { const { repository } = binding; const otherRepository = { repository: { $ne: repository } }; - await PRAutomation.updateOne( - { ...filter, ...RESTARTABLE, ...otherRepository }, + await write( + { ...RESTARTABLE, ...otherRepository }, reset({ ...revive, ...binding, trustedBots: [] }), - options, - ); - await PRAutomation.updateOne( - { ...filter, ...RESTARTABLE, repository }, - reset({ ...revive, ...binding }), - options, ); - await PRAutomation.updateOne( - { ...filter, state: { $ne: 'stopped' }, ...otherRepository }, - reset({ ...binding, trustedBots: [] }), - options, + await write({ ...RESTARTABLE, repository }, reset({ ...revive, ...binding })); + await write( + { state: { $ne: 'stopped' }, ...otherRepository }, + reset({ ...revive, ...binding, trustedBots: [] }), ); - await PRAutomation.updateOne( - { - ...filter, - state: { $ne: 'stopped' }, - repository, - pullNumber: { $ne: binding.pullNumber }, - }, - reset({ ...binding }), - options, + await write( + { state: { $ne: 'stopped' }, repository, pullNumber: { $ne: binding.pullNumber } }, + reset({ ...revive, ...binding }), ); } const after = await checkLiveness(key); @@ -297,7 +345,9 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: false, error: { code: goneCode(after) } }; } if (after !== 'live') { - await undoEnable({ ...filter, epoch: attempt }, previous); + if (undo != null) { + await undoEnable({ ...filter, epoch: attempt }, undo.previous); + } return { ok: false, error: { code: after } }; } const record = await getPRAutomation(key); @@ -308,10 +358,9 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } /** - * Puts the record back as this enable found it when an account deletion started during the - * write, which can still be cancelled. `guard` names the epoch only this attempt wrote, so - * an attempt that changed nothing, or a record a later enable already replaced, is left - * alone. + * Puts the record back as this enable's own write found it when an account deletion started + * during the write, which can still be cancelled. `guard` names the epoch only this attempt + * wrote, so a record a later enable already replaced is left alone. */ async function undoEnable( guard: ReturnType & { epoch: string }, @@ -349,13 +398,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): binding, maxRounds, maxMinutes, - headSha, + headSha: rawHeadSha, now = new Date(), ...key }: t.ClaimPRAutomationRoundParams): Promise { - if (!COMMIT_SHA.test(headSha)) { + if (!COMMIT_SHA.test(rawHeadSha)) { throw new RangeError('headSha must be a full commit SHA'); } + /** One Git object, one identity: hex case must not let a commit spend two rounds. */ + const headSha = rawHeadSha.toLowerCase(); const PRAutomation = mongoose.models.PRAutomation; const repository = canonicalRepository(binding.repository); const filter = { diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index b7b4d732552..468b01de650 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -8,7 +8,15 @@ import type { IPRAutomationDocument } from '~/types/prAutomation'; const trustedBotSchema = new Schema( { - id: { type: Number, required: true, min: 1 }, + id: { + type: Number, + required: true, + min: 1, + validate: { + validator: Number.isSafeInteger, + message: 'A bot account id is a positive whole number', + }, + }, login: { type: String, maxlength: 128 }, }, { _id: false }, From 860e0f51429f0477f088d6dddee3c831ee07ec13 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:54:43 +0200 Subject: [PATCH 13/25] fix: Keep Rejected PR Automation Enables Unclaimable and Scope Lineage to the Tenant --- .../src/methods/prAutomation.spec.ts | 88 ++++++++++++++++++- .../data-schemas/src/methods/prAutomation.ts | 36 +++++++- .../data-schemas/src/schema/prAutomation.ts | 1 + .../data-schemas/src/types/prAutomation.ts | 5 ++ 4 files changed, 122 insertions(+), 8 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index af3bd41b3e8..aead4a97c51 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -4,6 +4,7 @@ import { MongoMemoryServer } from 'mongodb-memory-server'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationEventStopCode } from '~/types/prAutomation'; import { createPRAutomationMethods } from './prAutomation'; +import { tenantStorage } from '~/config/tenantContext'; logger.silent = true; @@ -423,10 +424,8 @@ describe('enabling for a conversation or owner that no longer exists', () => { expect((await methods.getPRAutomation(key))?.pullNumber).toBe(2); }); - test('restores the enable that landed between the crossing enable writes, not an older one', async () => { + test('never brings back an older binding when an enable crosses a deletion', async () => { await enable(pullOne); - /** A newer enable lands after the crossing enable's first write and before its rebind, - * then account deletion starts before the crossing enable checks again. */ const spy = interleave(async () => { await enable(otherRepository); await startAccountDeletionNow(); @@ -437,7 +436,70 @@ describe('enabling for a conversation or owner that no longer exists', () => { spy.mockRestore(); } await cancelAccountDeletion(); - expect(await methods.getPRAutomation(key)).toMatchObject(otherRepository); + expect(await methods.getPRAutomation(key)).not.toMatchObject(pullOne); + }); + + test('leaves nothing claimable when two enables cross the same deletion', async () => { + await enable(pullOne); + /** + * Owner reads, in order: A before, A after, B before, B after. A writes, B writes over it, + * the deletion starts, A sees it and rolls back first, then B sees it and rolls back. + */ + const User = mongoose.models.User; + const original = User.findById.bind(User); + let reads = 0; + let reachedB = () => {}; + const atB = new Promise((resolve) => (reachedB = resolve)); + let releaseB = () => {}; + const gateB = new Promise((resolve) => (releaseB = resolve)); + let enableB: Promise = Promise.resolve(); + const before: Record Promise> = { + 2: async () => { + enableB = methods.enablePRAutomation({ ...key, binding: otherRepository }); + await atB; + await startAccountDeletionNow(); + }, + 4: async () => { + reachedB(); + await gateB; + }, + }; + const spy = jest.spyOn(User, 'findById').mockImplementation((( + ...args: Parameters + ) => { + const hook = before[++reads]; + const query = original(...args); + if (hook == null) { + return query; + } + const lean = query.lean.bind(query); + query.lean = ((...leanArgs: Parameters) => ({ + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + hook() + .then(() => lean(...leanArgs)) + .then(resolve, reject), + })) as unknown as typeof query.lean; + return query; + }) as typeof User.findById); + try { + await methods.enablePRAutomation({ ...key, binding: pullTwo }); + releaseB(); + await enableB; + } finally { + spy.mockRestore(); + } + await cancelAccountDeletion(); + expect(await claim(1, { binding: pullTwo })).not.toMatchObject({ ok: true }); + }); + + test('lets the next enable recover a record an interrupted enable left pending', async () => { + await enable(pullOne); + await PRAutomation.updateOne( + { user: userId, conversationId: key.conversationId }, + { $set: { pending: 'interrupted-attempt' } }, + ); + await enable(pullOne); + expect((await claim(1)).ok).toBe(true); }); test('keeps a record the crossing enable did not change', async () => { @@ -599,6 +661,24 @@ describe('claiming for a subagent thread whose root conversation is gone', () => ).toEqual({ ok: false, error: { code: 'conversation_gone' } }); }); + test("does not let another tenant's conversation stand in for a deleted parent", async () => { + await mongoose.models.Conversation.deleteMany({}); + const inTenant = (tenantId: string, run: () => Promise) => + tenantStorage.run({ tenantId }, run); + await inTenant('tenant-b', () => seedConversation()); + await mongoose.models.User.deleteMany({}); + await inTenant('tenant-a', () => + mongoose.models.User.create({ _id: userId, email: 'owner@example.com', provider: 'local' }), + ); + await inTenant('tenant-a', async () => { + await seedConversation(); + await seedChild(); + await enableWith({ ...childKey, binding: pullOne }); + await mongoose.models.Conversation.deleteOne({ conversationId: key.conversationId }); + expect(await claimChild()).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + }); + }); + test('accepts a claim for a child while its root is active', async () => { await seedChild(); await methods.enablePRAutomation({ ...childKey, binding: pullOne }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 170f61de770..a3bb923f896 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -38,6 +38,7 @@ const canonicalRepository = (repository: string) => repository.toLowerCase(); const LINK_FIELDS = [ 'conversationId', 'user', + 'tenantId', 'expiredAt', 'subagentThread.rootConversationId', 'subagentThread.parentConversationId', @@ -47,7 +48,8 @@ const LINK_FIELDS = [ * One aggregation that reads a conversation and up to `MAX_SUBAGENT_DEPTH` ancestors, one * `$lookup` per generation, so a liveness check is one round trip at any depth. `$graphLookup` * would be shorter but Amazon DocumentDB rejects it. Each lookup is served by the unique - * `{ conversationId, user, tenantId }` index; ownership and retention are judged on the result. + * `{ conversationId, user, tenantId }` index. Nested lookups bypass the tenant plugin, so + * ownership, tenant and retention are all judged on the result. */ function lineagePipeline({ userId, conversationId }: t.PRAutomationKey): PipelineStage[] { const stages: PipelineStage[] = [ @@ -165,6 +167,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): type Link = { conversationId: string; user: string; + tenantId?: string | null; expiredAt?: Date | null; subagentThread?: { rootConversationId?: string; parentConversationId?: string }; }; @@ -180,13 +183,19 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (owner.agentTriggerDeletionStartedAt != null) { return 'owner_inactive'; } - /** Links of other users never count: a lookup by `conversationId` is not scoped by owner. */ + /** + * Links of other users or other tenants never count: a lookup by `conversationId` is scoped + * by neither. The first link came through the tenant-scoped `$match`, so it names the tenant. + */ const now = Date.now(); + const tenant = rows[0]?.l0?.tenantId ?? null; const links = new Map(); for (const row of rows) { for (const link of Object.values(row)) { const active = link?.expiredAt == null || new Date(link.expiredAt).getTime() > now; - if (link != null && String(link.user) === key.userId && active) { + const owned = + link != null && String(link.user) === key.userId && (link.tenantId ?? null) === tenant; + if (link != null && owned && active) { links.set(link.conversationId, link); } } @@ -285,9 +294,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * atomically with it, and the last one this attempt did not already own is what a crossed * deletion restores: a later enable that landed between this attempt's read and write is * kept, and nothing a later enable wrote afterwards is reverted. + * + * The writes also mark the record `pending` with the attempt, and nothing claims a pending + * record. Only an attempt that passes its second liveness check clears the mark. A preimage + * that another rejected attempt left is therefore still pending when it is restored, so two + * enables crossing the same deletion leave nothing claimable. */ const attempt = randomUUID(); - const reset = (fields: Record) => runReset(attempt, fields); + const reset = (fields: Record) => + runReset(attempt, { ...fields, pending: attempt }); const options = { runValidators: true }; let undo: { previous: t.IPRAutomation | null } | undefined; const write = async (condition: Record, update: Record) => { @@ -309,6 +324,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): trustedBots: [], claimedHeads: [], epoch: attempt, + pending: attempt, trust: trust ?? 'approvedBots', ...binding, }, @@ -339,6 +355,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): reset({ ...revive, ...binding }), ); } + /** + * A record still pending for another attempt is either in flight or was left by an attempt + * that never finished. Taking it over under this attempt's epoch recovers the second case, + * and the in-flight attempt's own undo then misses on its epoch, so this attempt decides. + */ + await write({ pending: { $exists: true, $ne: attempt } }, reset({ ...revive, ...binding })); const after = await checkLiveness(key); if (isGone(after)) { await PRAutomation.deleteOne(filter); @@ -350,6 +372,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } return { ok: false, error: { code: after } }; } + await PRAutomation.updateOne({ ...filter, pending: attempt }, { $unset: { pending: '' } }); const record = await getPRAutomation(key); if (record == null) { throw new Error('PR automation record missing after enable'); @@ -437,6 +460,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): { ...filter, state: { $in: CLAIMABLE_STATES }, + pending: { $exists: false }, round: { $lt: maxRounds }, $or: [{ startedAt: { $exists: false } }, { startedAt: { $gte: cutoff } }], claimedHeads: { $ne: headSha }, @@ -503,6 +527,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (current.startedAt != null && current.startedAt < cutoff) { return { ok: false, error: { code: 'time_cap' } }; } + if (current.pending != null) { + /** An enable is still deciding; it either commits this run or replaces it. */ + return { ok: false, error: { code: 'conflict' } }; + } if (current.claimedHeads.includes(headSha)) { return { ok: false, error: { code: 'stale_head' } }; } diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 468b01de650..56d46394aa6 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -75,6 +75,7 @@ const prAutomationSchema: Schema = new Schema( claimedHeads: { type: [{ type: String, maxlength: 64 }], default: [] }, runId: { type: String, maxlength: 64 }, epoch: { type: String, maxlength: 64, required: true, default: () => randomUUID() }, + pending: { type: String, maxlength: 64 }, }, { timestamps: true }, ); diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 90a26e92f94..ba1cdffa5ca 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -40,6 +40,11 @@ export interface IPRAutomation { * request cannot end the current one. */ epoch: string; + /** + * Set to the enable attempt that wrote the record until that attempt confirms the owner and + * conversation are still live. Nothing claims a pending record. + */ + pending?: string; /** * Every head a round has been claimed for since the last restart. A delayed * delivery of an earlier head is rejected against this list. It holds at most From 89c41318615f1b73b08ed5bd1ec52dd39239705b Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:30:08 +0200 Subject: [PATCH 14/25] fix: Clamp PR Automation Trust to the Admin Ceiling and Commit Enables Only When Current --- .../src/methods/prAutomation.spec.ts | 136 +++++++++++++----- .../data-schemas/src/methods/prAutomation.ts | 54 ++++++- .../data-schemas/src/schema/prAutomation.ts | 7 +- .../data-schemas/src/types/prAutomation.ts | 5 +- 4 files changed, 154 insertions(+), 48 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index aead4a97c51..52b68c67f59 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -24,8 +24,13 @@ const otherRepository = { repository: 'acme/two', pullNumber: 2 }; const mismatch = { ok: false, error: { code: 'binding_mismatch' } }; /** Enables and returns the record, for setups that expect the enable to succeed. */ -const enableWith = async (params: Parameters[0]) => { - const result = await methods.enablePRAutomation(params); +type EnableParams = Parameters[0]; +/** The widest ceiling, so setups that pass a trust level store it as given. */ +const enableOpen = ( + params: Omit & { maxTrust?: EnableParams['maxTrust'] }, +) => methods.enablePRAutomation({ maxTrust: 'anyone', ...params }); +const enableWith = async (params: Parameters[0]) => { + const result = await enableOpen(params); if (!result.ok) { throw new Error(`enable failed: ${result.error.code}`); } @@ -192,6 +197,27 @@ describe('enablePRAutomation', () => { expect((await claim(1)).ok).toBe(true); }); + test('creates a record no wider than the administrator ceiling', async () => { + const record = await enableWith({ + ...key, + binding: pullOne, + trust: 'anyone', + maxTrust: 'collaborators', + }); + expect(record.trust).toBe('collaborators'); + }); + + test('rebinds a record no wider than the administrator ceiling', async () => { + await enable(pullOne); + const rebound = await enableWith({ + ...key, + binding: otherRepository, + trust: 'anyone', + maxTrust: 'approvedBots', + }); + expect(rebound.trust).toBe('approvedBots'); + }); + test('applies a narrower trust level passed with a rebind of a running record', async () => { await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); const rebound = await enableWith({ ...key, binding: otherRepository, trust: 'approvedBots' }); @@ -204,26 +230,33 @@ describe('enablePRAutomation', () => { expect(rebound.trust).toBe('collaborators'); }); + test('enables for a stored conversation whose id is longer than 256 characters', async () => { + const conversationId = 'c'.repeat(300); + await seedConversation({ conversationId }); + const result = await methods.enablePRAutomation({ userId, conversationId, maxTrust: 'anyone' }); + expect(result.ok).toBe(true); + }); + test('keeps records of different users apart', async () => { await enable(); expect(await methods.getPRAutomation({ ...key, userId: otherUserId })).toBeNull(); }); test('clears approved bots when the conversation is bound to a different repository', async () => { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); await addBot(101); const rebound = await enableWith({ ...key, binding: otherRepository }); expect(rebound).toMatchObject({ ...otherRepository, trustedBots: [] }); }); test('refuses a bot approved before any repository was bound', async () => { - await methods.enablePRAutomation(key); + await enableOpen(key); expect(await addBot(101)).toEqual({ ok: false, error: { code: 'binding_mismatch' } }); expect((await methods.getPRAutomation(key))?.trustedBots).toEqual([]); }); test('keeps approved bots when the same repository is bound to another pull request', async () => { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); await addBot(101); const rebound = await enableWith({ ...key, binding: pullTwo }); expect(rebound).toMatchObject({ pullNumber: 2, trustedBots: [{ id: 101 }] }); @@ -363,13 +396,13 @@ describe('a deletion that lands while a claim is in flight', () => { describe('enabling for a conversation or owner that no longer exists', () => { test('does not leave a record behind for a deleted conversation', async () => { await mongoose.models.Conversation.deleteMany({}); - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); expect(await methods.getPRAutomation(key)).toBeNull(); }); test('reports that the conversation is gone', async () => { await mongoose.models.Conversation.deleteMany({}); - expect(await methods.enablePRAutomation({ ...key, binding: pullOne })).toEqual({ + expect(await enableOpen({ ...key, binding: pullOne })).toEqual({ ok: false, error: { code: 'conversation_gone' }, }); @@ -381,7 +414,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { test('rejects an enable once account deletion started mid-write', async () => { const spy = interleaveEnable(startAccountDeletionNow); try { - expect(await methods.enablePRAutomation({ ...key, binding: pullOne })).toEqual({ + expect(await enableOpen({ ...key, binding: pullOne })).toEqual({ ok: false, error: { code: 'owner_inactive' }, }); @@ -393,7 +426,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { test('leaves nothing claimable after a cancelled deletion that an enable crossed', async () => { const spy = interleaveEnable(startAccountDeletionNow); try { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); } finally { spy.mockRestore(); } @@ -417,7 +450,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { ); }); try { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); } finally { spy.mockRestore(); } @@ -431,7 +464,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { await startAccountDeletionNow(); }); try { - await methods.enablePRAutomation({ ...key, binding: pullTwo }); + await enableOpen({ ...key, binding: pullTwo }); } finally { spy.mockRestore(); } @@ -455,7 +488,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { let enableB: Promise = Promise.resolve(); const before: Record Promise> = { 2: async () => { - enableB = methods.enablePRAutomation({ ...key, binding: otherRepository }); + enableB = enableOpen({ ...key, binding: otherRepository }); await atB; await startAccountDeletionNow(); }, @@ -482,7 +515,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { return query; }) as typeof User.findById); try { - await methods.enablePRAutomation({ ...key, binding: pullTwo }); + await enableOpen({ ...key, binding: pullTwo }); releaseB(); await enableB; } finally { @@ -492,6 +525,31 @@ describe('enabling for a conversation or owner that no longer exists', () => { expect(await claim(1, { binding: pullTwo })).not.toMatchObject({ ok: true }); }); + test('does not report success when another enable replaced its write before it committed', async () => { + await enable(pullOne); + /** Another enable's write lands between this enable's second check and its commit. */ + const original = PRAutomation.updateOne.bind(PRAutomation); + const spy = jest + .spyOn(PRAutomation, 'updateOne') + .mockImplementationOnce(((...args: Parameters) => + PRAutomation.collection + .updateOne( + { user: userId, conversationId: key.conversationId }, + { $set: { epoch: 'other-attempt', pending: 'other-attempt' } }, + ) + .then(() => original(...args))) as unknown as typeof PRAutomation.updateOne); + try { + const result = await methods.enablePRAutomation({ + ...key, + binding: pullTwo, + maxTrust: 'anyone', + }); + expect(result.ok).toBe(false); + } finally { + spy.mockRestore(); + } + }); + test('lets the next enable recover a record an interrupted enable left pending', async () => { await enable(pullOne); await PRAutomation.updateOne( @@ -507,7 +565,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { await startRound(1); const spy = interleaveEnable(startAccountDeletionNow); try { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); } finally { spy.mockRestore(); } @@ -516,7 +574,7 @@ describe('enabling for a conversation or owner that no longer exists', () => { test('does not leave a record behind for an owner that no longer exists', async () => { await mongoose.models.User.deleteMany({}); - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); expect(await methods.getPRAutomation(key)).toBeNull(); }); }); @@ -577,14 +635,14 @@ describe('claiming for a subagent thread whose root conversation is gone', () => test('rejects a claim for a child once its root was deleted', async () => { await seedChild(); - await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + await enableOpen({ ...childKey, binding: pullOne }); await mongoose.models.Conversation.deleteOne({ conversationId: key.conversationId }); expect(await claimChild()).toEqual({ ok: false, error: { code: 'conversation_gone' } }); }); test('removes the child record so nothing is left to claim later', async () => { await seedChild(); - await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + await enableOpen({ ...childKey, binding: pullOne }); await mongoose.models.Conversation.deleteOne({ conversationId: key.conversationId }); await claimChild(); expect(await methods.getPRAutomation(childKey)).toBeNull(); @@ -592,7 +650,7 @@ describe('claiming for a subagent thread whose root conversation is gone', () => test('rejects a claim for a child whose root passed its retention date', async () => { await seedChild(); - await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + await enableOpen({ ...childKey, binding: pullOne }); await mongoose.models.Conversation.updateOne( { conversationId: key.conversationId }, { $set: { expiredAt: new Date(Date.now() - 60_000) } }, @@ -617,7 +675,7 @@ describe('claiming for a subagent thread whose root conversation is gone', () => }, }); const grandKey = { userId, conversationId: 'grandchild-1' }; - await methods.enablePRAutomation({ ...grandKey, binding: pullOne }); + await enableOpen({ ...grandKey, binding: pullOne }); await mongoose.models.Conversation.deleteOne({ conversationId: childKey.conversationId }); expect( await methods.claimPRAutomationRound({ @@ -649,7 +707,7 @@ describe('claiming for a subagent thread whose root conversation is gone', () => await thread('grandchild-1', childKey.conversationId, 2); await thread('great-1', 'grandchild-1', 3); const greatKey = { userId, conversationId: 'great-1' }; - await methods.enablePRAutomation({ ...greatKey, binding: pullOne }); + await enableOpen({ ...greatKey, binding: pullOne }); await mongoose.models.Conversation.deleteOne({ conversationId: childKey.conversationId }); expect( await methods.claimPRAutomationRound({ @@ -681,7 +739,7 @@ describe('claiming for a subagent thread whose root conversation is gone', () => test('accepts a claim for a child while its root is active', async () => { await seedChild(); - await methods.enablePRAutomation({ ...childKey, binding: pullOne }); + await enableOpen({ ...childKey, binding: pullOne }); expect((await claimChild()).ok).toBe(true); }); }); @@ -703,7 +761,7 @@ describe('index guarantees', () => { test('builds the unique index before the first write when automatic indexing is off', async () => { await PRAutomation.collection.dropIndexes(); const fresh = createPRAutomationMethods(mongoose); - await fresh.enablePRAutomation({ ...key, binding: pullOne }); + await fresh.enablePRAutomation({ ...key, binding: pullOne, maxTrust: 'anyone' }); const indexes = await PRAutomation.collection.indexes(); expect( indexes.some( @@ -734,7 +792,7 @@ describe('binding a pull request', () => { }); test('starts a fresh run when bound to another pull request in the same repository', async () => { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); await startRound(1); await toWaiting(); await startRound(2); @@ -746,33 +804,33 @@ describe('binding a pull request', () => { }); test('lets the new pull request claim a head the old one claimed', async () => { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); await startRound(1); - await methods.enablePRAutomation({ ...key, binding: pullTwo }); + await enableOpen({ ...key, binding: pullTwo }); expect((await claim(1, { binding: pullTwo })).ok).toBe(true); }); test('does not carry the old pull request round count into the new one', async () => { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); for (let round = 1; round <= limits.maxRounds; round++) { await startRound(round); await toWaiting(); } - await methods.enablePRAutomation({ ...key, binding: pullTwo }); + await enableOpen({ ...key, binding: pullTwo }); expect((await claim(9, { binding: pullTwo })).ok).toBe(true); }); test('ignores a completion from the round of the pull request it replaced', async () => { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); const running = await startRound(1); - await methods.enablePRAutomation({ ...key, binding: pullTwo }); + await enableOpen({ ...key, binding: pullTwo }); expect(await settle(1, running.runId)).toBeNull(); expect((await methods.getPRAutomation(key))?.state).toBe('idle'); }); test('leaves a run alone when it is bound to the pair it already has', async () => { - await methods.enablePRAutomation({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: pullOne }); await startRound(1); const again = await enableWith({ ...key, binding: pullOne }); expect(again).toMatchObject({ state: 'fixing', round: 1 }); @@ -783,9 +841,7 @@ describe('binding a pull request', () => { await enable(); const pairs = [pullOne, otherRepository]; await Promise.all( - Array.from({ length: 24 }, (_, index) => - methods.enablePRAutomation({ ...key, binding: pairs[index % 2] }), - ), + Array.from({ length: 24 }, (_, index) => enableOpen({ ...key, binding: pairs[index % 2] })), ); const record = await methods.getPRAutomation(key); expect(pairs).toContainEqual({ @@ -941,7 +997,7 @@ describe('claimPRAutomationRound', () => { }); test('rejects a claim on a record that is bound to no pull request', async () => { - await methods.enablePRAutomation(key); + await enableOpen(key); expect(await claim(1)).toEqual(mismatch); }); @@ -1220,14 +1276,20 @@ describe('stopPRAutomation', () => { describe('setPRAutomationTrust', () => { test('changes the trust level', async () => { await enable(); - const updated = await methods.setPRAutomationTrust(key, 'collaborators'); + const updated = await methods.setPRAutomationTrust(key, 'collaborators', 'anyone'); expect(updated?.trust).toBe('collaborators'); }); + test('stores no level wider than the administrator ceiling', async () => { + await enable(); + const updated = await methods.setPRAutomationTrust(key, 'anyone', 'approvedBots'); + expect(updated?.trust).toBe('approvedBots'); + }); + test('rejects a level outside the enum', async () => { await enable(); await expect( - methods.setPRAutomationTrust(key, 'everyone' as unknown as 'anyone'), + methods.setPRAutomationTrust(key, 'everyone' as unknown as 'anyone', 'anyone'), ).rejects.toThrow(); }); }); @@ -1380,7 +1442,7 @@ describe('approved bots', () => { test('keeps each conversation allowlist separate', async () => { await enable(); await seedConversation({ conversationId: 'convo-2' }); - await methods.enablePRAutomation({ userId, conversationId: 'convo-2' }); + await enableOpen({ userId, conversationId: 'convo-2' }); await addBot(101); const other = await methods.getPRAutomation({ userId, conversationId: 'convo-2' }); expect(other?.trustedBots).toEqual([]); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index a3bb923f896..f7f896a21b9 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -3,6 +3,8 @@ import { MAX_SUBAGENT_DEPTH, MAX_PR_AUTOMATION_BOTS, PR_AUTOMATION_STOP_CODES, + PR_AUTOMATION_TRUST_LEVELS, + clampPRAutomationTrust, } from 'librechat-data-provider'; import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; @@ -111,6 +113,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): setPRAutomationTrust: ( key: t.PRAutomationKey, trust: PRAutomationTrustLevel, + maxTrust: PRAutomationTrustLevel, ) => Promise; addPRAutomationBot: ( key: t.PRAutomationKey, @@ -240,6 +243,20 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): }; } + /** + * Clamps a valid level to the administrator ceiling. An unknown level is refused rather than + * narrowed, so a malformed request fails loudly instead of being stored as something else. + */ + function boundTrust( + trust: PRAutomationTrustLevel, + maxTrust: PRAutomationTrustLevel, + ): PRAutomationTrustLevel { + if (!PR_AUTOMATION_TRUST_LEVELS.includes(trust)) { + throw new RangeError('Unknown PR automation trust level'); + } + return clampPRAutomationTrust(trust, maxTrust); + } + function assertStopCode(stopCode: PRAutomationStopCode): void { if (!PR_AUTOMATION_STOP_CODES.includes(stopCode)) { throw new RangeError('A stopped PR automation needs a known stop code'); @@ -271,9 +288,13 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * or owner that is already gone writes nothing. One that passed the first check but lost a * race with the deletion and its cleanup removes the record it wrote, so a late enable cannot * recreate a record nobody will clean up. + * + * `maxTrust` is the administrator ceiling, resolved by the caller. A requested trust level + * is clamped to it before any write, so no record stores a level the deployment forbids. */ async function enablePRAutomation({ - trust, + trust: requestedTrust, + maxTrust, binding: requested, ...key }: t.EnablePRAutomationParams): Promise { @@ -281,6 +302,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): requested == null ? undefined : { ...requested, repository: canonicalRepository(requested.repository) }; + const trust = requestedTrust == null ? undefined : boundTrust(requestedTrust, maxTrust); const before = await checkLiveness(key); if (before !== 'live') { return { ok: false, error: { code: isGone(before) ? goneCode(before) : before } }; @@ -305,12 +327,18 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): runReset(attempt, { ...fields, pending: attempt }); const options = { runValidators: true }; let undo: { previous: t.IPRAutomation | null } | undefined; + /** Whether any write of this attempt applied; an enable that changed nothing has no commit. */ + let wrote = false; const write = async (condition: Record, update: Record) => { const before = await PRAutomation.findOneAndUpdate({ ...filter, ...condition }, update, { ...options, new: false, }).lean(); - if (before != null && before.epoch !== attempt) { + if (before == null) { + return; + } + wrote = true; + if (before.epoch !== attempt) { undo = { previous: before }; } }; @@ -325,13 +353,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): claimedHeads: [], epoch: attempt, pending: attempt, - trust: trust ?? 'approvedBots', + trust: clampPRAutomationTrust(trust, maxTrust), ...binding, }, }, { upsert: true, new: false, ...options }, ).lean(); if (existing == null) { + wrote = true; undo = { previous: null }; } /** A trust level passed with the request applies to every write that starts a run. */ @@ -372,7 +401,20 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } return { ok: false, error: { code: after } }; } - await PRAutomation.updateOne({ ...filter, pending: attempt }, { $unset: { pending: '' } }); + /** + * Commits only if this attempt's write is still the record. A later enable that replaced it + * decides the outcome instead, so this attempt reports a retryable conflict rather than a + * success that a rollback of the later enable could leave unclaimable. + */ + if (wrote) { + const committed = await PRAutomation.updateOne( + { ...filter, epoch: attempt, pending: attempt }, + { $unset: { pending: '' } }, + ); + if (committed.matchedCount === 0) { + return { ok: false, error: { code: 'conflict' } }; + } + } const record = await getPRAutomation(key); if (record == null) { throw new Error('PR automation record missing after enable'); @@ -625,14 +667,16 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } } + /** `maxTrust` is the administrator ceiling; a wider request is stored as the ceiling. */ async function setPRAutomationTrust( key: t.PRAutomationKey, trust: PRAutomationTrustLevel, + maxTrust: PRAutomationTrustLevel, ): Promise { const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( keyFilter(key), - { $set: { trust } }, + { $set: { trust: boundTrust(trust, maxTrust) } }, { new: true, select: PROJECTION, runValidators: true }, ).lean(); } diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 56d46394aa6..4bece5173e4 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -34,11 +34,8 @@ const prAutomationSchema: Schema = new Schema( type: String, index: true, }, - conversationId: { - type: String, - required: true, - maxlength: 256, - }, + /** Unbounded, like the conversation schema it mirrors, so every stored conversation can enable. */ + conversationId: { type: String, required: true }, repository: { type: String, maxlength: 256 }, pullNumber: { type: Number, diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index ba1cdffa5ca..ccba30f02e3 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -83,10 +83,13 @@ export type PRAutomationEventStopCode = Exclude Date: Fri, 16 Oct 2026 10:32:41 +0200 Subject: [PATCH 15/25] fix: Reap PR Automation Records the Retention Index Leaves Behind Conversations also leave through the TTL index on expiredAt, which never runs deleteConvos, so an idle automation record outlived its conversation until a delivery happened to arrive. reapPRAutomations pages through the records and removes those whose conversation, ancestor or owner is gone, reusing the claim-time liveness check, and the elected replica runs it hourly from both server entries. --- api/server/experimental.js | 3 ++ api/server/index.js | 3 ++ packages/api/src/agents/automation.spec.ts | 36 ++++++++++++++ packages/api/src/agents/automation.ts | 49 +++++++++++++++++++ packages/api/src/agents/index.ts | 1 + .../src/methods/prAutomation.spec.ts | 49 +++++++++++++++++++ .../data-schemas/src/methods/prAutomation.ts | 37 +++++++++++++- .../data-schemas/src/types/prAutomation.ts | 13 +++++ 8 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/agents/automation.spec.ts create mode 100644 packages/api/src/agents/automation.ts diff --git a/api/server/experimental.js b/api/server/experimental.js index ee6cf8c8b77..508bd79fa95 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -54,6 +54,7 @@ const { GenerationJobManager, createAgentEventTerminalHandler, startCodeEnvironmentLifecycleReconciler, + startPRAutomationReaper, waitForKeyvRedisClient, createCodeApiUploadRegistry, cacheConfig, @@ -80,6 +81,7 @@ const { seedDatabase, recordAuditEntry, sweepOrphanedPreviews, + reapPRAutomations, } = require('~/models'); const { checkMigrations } = require('./services/start/migration'); const initializeMCPs = require('./services/initializeMCPs'); @@ -453,6 +455,7 @@ if (cluster.isMaster) { await connectDb(); logger.info(`Worker ${process.pid}: Connected to MongoDB`); startCodeEnvironmentLifecycleReconciler({ mongoose }); + startPRAutomationReaper({ reapPRAutomations }); /** Background index sync (non-blocking) */ indexSync().catch((err) => { diff --git a/api/server/index.js b/api/server/index.js index 535e7a46c50..e61a3abc806 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -62,6 +62,7 @@ const { createAgentEventTerminalHandler, createScheduleWriteGate, startCodeEnvironmentLifecycleReconciler, + startPRAutomationReaper, waitForKeyvRedisClient, warnOnUnreachableDeliveryPaths, createCodeApiUploadRegistry, @@ -73,6 +74,7 @@ const { getRoleByName, seedDatabase, recordAuditEntry, + reapPRAutomations, } = require('~/models'); const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); const { capabilityContextMiddleware } = require('./middleware/roles/capabilities'); @@ -202,6 +204,7 @@ const startServer = async () => { logger.info('Connected to MongoDB'); startCodeEnvironmentLifecycleReconciler({ mongoose }); + startPRAutomationReaper({ reapPRAutomations }); indexSync().catch((err) => { logger.error('[indexSync] Background sync failed:', err); }); diff --git a/packages/api/src/agents/automation.spec.ts b/packages/api/src/agents/automation.spec.ts new file mode 100644 index 00000000000..9748cf328ba --- /dev/null +++ b/packages/api/src/agents/automation.spec.ts @@ -0,0 +1,36 @@ +import { reapOrphanedPRAutomations, startPRAutomationReaper } from './automation'; +import { isLeader } from '~/cluster'; + +jest.mock('~/cluster', () => ({ isLeader: jest.fn() })); + +describe('PR automation reaper', () => { + afterEach(() => { + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + test('follows the cursor until every page was read', async () => { + const reapPRAutomations = jest + .fn() + .mockResolvedValueOnce({ scanned: 100, removed: 2, next: 'a' }) + .mockResolvedValueOnce({ scanned: 3, removed: 1 }); + + expect(await reapOrphanedPRAutomations({ reapPRAutomations })).toBe(3); + expect(reapPRAutomations.mock.calls).toEqual([[{ after: undefined }], [{ after: 'a' }]]); + }); + + test('reaps only on the elected replica, then again every hour', async () => { + jest.useFakeTimers(); + const leader = jest.mocked(isLeader); + leader.mockResolvedValueOnce(false).mockResolvedValueOnce(true); + const reapPRAutomations = jest.fn().mockResolvedValue({ scanned: 0, removed: 0 }); + + startPRAutomationReaper({ reapPRAutomations }); + await jest.advanceTimersByTimeAsync(0); + expect(reapPRAutomations).not.toHaveBeenCalled(); + + await jest.advanceTimersByTimeAsync(60 * 60_000); + expect(leader).toHaveBeenCalledTimes(2); + expect(reapPRAutomations).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/agents/automation.ts b/packages/api/src/agents/automation.ts new file mode 100644 index 00000000000..b2cba64db8b --- /dev/null +++ b/packages/api/src/agents/automation.ts @@ -0,0 +1,49 @@ +import { logger, runAsSystem } from '@librechat/data-schemas'; +import type { PRAutomationMethods } from '@librechat/data-schemas'; +import { isLeader } from '~/cluster'; + +const REAP_INTERVAL_MS = 60 * 60_000; +let reapTimer: NodeJS.Timeout | undefined; +let reapInFlight: Promise | undefined; + +/** Reads every PR automation record once and removes those whose conversation or owner is gone. */ +export async function reapOrphanedPRAutomations({ + reapPRAutomations: reapPage, +}: Pick): Promise { + let removed = 0; + let after: string | undefined; + do { + const page = await reapPage({ after }); + removed += page.removed; + after = page.next; + } while (after != null); + return removed; +} + +/** + * Conversations also leave through the retention index, which runs no application code, so the + * elected replica periodically removes the PR automation records they leave behind. + */ +export function startPRAutomationReaper( + methods: Pick, +): void { + if (reapTimer != null) return; + const run = (): void => { + if (reapInFlight != null) return; + const current = isLeader() + .then(async (leader) => { + if (!leader) return; + await runAsSystem(() => reapOrphanedPRAutomations(methods)); + }) + .catch((error) => { + logger.error('[prAutomation] reaping orphaned records failed:', error); + }) + .finally(() => { + if (reapInFlight === current) reapInFlight = undefined; + }); + reapInFlight = current; + }; + run(); + reapTimer = setInterval(run, REAP_INTERVAL_MS); + reapTimer.unref(); +} diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index b8c34f34abd..0d4bd369414 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -1,5 +1,6 @@ export * from './avatars'; export * from './attachments'; +export * from './automation'; export * from './chain'; export * from './callerCapabilities'; export * from './client'; diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 52b68c67f59..241be6c8340 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -1090,6 +1090,55 @@ describe('claiming for a conversation that no longer exists', () => { }); }); +describe('reaping records the retention index left behind', () => { + test('removes an idle record once its conversation was purged, with no delivery', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + expect(await methods.reapPRAutomations()).toEqual({ scanned: 1, removed: 1 }); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('removes a record whose conversation is past its retention date', async () => { + await enable(); + await mongoose.models.Conversation.updateOne( + { conversationId: key.conversationId }, + { $set: { expiredAt: new Date(Date.now() - 60_000) } }, + ); + await methods.reapPRAutomations(); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('keeps live records and records whose owner deletion can still be cancelled', async () => { + await enable(); + const liveKey = { userId, conversationId: 'convo-live' }; + await seedConversation({ conversationId: liveKey.conversationId }); + await enableOpen({ ...liveKey, binding: pullOne }); + await mongoose.models.Conversation.deleteMany({ conversationId: key.conversationId }); + await startAccountDeletionNow(); + expect(await methods.reapPRAutomations()).toEqual({ scanned: 2, removed: 0 }); + await cancelAccountDeletion(); + expect(await methods.reapPRAutomations()).toEqual({ scanned: 2, removed: 1 }); + expect(await methods.getPRAutomation(liveKey)).not.toBeNull(); + }); + + test('pages through every record', async () => { + const ids = Array.from({ length: 5 }, (_, index) => `orphan-${index}`); + await PRAutomation.insertMany(ids.map((conversationId) => ({ user: userId, conversationId }))); + let next: string | undefined; + let removed = 0; + let pages = 0; + do { + const result = await methods.reapPRAutomations({ after: next, limit: 2 }); + removed += result.removed; + next = result.next; + pages++; + } while (next != null); + expect(pages).toBe(3); + expect(removed).toBe(5); + expect(await PRAutomation.countDocuments({})).toBe(0); + }); +}); + describe('settlePRAutomationRound', () => { test('settles the round that owns it, including from needs_user', async () => { await enable(); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index f7f896a21b9..7d1b26cc933 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -9,7 +9,7 @@ import { import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationState } from 'librechat-data-provider'; -import type { PipelineStage } from 'mongoose'; +import type { PipelineStage, Types } from 'mongoose'; import type * as t from '~/types/prAutomation'; import { isValidObjectIdString } from '~/utils/objectId'; import { createIndexesWithRetry } from '~/utils/retry'; @@ -110,6 +110,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ): Promise; }; deletePRAutomations: (userId: string, conversationIds?: string[]) => Promise; + reapPRAutomations: (params?: t.ReapPRAutomationsParams) => Promise; setPRAutomationTrust: ( key: t.PRAutomationKey, trust: PRAutomationTrustLevel, @@ -667,6 +668,39 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } } + /** + * Removes records whose conversation, an ancestor of it, or owner is gone, one page at a time + * in `_id` order. The retention index deletes conversations without running `deleteConvos`, + * so an idle record would otherwise outlive its conversation; this is the periodic pass that + * catches it. A deletion in progress keeps its records, as it does for a claim. + */ + async function reapPRAutomations({ + after, + limit = 100, + }: t.ReapPRAutomationsParams = {}): Promise { + const PRAutomation = mongoose.models.PRAutomation; + const page = await PRAutomation.find(after == null ? {} : { _id: { $gt: after } }) + .sort({ _id: 1 }) + .limit(limit) + .select({ _id: 1, user: 1, conversationId: 1 }) + .lean>(); + let removed = 0; + for (const record of page) { + const key = { userId: String(record.user), conversationId: record.conversationId }; + if (!isGone(await checkLiveness(key))) { + continue; + } + const result = await PRAutomation.deleteOne({ _id: record._id, ...keyFilter(key) }); + removed += result.deletedCount; + } + const last = page.length === limit ? page[page.length - 1] : undefined; + return { + scanned: page.length, + removed, + ...(last == null ? {} : { next: String(last._id) }), + }; + } + /** `maxTrust` is the administrator ceiling; a wider request is stored as the ceiling. */ async function setPRAutomationTrust( key: t.PRAutomationKey, @@ -754,6 +788,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): settlePRAutomationRound, stopPRAutomation, deletePRAutomations, + reapPRAutomations, setPRAutomationTrust, addPRAutomationBot, removePRAutomationBot, diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index ccba30f02e3..15c1dc91607 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -135,3 +135,16 @@ export type ClaimPRAutomationRoundResult = export type PRAutomationBotResult = | { ok: true; value: IPRAutomation } | { ok: false; error: { code: 'not_found' | 'bot_limit' | 'binding_mismatch' } }; + +export interface ReapPRAutomationsParams { + /** Continue after this record, as returned in `next` by the previous call. */ + after?: string; + limit?: number; +} + +export interface ReapPRAutomationsResult { + scanned: number; + removed: number; + /** Where the next call continues; absent once every record has been read. */ + next?: string; +} From 5f5fec5f215da4f46e0e40dd5c96bbd4c513a9e2 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:04:40 +0200 Subject: [PATCH 16/25] fix: Roll Back Only What a Rejected PR Automation Enable Wrote A rejected enable restored the whole preimage under an epoch-only guard, so a stop or a trust change that landed after its write, which keeps the epoch, was discarded when an account deletion started before the enable's second check. The rollback now restores only the fields the attempt wrote, and only while each still holds the value the attempt left; otherwise the record stays pending and unclaimable. --- .../src/methods/prAutomation.spec.ts | 69 +++++++++++++++++++ .../data-schemas/src/methods/prAutomation.ts | 67 +++++++++++++----- 2 files changed, 119 insertions(+), 17 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 241be6c8340..aec3028d46d 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -525,6 +525,75 @@ describe('enabling for a conversation or owner that no longer exists', () => { expect(await claim(1, { binding: pullTwo })).not.toMatchObject({ ok: true }); }); + /** + * Enables `binding` and runs `during` at the enable's second owner read: its writes are done + * and its rollback is next. Account deletion starts right after `during`. + */ + const enableCrossedBy = async (during: () => Promise, binding = pullTwo) => { + const User = mongoose.models.User; + const original = User.findById.bind(User); + let reads = 0; + const hook = async () => { + await during(); + await startAccountDeletionNow(); + }; + const spy = jest.spyOn(User, 'findById').mockImplementation((( + ...args: Parameters + ) => { + const query = original(...args); + if (++reads !== 2) { + return query; + } + const lean = query.lean.bind(query); + query.lean = ((...leanArgs: Parameters) => ({ + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + hook() + .then(() => lean(...leanArgs)) + .then(resolve, reject), + })) as unknown as typeof query.lean; + return query; + }) as typeof User.findById); + try { + return await enableOpen({ ...key, binding }); + } finally { + spy.mockRestore(); + await cancelAccountDeletion(); + } + }; + + test('keeps a user stop that lands while a rejected enable rolls back', async () => { + await enable(pullOne); + const result = await enableCrossedBy(() => methods.stopPRAutomation(key, 'user_stopped')); + expect(result).toEqual({ ok: false, error: { code: 'owner_inactive' } }); + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'stopped', + stopCode: 'user_stopped', + }); + expect(await claim(1)).not.toMatchObject({ ok: true }); + }); + + test('keeps a trust change that lands while a rejected enable rolls back', async () => { + await enable(pullOne); + const result = await enableCrossedBy(() => + methods.setPRAutomationTrust(key, 'collaborators', 'anyone'), + ); + expect(result).toEqual({ ok: false, error: { code: 'owner_inactive' } }); + const record = await methods.getPRAutomation(key); + expect(record).toMatchObject({ trust: 'collaborators', ...pullOne, state: 'idle' }); + expect(record?.pending).toBeUndefined(); + expect((await claim(1)).ok).toBe(true); + }); + + test('keeps a trust change on a record a rejected first enable inserted', async () => { + await PRAutomation.deleteMany({}); + await enableCrossedBy( + () => methods.setPRAutomationTrust(key, 'collaborators', 'anyone'), + pullOne, + ); + expect(await methods.getPRAutomation(key)).toMatchObject({ trust: 'collaborators' }); + expect(await claim(1)).not.toMatchObject({ ok: true }); + }); + test('does not report success when another enable replaced its write before it committed', async () => { await enable(pullOne); /** Another enable's write lands between this enable's second check and its commit. */ diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 7d1b26cc933..3e0e9cab89c 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -27,6 +27,8 @@ const COMMIT_SHA = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i; */ type Liveness = 'live' | 'conversation_gone' | 'owner_gone' | 'owner_inactive'; type Gone = Extract; +/** Field values an enable attempt left on the record; `undefined` marks a field it removed. */ +type Written = Record; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; @@ -328,6 +330,17 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): runReset(attempt, { ...fields, pending: attempt }); const options = { runValidators: true }; let undo: { previous: t.IPRAutomation | null } | undefined; + /** Every field this attempt wrote, with the value it left; `undefined` means it unset it. */ + const written: Written = {}; + const track = (update: { + $set?: Record; + $unset?: Record; + }) => { + Object.assign(written, update.$set); + for (const field of Object.keys(update.$unset ?? {})) { + written[field] = undefined; + } + }; /** Whether any write of this attempt applied; an enable that changed nothing has no commit. */ let wrote = false; const write = async (condition: Record, update: Record) => { @@ -339,29 +352,31 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return; } wrote = true; + track(update); if (before.epoch !== attempt) { undo = { previous: before }; } }; + const inserted = { + state: 'idle', + round: 0, + trustedBots: [], + claimedHeads: [], + epoch: attempt, + pending: attempt, + trust: clampPRAutomationTrust(trust, maxTrust), + ...binding, + }; const existing = await PRAutomation.findOneAndUpdate( filter, { - $setOnInsert: { - ...filter, - state: 'idle', - round: 0, - trustedBots: [], - claimedHeads: [], - epoch: attempt, - pending: attempt, - trust: clampPRAutomationTrust(trust, maxTrust), - ...binding, - }, + $setOnInsert: { ...filter, ...inserted }, }, { upsert: true, new: false, ...options }, ).lean(); if (existing == null) { wrote = true; + track({ $set: inserted }); undo = { previous: null }; } /** A trust level passed with the request applies to every write that starts a run. */ @@ -398,7 +413,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } if (after !== 'live') { if (undo != null) { - await undoEnable({ ...filter, epoch: attempt }, undo.previous); + await undoEnable(filter, written, undo.previous); } return { ok: false, error: { code: after } }; } @@ -424,20 +439,38 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } /** - * Puts the record back as this enable's own write found it when an account deletion started - * during the write, which can still be cancelled. `guard` names the epoch only this attempt - * wrote, so a record a later enable already replaced is left alone. + * Puts back what this enable wrote when an account deletion started during the write, which + * can still be cancelled. Only the fields the attempt wrote are restored, and only while every + * one of them still holds the value the attempt left, its epoch and pending mark included. A + * later enable that replaced the record, a stop, or a trust change made meanwhile therefore + * survives, and the record then stays pending, which nothing claims. */ async function undoEnable( - guard: ReturnType & { epoch: string }, + key: ReturnType, + written: Written, previous: t.IPRAutomation | null, ): Promise { const PRAutomation = mongoose.models.PRAutomation; + const guard: Record = { ...key }; + for (const [field, value] of Object.entries(written)) { + guard[field] = value === undefined ? { $exists: false } : value; + } if (previous == null) { await PRAutomation.deleteOne(guard); return; } - await PRAutomation.replaceOne(guard, previous); + const $set: Record = {}; + const $unset: Record = {}; + const stored = new Map(Object.entries(previous)); + for (const field of Object.keys(written)) { + if (stored.get(field) === undefined) { + $unset[field] = ''; + } else { + $set[field] = stored.get(field); + } + } + const update = Object.keys($unset).length > 0 ? { $set, $unset } : { $set }; + await PRAutomation.updateOne(guard, update, { timestamps: false }); } async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { From e6e5f5766e58083fca0c91b1266d512a68071e55 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:28:11 +0200 Subject: [PATCH 17/25] fix: Scope the PR Automation Reaper and Its Records to the Tenant The reaper runs as system, so its liveness check matched the conversation by user and id alone and could judge a record by another tenant's copy. It now passes each record's tenant into the first match. Records are unique per user, tenant and conversation, like the conversations they belong to. In the clustered server without Redis every worker counts as the leader, so the reaper now runs only in the worker the primary assigns the file-retention sweep to. --- api/server/experimental.js | 8 +++- packages/api/src/agents/automation.spec.ts | 46 ++++++++++++++++-- packages/api/src/agents/automation.ts | 48 ++++++++++++++++--- .../src/methods/prAutomation.spec.ts | 29 ++++++++++- .../data-schemas/src/methods/prAutomation.ts | 28 +++++++---- .../data-schemas/src/schema/prAutomation.ts | 11 +++-- 6 files changed, 144 insertions(+), 26 deletions(-) diff --git a/api/server/experimental.js b/api/server/experimental.js index 508bd79fa95..92b8fa9eb1e 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -54,7 +54,7 @@ const { GenerationJobManager, createAgentEventTerminalHandler, startCodeEnvironmentLifecycleReconciler, - startPRAutomationReaper, + createClusteredPRAutomationReaper, waitForKeyvRedisClient, createCodeApiUploadRegistry, cacheConfig, @@ -409,6 +409,9 @@ if (cluster.isMaster) { // Tear down stream resources before shared caches and telemetry exporters shut down. registerShutdownTask('generation job manager', destroyGenerationJobManager, { priority: 100 }); const expiredFileSweep = createClusteredFileSweep(cacheConfig.USE_REDIS, startExpiredFileSweep); + const prAutomationReaper = createClusteredPRAutomationReaper(cacheConfig.USE_REDIS, { + reapPRAutomations, + }); const SCHEDULE_ENGINE_OPTIONAL_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'DELETE']); const rejectScheduleWritesUntilReady = (req, res, next) => { @@ -440,6 +443,7 @@ if (cluster.isMaster) { if (msg.type === 'file-retention-sweep-worker') { logger.info(wrapLogMessage(`Worker ${process.pid} is assigned file-retention sweep`)); expiredFileSweep.assign(); + prAutomationReaper.assign(); } }); const startServer = async () => { @@ -455,7 +459,7 @@ if (cluster.isMaster) { await connectDb(); logger.info(`Worker ${process.pid}: Connected to MongoDB`); startCodeEnvironmentLifecycleReconciler({ mongoose }); - startPRAutomationReaper({ reapPRAutomations }); + prAutomationReaper.start(); /** Background index sync (non-blocking) */ indexSync().catch((err) => { diff --git a/packages/api/src/agents/automation.spec.ts b/packages/api/src/agents/automation.spec.ts index 9748cf328ba..17c65afa767 100644 --- a/packages/api/src/agents/automation.spec.ts +++ b/packages/api/src/agents/automation.spec.ts @@ -1,10 +1,21 @@ -import { reapOrphanedPRAutomations, startPRAutomationReaper } from './automation'; +import { + startPRAutomationReaper, + reapOrphanedPRAutomations, + createClusteredPRAutomationReaper, +} from './automation'; import { isLeader } from '~/cluster'; jest.mock('~/cluster', () => ({ isLeader: jest.fn() })); +const emptyPage = () => jest.fn().mockResolvedValue({ scanned: 0, removed: 0 }); + describe('PR automation reaper', () => { + let stop: (() => void) | undefined; + afterEach(() => { + stop?.(); + stop = undefined; + jest.mocked(isLeader).mockReset(); jest.clearAllTimers(); jest.useRealTimers(); }); @@ -23,9 +34,9 @@ describe('PR automation reaper', () => { jest.useFakeTimers(); const leader = jest.mocked(isLeader); leader.mockResolvedValueOnce(false).mockResolvedValueOnce(true); - const reapPRAutomations = jest.fn().mockResolvedValue({ scanned: 0, removed: 0 }); + const reapPRAutomations = emptyPage(); - startPRAutomationReaper({ reapPRAutomations }); + stop = startPRAutomationReaper({ reapPRAutomations }); await jest.advanceTimersByTimeAsync(0); expect(reapPRAutomations).not.toHaveBeenCalled(); @@ -33,4 +44,33 @@ describe('PR automation reaper', () => { expect(leader).toHaveBeenCalledTimes(2); expect(reapPRAutomations).toHaveBeenCalledTimes(1); }); + + test('without Redis, waits in a worker until the primary assigns it', async () => { + jest.useFakeTimers(); + jest.mocked(isLeader).mockResolvedValue(true); + const reapPRAutomations = emptyPage(); + const reaper = createClusteredPRAutomationReaper(false, { reapPRAutomations }); + + stop = reaper.stop; + reaper.start(); + await jest.advanceTimersByTimeAsync(0); + expect(reapPRAutomations).not.toHaveBeenCalled(); + + reaper.assign(); + stop = reaper.stop; + await jest.advanceTimersByTimeAsync(0); + expect(reapPRAutomations).toHaveBeenCalledTimes(1); + }); + + test('with Redis, every worker starts and leader election picks one', async () => { + jest.useFakeTimers(); + jest.mocked(isLeader).mockResolvedValue(true); + const reapPRAutomations = emptyPage(); + const reaper = createClusteredPRAutomationReaper(true, { reapPRAutomations }); + + reaper.start(); + stop = reaper.stop; + await jest.advanceTimersByTimeAsync(0); + expect(reapPRAutomations).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/api/src/agents/automation.ts b/packages/api/src/agents/automation.ts index b2cba64db8b..8589311265c 100644 --- a/packages/api/src/agents/automation.ts +++ b/packages/api/src/agents/automation.ts @@ -6,10 +6,12 @@ const REAP_INTERVAL_MS = 60 * 60_000; let reapTimer: NodeJS.Timeout | undefined; let reapInFlight: Promise | undefined; +type ReaperMethods = Pick; + /** Reads every PR automation record once and removes those whose conversation or owner is gone. */ export async function reapOrphanedPRAutomations({ reapPRAutomations: reapPage, -}: Pick): Promise { +}: ReaperMethods): Promise { let removed = 0; let after: string | undefined; do { @@ -22,12 +24,11 @@ export async function reapOrphanedPRAutomations({ /** * Conversations also leave through the retention index, which runs no application code, so the - * elected replica periodically removes the PR automation records they leave behind. + * elected replica periodically removes the PR automation records they leave behind. Returns a + * function that stops the schedule. */ -export function startPRAutomationReaper( - methods: Pick, -): void { - if (reapTimer != null) return; +export function startPRAutomationReaper(methods: ReaperMethods): () => void { + if (reapTimer != null) return stopPRAutomationReaper; const run = (): void => { if (reapInFlight != null) return; const current = isLeader() @@ -46,4 +47,39 @@ export function startPRAutomationReaper( run(); reapTimer = setInterval(run, REAP_INTERVAL_MS); reapTimer.unref(); + return stopPRAutomationReaper; +} + +function stopPRAutomationReaper(): void { + clearInterval(reapTimer); + reapTimer = undefined; +} + +/** + * Starts the reaper in one worker of a clustered server. Leader election needs Redis; without + * it every worker counts as the leader, so the worker the primary assigns the file-retention + * sweep to runs the reaper too. `start` marks the database ready, `assign` marks the worker. + */ +export function createClusteredPRAutomationReaper( + distributed: boolean, + methods: ReaperMethods, +): { start: () => void; assign: () => void; stop: () => void } { + let eligible = distributed; + let ready = false; + let stop: (() => void) | undefined; + const maybeStart = () => { + if (!eligible || !ready || stop != null) return; + stop = startPRAutomationReaper(methods); + }; + return { + start: () => { + ready = true; + maybeStart(); + }, + assign: () => { + eligible = true; + maybeStart(); + }, + stop: () => stop?.(), + }; } diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index aec3028d46d..f84abb6849c 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -3,8 +3,8 @@ import { logger, createModels } from '..'; import { MongoMemoryServer } from 'mongodb-memory-server'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationEventStopCode } from '~/types/prAutomation'; +import { runAsSystem, tenantStorage } from '~/config/tenantContext'; import { createPRAutomationMethods } from './prAutomation'; -import { tenantStorage } from '~/config/tenantContext'; logger.silent = true; @@ -72,6 +72,7 @@ const addBot = async (id: number, login?: string, limit = maxBots, repository = repository, epoch: (await methods.getPRAutomation(key))?.epoch ?? 'none', }); +const keyFilterOf = ({ userId: user, conversationId }: typeof key) => ({ user, conversationId }); const seedConversation = (fields: Record = {}) => mongoose.models.Conversation.create({ conversationId: key.conversationId, @@ -827,6 +828,17 @@ describe('pull request numbers', () => { }); describe('index guarantees', () => { + test('keeps one record per tenant for the same user and conversation id', async () => { + const inTenant = (tenantId: string, run: () => Promise) => + tenantStorage.run({ tenantId }, run); + await inTenant('tenant-a', () => PRAutomation.create(keyFilterOf(key))); + await inTenant('tenant-b', () => PRAutomation.create(keyFilterOf(key))); + await expect(inTenant('tenant-b', () => PRAutomation.create(keyFilterOf(key)))).rejects.toThrow( + /E11000/, + ); + expect(await runAsSystem(() => PRAutomation.countDocuments({}))).toBe(2); + }); + test('builds the unique index before the first write when automatic indexing is off', async () => { await PRAutomation.collection.dropIndexes(); const fresh = createPRAutomationMethods(mongoose); @@ -1190,6 +1202,21 @@ describe('reaping records the retention index left behind', () => { expect(await methods.getPRAutomation(liveKey)).not.toBeNull(); }); + test('judges each record by its own tenant when the same chat exists in two', async () => { + const inTenant = (tenantId: string, run: () => Promise) => + tenantStorage.run({ tenantId }, run); + await mongoose.models.Conversation.deleteMany({}); + await inTenant('tenant-b', () => seedConversation()); + await inTenant('tenant-a', () => PRAutomation.create({ ...keyFilterOf(key), ...pullOne })); + await inTenant('tenant-b', () => PRAutomation.create({ ...keyFilterOf(key), ...pullOne })); + expect(await runAsSystem(() => methods.reapPRAutomations())).toEqual({ + scanned: 2, + removed: 1, + }); + const left = await runAsSystem(() => PRAutomation.find({}).lean<{ tenantId?: string }[]>()); + expect(left.map((record) => record.tenantId)).toEqual(['tenant-b']); + }); + test('pages through every record', async () => { const ids = Array.from({ length: 5 }, (_, index) => `orphan-${index}`); await PRAutomation.insertMany(ids.map((conversationId) => ({ user: userId, conversationId }))); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 3e0e9cab89c..3f66ba51a45 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -29,6 +29,8 @@ type Liveness = 'live' | 'conversation_gone' | 'owner_gone' | 'owner_inactive'; type Gone = Extract; /** Field values an enable attempt left on the record; `undefined` marks a field it removed. */ type Written = Record; +/** The tenant a record belongs to; `null` or absent is a record stored without one. */ +type LivenessScope = { tenantId?: string | null }; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; @@ -53,11 +55,17 @@ const LINK_FIELDS = [ * `$lookup` per generation, so a liveness check is one round trip at any depth. `$graphLookup` * would be shorter but Amazon DocumentDB rejects it. Each lookup is served by the unique * `{ conversationId, user, tenantId }` index. Nested lookups bypass the tenant plugin, so - * ownership, tenant and retention are all judged on the result. + * ownership, tenant and retention are all judged on the result. The first match is scoped by + * the tenant plugin inside a tenant context; a caller without one, such as the reaper, names the + * record's tenant, since the same user and conversation id may exist in more than one tenant. */ -function lineagePipeline({ userId, conversationId }: t.PRAutomationKey): PipelineStage[] { +function lineagePipeline( + { userId, conversationId }: t.PRAutomationKey, + scope?: LivenessScope, +): PipelineStage[] { + const tenant = scope == null ? {} : { tenantId: scope.tenantId ?? null }; const stages: PipelineStage[] = [ - { $match: { user: userId, conversationId } }, + { $match: { user: userId, conversationId, ...tenant } }, { $limit: 1 }, { $project: { @@ -133,7 +141,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): let indexesPromise: Promise | null = null; /** - * The unique `{ user, conversationId }` index is what keeps concurrent first-time enables + * The unique `{ user, tenantId, conversationId }` index is what keeps concurrent first-time enables * from inserting two records that are then claimed independently and exceed the round cap. * With `MONGO_AUTO_INDEX=false` the schema declaration alone never builds it, so it is * ensured before the first write. @@ -165,7 +173,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * Deletions do not fence the record; enable and claim ask this on both sides of their write * instead. */ - async function checkLiveness(key: t.PRAutomationKey): Promise { + async function checkLiveness(key: t.PRAutomationKey, scope?: LivenessScope): Promise { if (!isValidObjectIdString(key.userId)) { return 'owner_inactive'; } @@ -181,7 +189,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): mongoose.models.User.findById(key.userId) .select('agentTriggerDeletionStartedAt') .lean<{ agentTriggerDeletionStartedAt?: Date }>(), - Conversation.aggregate>(lineagePipeline(key)), + Conversation.aggregate>(lineagePipeline(key, scope)), ]); if (owner == null) { return 'owner_gone'; @@ -715,12 +723,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const page = await PRAutomation.find(after == null ? {} : { _id: { $gt: after } }) .sort({ _id: 1 }) .limit(limit) - .select({ _id: 1, user: 1, conversationId: 1 }) - .lean>(); + .select({ _id: 1, user: 1, tenantId: 1, conversationId: 1 }) + .lean< + Array<{ _id: Types.ObjectId; user: string; tenantId?: string; conversationId: string }> + >(); let removed = 0; for (const record of page) { const key = { userId: String(record.user), conversationId: record.conversationId }; - if (!isGone(await checkLiveness(key))) { + if (!isGone(await checkLiveness(key, { tenantId: record.tenantId }))) { continue; } const result = await PRAutomation.deleteOne({ _id: record._id, ...keyFilter(key) }); diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 4bece5173e4..dc5318b897c 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -24,7 +24,7 @@ const trustedBotSchema = new Schema( const prAutomationSchema: Schema = new Schema( { - /** Served by the unique `{ user, conversationId }` index below; a second index only adds + /** Served by the unique `{ user, tenantId, conversationId }` index below; a second index only adds * write cost. */ user: { type: String, @@ -78,10 +78,11 @@ const prAutomationSchema: Schema = new Schema( ); /** - * One automation per user and conversation. The unique index makes enabling - * idempotent and race-free. It excludes `tenantId` for the same reason as - * `ToolFavorite`: user ObjectIds are globally unique. + * One automation per conversation, keyed like the conversation it belongs to: + * `{ conversationId, user, tenantId }` is unique there, so the same user and + * conversation id can exist in two tenants. The unique index makes enabling + * idempotent and race-free within a tenant. */ -prAutomationSchema.index({ user: 1, conversationId: 1 }, { unique: true }); +prAutomationSchema.index({ user: 1, tenantId: 1, conversationId: 1 }, { unique: true }); export default prAutomationSchema; From c13ca3750a483601f2e2059d7587dbbfa6e02bad Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:08:49 +0200 Subject: [PATCH 18/25] fix: Fence PR Automation Writes on a Record Version Writes that acted on an earlier read had nothing to compare against, so a user stop that landed between an enable's read and its write was revived, and a reaper or claim delete could remove a record renewed after its liveness check. Every write now increments version. Enable reads the record once and makes one write fenced on the version it read, returning a conflict when it moved; its rollback and commit fence on the version it wrote, which replaces the per-field rollback. With that read in hand, an enable also narrows a stored trust level above the current ceiling. A rejected claim still undoes itself by its run id, so a trust or bot change made meanwhile cannot strand the round. The reaper, the claim refusal and the enable's own cleanup delete only the version they judged. Account deletion removes automation records in every tenant. --- .../src/methods/prAutomation.spec.ts | 217 ++++++++++---- .../data-schemas/src/methods/prAutomation.ts | 277 +++++++++--------- .../src/methods/user.methods.spec.ts | 13 + packages/data-schemas/src/methods/user.ts | 4 +- .../data-schemas/src/schema/prAutomation.ts | 2 + .../data-schemas/src/types/prAutomation.ts | 2 + 6 files changed, 326 insertions(+), 189 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index f84abb6849c..ed93edd74c8 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -88,7 +88,7 @@ const seedConversation = (fields: Record = {}) => function interleave( during: () => Promise, model: mongoose.Model = PRAutomation, - method: 'findOneAndUpdate' | 'findById' = 'findOneAndUpdate', + method: 'findOneAndUpdate' | 'findById' | 'findOne' = 'findOneAndUpdate', ) { const original = (model[method] as typeof PRAutomation.findOneAndUpdate).bind(model); return jest.spyOn(model, method as 'findOneAndUpdate').mockImplementationOnce((( @@ -112,6 +112,43 @@ function interleave( return query; }) as typeof PRAutomation.findOneAndUpdate); } +/** + * Runs `during` right before the `nth` owner read. An enable reads the owner once before its + * write and once after it, so the second read of an enable is where its rollback is decided. + */ +function beforeOwnerRead(during: () => Promise, nth = 2) { + const User = mongoose.models.User; + const original = User.findById.bind(User); + let reads = 0; + return jest.spyOn(User, 'findById').mockImplementation((( + ...args: Parameters + ) => { + const query = original(...args); + if (++reads !== nth) { + return query; + } + const lean = query.lean.bind(query); + query.lean = ((...leanArgs: Parameters) => ({ + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + during() + .then(() => lean(...leanArgs)) + .then(resolve, reject), + })) as unknown as typeof query.lean; + return query; + }) as typeof User.findById); +} +/** Runs `during` right before the next `replaceOne` of the record, which is an enable's rollback. */ +function beforeRollback(during: () => Promise) { + const original = PRAutomation.replaceOne.bind(PRAutomation); + return jest.spyOn(PRAutomation, 'replaceOne').mockImplementationOnce((( + ...args: Parameters + ) => ({ + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + during() + .then(() => original(...args)) + .then(resolve, reject), + })) as unknown as typeof PRAutomation.replaceOne); +} /** What an event handler captured about the run it acts on. */ const fenceFor = async (binding = pullOne) => ({ ...binding, @@ -284,6 +321,27 @@ describe('claiming while the owner is being deleted', () => { expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'idle', round: 0 }); }); + test('undoes a rejected claim even when a trust change landed after it', async () => { + await enable(pullOne); + const spy = interleave(async () => { + await methods.setPRAutomationTrust(key, 'collaborators', 'anyone'); + await startAccountDeletion(); + }); + try { + expect(await claim(1)).toEqual({ ok: false, error: { code: 'owner_inactive' } }); + } finally { + spy.mockRestore(); + } + await cancelAccountDeletion(); + expect(await methods.getPRAutomation(key)).toMatchObject({ + state: 'idle', + round: 0, + claimedHeads: [], + trust: 'collaborators', + }); + expect((await claim(1)).ok).toBe(true); + }); + test('refuses a claim for an owner that no longer exists', async () => { await enable(pullOne); await mongoose.models.User.deleteMany({}); @@ -394,6 +452,21 @@ describe('a deletion that lands while a claim is in flight', () => { }); }); +describe('enabling under a lowered trust ceiling', () => { + test('restarts a stopped record at the current ceiling when no trust is requested', async () => { + await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); + await methods.stopPRAutomation(key, 'user_stopped'); + await enableWith({ ...key, binding: pullOne, maxTrust: 'approvedBots' }); + expect(await methods.getPRAutomation(key)).toMatchObject({ trust: 'approvedBots' }); + }); + + test('narrows a running record when it is enabled again', async () => { + await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); + await enableWith({ ...key, binding: pullOne, maxTrust: 'collaborators' }); + expect(await methods.getPRAutomation(key)).toMatchObject({ trust: 'collaborators' }); + }); +}); + describe('enabling for a conversation or owner that no longer exists', () => { test('does not leave a record behind for a deleted conversation', async () => { await mongoose.models.Conversation.deleteMany({}); @@ -409,8 +482,8 @@ describe('enabling for a conversation or owner that no longer exists', () => { }); }); - /** Runs `during` right after the enable's first write, before it checks again. */ - const interleaveEnable = (during: () => Promise) => interleave(during); + /** Runs `during` right after the enable's write, before it checks again. */ + const interleaveEnable = (during: () => Promise) => beforeOwnerRead(during); test('rejects an enable once account deletion started mid-write', async () => { const spy = interleaveEnable(startAccountDeletionNow); @@ -437,23 +510,18 @@ describe('enabling for a conversation or owner that no longer exists', () => { test('does not roll back an enable that landed after the crossing enable was rejected', async () => { await enable(pullOne); - const spy = interleaveEnable(async () => { - await startAccountDeletionNow(); - /** After the rejected enable reads the owner, the deletion is cancelled and a newer - * enable rebinds the record before the rejected one undoes its write. */ - interleave( - async () => { - await cancelAccountDeletion(); - await enable(pullTwo); - }, - mongoose.models.User as mongoose.Model, - 'findById', - ); + const spy = interleaveEnable(startAccountDeletionNow); + /** After the rejected enable reads the owner, the deletion is cancelled and a newer + * enable rebinds the record before the rejected one undoes its write. */ + const rollback = beforeRollback(async () => { + await cancelAccountDeletion(); + await enable(pullTwo); }); try { - await enableOpen({ ...key, binding: pullOne }); + await enableOpen({ ...key, binding: otherRepository }); } finally { spy.mockRestore(); + rollback.mockRestore(); } expect((await methods.getPRAutomation(key))?.pullNumber).toBe(2); }); @@ -531,29 +599,10 @@ describe('enabling for a conversation or owner that no longer exists', () => { * and its rollback is next. Account deletion starts right after `during`. */ const enableCrossedBy = async (during: () => Promise, binding = pullTwo) => { - const User = mongoose.models.User; - const original = User.findById.bind(User); - let reads = 0; - const hook = async () => { + const spy = beforeOwnerRead(async () => { await during(); await startAccountDeletionNow(); - }; - const spy = jest.spyOn(User, 'findById').mockImplementation((( - ...args: Parameters - ) => { - const query = original(...args); - if (++reads !== 2) { - return query; - } - const lean = query.lean.bind(query); - query.lean = ((...leanArgs: Parameters) => ({ - then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => - hook() - .then(() => lean(...leanArgs)) - .then(resolve, reject), - })) as unknown as typeof query.lean; - return query; - }) as typeof User.findById); + }); try { return await enableOpen({ ...key, binding }); } finally { @@ -562,6 +611,52 @@ describe('enabling for a conversation or owner that no longer exists', () => { } }; + test('keeps a user stop that lands between an enable reading the record and writing it', async () => { + await enable(pullOne); + const spy = interleave( + () => methods.stopPRAutomation(key, 'user_stopped'), + PRAutomation, + 'findOne', + ); + try { + expect(await enableOpen({ ...key, binding: pullTwo })).toEqual({ + ok: false, + error: { code: 'conflict' }, + }); + } finally { + spy.mockRestore(); + } + expect(await methods.getPRAutomation(key)).toMatchObject({ + ...pullOne, + state: 'stopped', + stopCode: 'user_stopped', + }); + }); + + test('moves the version on every write', async () => { + const versions: number[] = []; + const read = async () => versions.push((await methods.getPRAutomation(key))?.version ?? -1); + await enable(pullOne); + await read(); + await enable(pullOne); + await read(); + await methods.setPRAutomationTrust(key, 'collaborators', 'anyone'); + await read(); + await addBot(7); + await read(); + const run = await startRound(1); + await read(); + await settle(run.round, run.runId); + await read(); + await methods.removePRAutomationBot(key, 7, await fenceFor()); + await read(); + await methods.stopPRAutomation(key, 'user_stopped'); + await read(); + expect(versions.every((version, index) => index === 0 || version > versions[index - 1])).toBe( + true, + ); + }); + test('keeps a user stop that lands while a rejected enable rolls back', async () => { await enable(pullOne); const result = await enableCrossedBy(() => methods.stopPRAutomation(key, 'user_stopped')); @@ -579,9 +674,13 @@ describe('enabling for a conversation or owner that no longer exists', () => { methods.setPRAutomationTrust(key, 'collaborators', 'anyone'), ); expect(result).toEqual({ ok: false, error: { code: 'owner_inactive' } }); - const record = await methods.getPRAutomation(key); - expect(record).toMatchObject({ trust: 'collaborators', ...pullOne, state: 'idle' }); - expect(record?.pending).toBeUndefined(); + expect(await methods.getPRAutomation(key)).toMatchObject({ trust: 'collaborators' }); + expect(await claim(1, { binding: pullTwo })).not.toMatchObject({ ok: true }); + await enable(pullOne); + expect(await methods.getPRAutomation(key)).toMatchObject({ + trust: 'collaborators', + ...pullOne, + }); expect((await claim(1)).ok).toBe(true); }); @@ -598,16 +697,12 @@ describe('enabling for a conversation or owner that no longer exists', () => { test('does not report success when another enable replaced its write before it committed', async () => { await enable(pullOne); /** Another enable's write lands between this enable's second check and its commit. */ - const original = PRAutomation.updateOne.bind(PRAutomation); - const spy = jest - .spyOn(PRAutomation, 'updateOne') - .mockImplementationOnce(((...args: Parameters) => - PRAutomation.collection - .updateOne( - { user: userId, conversationId: key.conversationId }, - { $set: { epoch: 'other-attempt', pending: 'other-attempt' } }, - ) - .then(() => original(...args))) as unknown as typeof PRAutomation.updateOne); + const spy = beforeOwnerRead(() => + PRAutomation.collection.updateOne( + { user: userId, conversationId: key.conversationId }, + { $set: { epoch: 'other-attempt', pending: 'other-attempt' }, $inc: { version: 1 } }, + ), + ); try { const result = await methods.enablePRAutomation({ ...key, @@ -1217,6 +1312,28 @@ describe('reaping records the retention index left behind', () => { expect(left.map((record) => record.tenantId)).toEqual(['tenant-b']); }); + test('keeps a record renewed between its check and its delete', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + const original = PRAutomation.deleteOne.bind(PRAutomation); + const spy = jest.spyOn(PRAutomation, 'deleteOne').mockImplementationOnce((( + ...args: Parameters + ) => ({ + then: (resolve: (value: unknown) => unknown, reject: (error: unknown) => unknown) => + seedConversation() + .then(() => enable()) + .then(() => original(...args)) + .then(resolve, reject), + })) as unknown as typeof PRAutomation.deleteOne); + try { + expect(await methods.reapPRAutomations()).toEqual({ scanned: 1, removed: 0 }); + } finally { + spy.mockRestore(); + } + expect(await methods.getPRAutomation(key)).toMatchObject(pullOne); + expect((await claim(1)).ok).toBe(true); + }); + test('pages through every record', async () => { const ids = Array.from({ length: 5 }, (_, index) => `orphan-${index}`); await PRAutomation.insertMany(ids.map((conversationId) => ({ user: userId, conversationId }))); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 3f66ba51a45..0b5029baf6f 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -15,11 +15,11 @@ import { isValidObjectIdString } from '~/utils/objectId'; import { createIndexesWithRetry } from '~/utils/retry'; const PROJECTION = '-_id -__v'; -const RESTARTABLE = { state: 'stopped' }; /** Records removed per write, so a cleanup never holds one unbounded `$in`. */ const DELETE_BATCH = 1000; /** A full SHA-1 or SHA-256 commit id; anything else is refused before it is stored. */ const COMMIT_SHA = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i; +const DUPLICATE_KEY = 11000; /** * `owner_inactive` is an account deletion in progress, which can still be cancelled, so the @@ -27,8 +27,6 @@ const COMMIT_SHA = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i; */ type Liveness = 'live' | 'conversation_gone' | 'owner_gone' | 'owner_inactive'; type Gone = Extract; -/** Field values an enable attempt left on the record; `undefined` marks a field it removed. */ -type Written = Record; /** The tenant a record belongs to; `null` or absent is a record stored without one. */ type LivenessScope = { tenantId?: string | null }; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; @@ -39,6 +37,12 @@ const runReset = (epoch: string, fields: Record = {}) => ({ $set: { state: 'idle', round: 0, claimedHeads: [], epoch, ...fields }, $unset: { stopCode: '', startedAt: '', lastHeadSha: '', runId: '' }, }); +/** + * Every write to a record goes through `$inc: { version: 1 }`. A write that acts on a record it + * read first names the version it read, so any change in between, a stop or a trust change + * included, makes it miss instead of being overwritten. + */ +const BUMP = { $inc: { version: 1 } }; /** GitHub compares `owner/name` without case, so records store and match the lower case form. */ const canonicalRepository = (repository: string) => repository.toLowerCase(); const LINK_FIELDS = [ @@ -238,16 +242,22 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * A record whose conversation or owner is gone can never run again, so it is removed. This - * also converges a cleanup that failed after the delete committed. + * also converges a cleanup that failed after the delete committed. The delete names the + * version read before the liveness check, so a record renewed after the check is kept; without + * one nothing is removed and the reaper decides later. */ async function refuseClaim( key: t.PRAutomationKey, liveness: Exclude, + version: number | undefined, ): Promise { if (!isGone(liveness)) { return { ok: false, error: { code: liveness } }; } - const removed = await mongoose.models.PRAutomation.deleteOne(keyFilter(key)); + const removed = + version == null + ? { deletedCount: 0 } + : await mongoose.models.PRAutomation.deleteOne({ ...keyFilter(key), version }); return { ok: false, error: { code: removed.deletedCount > 0 ? goneCode(liveness) : 'not_found' }, @@ -301,7 +311,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * recreate a record nobody will clean up. * * `maxTrust` is the administrator ceiling, resolved by the caller. A requested trust level - * is clamped to it before any write, so no record stores a level the deployment forbids. + * is clamped to it before any write, and a stored level above it is narrowed, so an enabled + * record never keeps a level the deployment now forbids. */ async function enablePRAutomation({ trust: requestedTrust, @@ -322,118 +333,71 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const PRAutomation = mongoose.models.PRAutomation; const filter = keyFilter(key); /** - * Every write of this attempt stamps the same new epoch, so the record carries it only when - * this attempt changed it. Each write returns the record as it was just before that write, - * atomically with it, and the last one this attempt did not already own is what a crossed - * deletion restores: a later enable that landed between this attempt's read and write is - * kept, and nothing a later enable wrote afterwards is reverted. - * - * The writes also mark the record `pending` with the attempt, and nothing claims a pending - * record. Only an attempt that passes its second liveness check clears the mark. A preimage - * that another rejected attempt left is therefore still pending when it is restored, so two + * The enable reads the record once and makes at most one write, fenced on the version it + * read, so a stop, a trust change or another enable that lands in between makes it report a + * retryable `conflict` instead of reviving or overwriting what that change left. The write + * marks the record `pending` with the attempt, and nothing claims a pending record. Only an + * attempt that passes its second liveness check clears the mark; one that sees an account + * deletion start puts the record back as it read it, fenced on the version it wrote. A + * preimage another rejected attempt left is still pending when it is restored, so two * enables crossing the same deletion leave nothing claimable. */ const attempt = randomUUID(); - const reset = (fields: Record) => - runReset(attempt, { ...fields, pending: attempt }); - const options = { runValidators: true }; - let undo: { previous: t.IPRAutomation | null } | undefined; - /** Every field this attempt wrote, with the value it left; `undefined` means it unset it. */ - const written: Written = {}; - const track = (update: { - $set?: Record; - $unset?: Record; - }) => { - Object.assign(written, update.$set); - for (const field of Object.keys(update.$unset ?? {})) { - written[field] = undefined; - } - }; - /** Whether any write of this attempt applied; an enable that changed nothing has no commit. */ - let wrote = false; - const write = async (condition: Record, update: Record) => { - const before = await PRAutomation.findOneAndUpdate({ ...filter, ...condition }, update, { - ...options, - new: false, - }).lean(); - if (before == null) { - return; - } - wrote = true; - track(update); - if (before.epoch !== attempt) { - undo = { previous: before }; + const observed = await PRAutomation.findOne(filter).lean(); + let written: number; + let restartedRun = true; + if (observed == null) { + written = 1; + try { + await PRAutomation.create({ + ...filter, + state: 'idle', + round: 0, + trustedBots: [], + claimedHeads: [], + epoch: attempt, + pending: attempt, + version: written, + trust: clampPRAutomationTrust(trust, maxTrust), + ...binding, + }); + } catch (error) { + if ((error as { code?: number }).code === DUPLICATE_KEY) { + return { ok: false, error: { code: 'conflict' } }; + } + throw error; } - }; - const inserted = { - state: 'idle', - round: 0, - trustedBots: [], - claimedHeads: [], - epoch: attempt, - pending: attempt, - trust: clampPRAutomationTrust(trust, maxTrust), - ...binding, - }; - const existing = await PRAutomation.findOneAndUpdate( - filter, - { - $setOnInsert: { ...filter, ...inserted }, - }, - { upsert: true, new: false, ...options }, - ).lean(); - if (existing == null) { - wrote = true; - track({ $set: inserted }); - undo = { previous: null }; - } - /** A trust level passed with the request applies to every write that starts a run. */ - const revive = trust != null ? { trust } : {}; - if (binding == null) { - await write(RESTARTABLE, reset(revive)); } else { - const { repository } = binding; - const otherRepository = { repository: { $ne: repository } }; - await write( - { ...RESTARTABLE, ...otherRepository }, - reset({ ...revive, ...binding, trustedBots: [] }), - ); - await write({ ...RESTARTABLE, repository }, reset({ ...revive, ...binding })); - await write( - { state: { $ne: 'stopped' }, ...otherRepository }, - reset({ ...revive, ...binding, trustedBots: [] }), - ); - await write( - { state: { $ne: 'stopped' }, repository, pullNumber: { $ne: binding.pullNumber } }, - reset({ ...revive, ...binding }), + written = observed.version + 1; + const restart = enableRestart(observed, attempt, binding, trust ?? observed.trust, maxTrust); + restartedRun = restart != null; + /** An enable also narrows a stored level the current ceiling no longer allows. */ + const narrowed = clampPRAutomationTrust(observed.trust, maxTrust); + const update = restart ?? (narrowed === observed.trust ? {} : { $set: { trust: narrowed } }); + const applied = await PRAutomation.updateOne( + { ...filter, version: observed.version }, + { ...update, ...BUMP }, + { runValidators: true }, ); + if (applied.matchedCount === 0) { + return { ok: false, error: { code: 'conflict' } }; + } } - /** - * A record still pending for another attempt is either in flight or was left by an attempt - * that never finished. Taking it over under this attempt's epoch recovers the second case, - * and the in-flight attempt's own undo then misses on its epoch, so this attempt decides. - */ - await write({ pending: { $exists: true, $ne: attempt } }, reset({ ...revive, ...binding })); const after = await checkLiveness(key); if (isGone(after)) { - await PRAutomation.deleteOne(filter); + await PRAutomation.deleteOne({ ...filter, version: written }); return { ok: false, error: { code: goneCode(after) } }; } if (after !== 'live') { - if (undo != null) { - await undoEnable(filter, written, undo.previous); + if (restartedRun) { + await undoEnable(filter, written, observed); } return { ok: false, error: { code: after } }; } - /** - * Commits only if this attempt's write is still the record. A later enable that replaced it - * decides the outcome instead, so this attempt reports a retryable conflict rather than a - * success that a rollback of the later enable could leave unclaimable. - */ - if (wrote) { + if (restartedRun) { const committed = await PRAutomation.updateOne( - { ...filter, epoch: attempt, pending: attempt }, - { $unset: { pending: '' } }, + { ...filter, version: written, pending: attempt }, + { $unset: { pending: '' }, ...BUMP }, ); if (committed.matchedCount === 0) { return { ok: false, error: { code: 'conflict' } }; @@ -447,38 +411,54 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } /** - * Puts back what this enable wrote when an account deletion started during the write, which - * can still be cancelled. Only the fields the attempt wrote are restored, and only while every - * one of them still holds the value the attempt left, its epoch and pending mark included. A - * later enable that replaced the record, a stop, or a trust change made meanwhile therefore - * survives, and the record then stays pending, which nothing claims. + * The run-starting write an enable makes on an existing record, or `null` when the record is + * already running on the requested binding. A stopped record, a different binding, or a record + * another attempt left pending starts a fresh run under the attempt's epoch. A different + * repository also clears the approved bots, which belong to one repository. The run starts + * with the requested trust, or the stored one, clamped to the current ceiling. + */ + function enableRestart( + current: t.IPRAutomation, + attempt: string, + binding: t.PRAutomationBinding | undefined, + trust: PRAutomationTrustLevel, + maxTrust: PRAutomationTrustLevel, + ): ReturnType | null { + const otherRepository = binding != null && current.repository !== binding.repository; + const restart = + current.state === 'stopped' || + current.pending != null || + otherRepository || + (binding != null && current.pullNumber !== binding.pullNumber); + if (!restart) { + return null; + } + return runReset(attempt, { + trust: clampPRAutomationTrust(trust, maxTrust), + ...binding, + ...(otherRepository && { trustedBots: [] }), + pending: attempt, + }); + } + + /** + * Puts the record back as the enable read it when an account deletion started during the + * write, which can still be cancelled. It applies only while the record still carries the + * version the enable wrote, so anything that changed it since, a stop, a trust change or a + * later enable, is kept, and the record then stays pending, which nothing claims. */ async function undoEnable( key: ReturnType, - written: Written, - previous: t.IPRAutomation | null, + written: number, + previous: (t.IPRAutomation & { _id: unknown }) | null, ): Promise { const PRAutomation = mongoose.models.PRAutomation; - const guard: Record = { ...key }; - for (const [field, value] of Object.entries(written)) { - guard[field] = value === undefined ? { $exists: false } : value; - } + const guard = { ...key, version: written }; if (previous == null) { await PRAutomation.deleteOne(guard); return; } - const $set: Record = {}; - const $unset: Record = {}; - const stored = new Map(Object.entries(previous)); - for (const field of Object.keys(written)) { - if (stored.get(field) === undefined) { - $unset[field] = ''; - } else { - $set[field] = stored.get(field); - } - } - const update = Object.keys($unset).length > 0 ? { $set, $unset } : { $set }; - await PRAutomation.updateOne(guard, update, { timestamps: false }); + await PRAutomation.replaceOne(guard, { ...previous, version: written + 1 }); } async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { @@ -529,9 +509,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * Conversations also leave through the retention index, which never runs * `deleteConvos`, so liveness is checked here rather than trusted to a cascade. */ + const seen = await PRAutomation.findOne(keyFilter(key)) + .select({ _id: 0, version: 1 }) + .lean<{ version: number }>(); const before = await checkLiveness(key); if (before !== 'live') { - return refuseClaim(key, before); + return refuseClaim(key, before, seen?.version); } /** @@ -551,7 +534,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): }, { $min: { startedAt: now }, - $inc: { round: 1 }, + $inc: { round: 1, version: 1 }, $set: { state: 'fixing', lastHeadSha: headSha, runId }, $push: { claimedHeads: headSha }, $unset: { stopCode: '' }, @@ -573,9 +556,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } const previousHeads = claimed.claimedHeads.slice(0, -1); const lastHeadSha = previousHeads[previousHeads.length - 1]; - await PRAutomation.updateOne( + /** + * Fenced on the claim's own run identity rather than the version: a trust or bot change + * made meanwhile touches nothing the claim owns, and must not strand the round. + */ + const undone = await PRAutomation.findOneAndUpdate( { ...keyFilter(key), runId, state: 'fixing', round: claimed.round }, { + ...BUMP, $set: { state: lastHeadSha == null ? 'idle' : 'waiting', round: claimed.round - 1, @@ -587,8 +575,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ...(lastHeadSha == null && { startedAt: '', lastHeadSha: '' }), }, }, - ); - return refuseClaim(key, after); + { new: true, projection: { _id: 0, version: 1 } }, + ).lean<{ version: number }>(); + /** Only a claim that undid itself knows the version it left; otherwise the reaper decides. */ + return refuseClaim(key, after, undone?.version); } const current = await getPRAutomation(key); @@ -645,7 +635,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( { ...keyFilter(key), state: { $in: SETTLEABLE_STATES }, round, runId }, - { $set: { state } }, + { $set: { state }, ...BUMP }, { new: true, select: PROJECTION }, ).lean(); } @@ -682,7 +672,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( { ...keyFilter(key), ...condition, state: { $ne: 'stopped' } }, - { $set: { state: 'stopped', stopCode } }, + { $set: { state: 'stopped', stopCode }, ...BUMP }, { new: true, select: PROJECTION }, ).lean(); } @@ -723,9 +713,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const page = await PRAutomation.find(after == null ? {} : { _id: { $gt: after } }) .sort({ _id: 1 }) .limit(limit) - .select({ _id: 1, user: 1, tenantId: 1, conversationId: 1 }) + .select({ _id: 1, user: 1, tenantId: 1, conversationId: 1, version: 1 }) .lean< - Array<{ _id: Types.ObjectId; user: string; tenantId?: string; conversationId: string }> + Array<{ + _id: Types.ObjectId; + user: string; + tenantId?: string; + conversationId: string; + version: number; + }> >(); let removed = 0; for (const record of page) { @@ -733,7 +729,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (!isGone(await checkLiveness(key, { tenantId: record.tenantId }))) { continue; } - const result = await PRAutomation.deleteOne({ _id: record._id, ...keyFilter(key) }); + /** A record written after the check, a renewed enable included, has a new version. */ + const result = await PRAutomation.deleteOne({ + _id: record._id, + ...keyFilter(key), + version: record.version, + }); removed += result.deletedCount; } const last = page.length === limit ? page[page.length - 1] : undefined; @@ -753,7 +754,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( keyFilter(key), - { $set: { trust: boundTrust(trust, maxTrust) } }, + { $set: { trust: boundTrust(trust, maxTrust) }, ...BUMP }, { new: true, select: PROJECTION, runValidators: true }, ).lean(); } @@ -786,7 +787,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): 'trustedBots.id': { $ne: bot.id }, [`trustedBots.${maxBots - 1}`]: { $exists: false }, }, - { $push: { trustedBots: bot } }, + { $push: { trustedBots: bot }, ...BUMP }, { new: true, select: PROJECTION, runValidators: true }, ).lean(); if (updated != null) { @@ -818,7 +819,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const PRAutomation = mongoose.models.PRAutomation; return PRAutomation.findOneAndUpdate( { ...keyFilter(key), repository: canonicalRepository(repository), epoch }, - { $pull: { trustedBots: { id: botId } } }, + { $pull: { trustedBots: { id: botId } }, ...BUMP }, { new: true, select: PROJECTION }, ).lean(); } diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index 9acc6d3d060..f77b27b94ce 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -8,6 +8,7 @@ import { import type * as t from '~/types'; import { createToolApprovalGrantModel } from '~/models/toolApprovalGrant'; import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user'; +import { runAsSystem, tenantStorage } from '~/config/tenantContext'; import { createPRAutomationModel } from '~/models/prAutomation'; import { createConversationModel } from '~/models/convo'; import balanceSchema from '~/schema/balance'; @@ -2204,6 +2205,18 @@ describe('User Methods - Database Tests', () => { }); describe('PR automation cleanup on account deletion', () => { + it('removes the records in every tenant when the deletion runs in one', async () => { + const user = await User.create({ email: 'delete-pr-tenants@example.com', provider: 'local' }); + const id = user._id.toString(); + const PRAutomation = mongoose.models.PRAutomation; + const inTenant = (tenantId: string, run: () => Promise) => + tenantStorage.run({ tenantId }, run); + await inTenant('tenant-a', () => PRAutomation.create({ user: id, conversationId: 'chat-a' })); + await inTenant('tenant-b', () => PRAutomation.create({ user: id, conversationId: 'chat-a' })); + await inTenant('tenant-a', () => methods.deleteUserById(id)); + expect(await runAsSystem(() => PRAutomation.countDocuments({ user: id }))).toBe(0); + }); + it('keeps the records exactly as they were when the account delete fails', async () => { const user = await User.create({ email: 'delete-pr-fail@example.com', provider: 'local' }); const id = user._id.toString(); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index b9fadcbc109..64bdc2a4528 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -17,6 +17,7 @@ import type { import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types'; import type { CacheStore } from '~/types'; import { createPRAutomationMethods } from './prAutomation'; +import { runAsSystem } from '~/config/tenantContext'; import { evictAuthUserDocs } from '~/utils/eviction'; import { escapeRegExp } from '~/utils/string'; import { signPayload } from '~/crypto'; @@ -656,7 +657,8 @@ export function createUserMethods( * failed: the next claim removes any record whose owner no longer exists. A retry on an * account that was already gone has committed nothing, so its failure still throws. */ try { - await prAutomation.deletePRAutomations(userId); + /** The account is gone in every tenant, so its automation is removed from all of them. */ + await runAsSystem(() => prAutomation.deletePRAutomations(userId)); } catch (error) { if (result.deletedCount === 0) { throw error; diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index dc5318b897c..8724debc2fd 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -73,6 +73,8 @@ const prAutomationSchema: Schema = new Schema( runId: { type: String, maxlength: 64 }, epoch: { type: String, maxlength: 64, required: true, default: () => randomUUID() }, pending: { type: String, maxlength: 64 }, + /** Moves on every write, so a write based on an earlier read can be fenced on it. */ + version: { type: Number, default: 0, min: 0, required: true }, }, { timestamps: true }, ); diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 15c1dc91607..705c1729afb 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -51,6 +51,8 @@ export interface IPRAutomation { * one entry per round, so the round cap bounds it. */ claimedHeads: string[]; + /** Moves on every write; a write based on an earlier read is fenced on it. */ + version: number; createdAt?: Date; updatedAt?: Date; } From c02ba484ea5168732475f7208d3f9ea82773cb6e Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:28:19 +0200 Subject: [PATCH 19/25] fix: Hand Back a PR Automation Claim When Its Second Check Fails A claim whose post-write liveness read threw had already spent a round and recorded the head, so a retry saw a record that was not active and nothing ever released it. The claim now undoes itself by its run id before rethrowing, through the same release a rejected claim uses. --- .../src/methods/prAutomation.spec.ts | 22 ++++++ .../data-schemas/src/methods/prAutomation.ts | 67 ++++++++++++------- 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index ed93edd74c8..27f182cf12e 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -342,6 +342,28 @@ describe('claiming while the owner is being deleted', () => { expect((await claim(1)).ok).toBe(true); }); + test('hands the round back when the check after the claim fails', async () => { + await enable(pullOne); + const Conversation = mongoose.models.Conversation; + const original = Conversation.aggregate.bind(Conversation); + let calls = 0; + const spy = jest.spyOn(Conversation, 'aggregate').mockImplementation((( + ...args: Parameters + ) => { + if (++calls === 2) { + throw new Error('synthetic read failure'); + } + return original(...args); + }) as typeof Conversation.aggregate); + try { + await expect(claim(1)).rejects.toThrow('synthetic read failure'); + } finally { + spy.mockRestore(); + } + expect(await methods.getPRAutomation(key)).toMatchObject({ state: 'idle', round: 0 }); + expect((await claim(1)).ok).toBe(true); + }); + test('refuses a claim for an owner that no longer exists', async () => { await enable(pullOne); await mongoose.models.User.deleteMany({}); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 0b5029baf6f..35bf2155a9e 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -550,35 +550,20 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * round, so the state before the claim follows from the claimed record. A record whose * conversation or owner is gone is removed. */ - const after = await checkLiveness(key); + let after: Liveness; + try { + after = await checkLiveness(key); + } catch (error) { + /** A check that fails leaves no answer to act on, so the round is handed back. */ + await releaseClaim(key, claimed, runId).catch(() => undefined); + throw error; + } if (after === 'live') { return { ok: true, value: { ...claimed, runId } }; } - const previousHeads = claimed.claimedHeads.slice(0, -1); - const lastHeadSha = previousHeads[previousHeads.length - 1]; - /** - * Fenced on the claim's own run identity rather than the version: a trust or bot change - * made meanwhile touches nothing the claim owns, and must not strand the round. - */ - const undone = await PRAutomation.findOneAndUpdate( - { ...keyFilter(key), runId, state: 'fixing', round: claimed.round }, - { - ...BUMP, - $set: { - state: lastHeadSha == null ? 'idle' : 'waiting', - round: claimed.round - 1, - claimedHeads: previousHeads, - ...(lastHeadSha != null && { lastHeadSha }), - }, - $unset: { - runId: '', - ...(lastHeadSha == null && { startedAt: '', lastHeadSha: '' }), - }, - }, - { new: true, projection: { _id: 0, version: 1 } }, - ).lean<{ version: number }>(); + const released = await releaseClaim(key, claimed, runId); /** Only a claim that undid itself knows the version it left; otherwise the reaper decides. */ - return refuseClaim(key, after, undone?.version); + return refuseClaim(key, after, released?.version); } const current = await getPRAutomation(key); @@ -612,6 +597,38 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return { ok: false, error: { code: 'conflict' } }; } + /** + * Undoes a claim that will not run. It is fenced on the claim's own run identity rather than + * the version: a trust or bot change made meanwhile touches nothing the claim owns, and must + * not strand the round. Rounds and heads are claimed one for one and the window opens with the + * first round, so the state before the claim follows from the claimed record. + */ + async function releaseClaim( + key: t.PRAutomationKey, + claimed: t.IPRAutomation, + runId: string, + ): Promise<{ version: number } | null> { + const previousHeads = claimed.claimedHeads.slice(0, -1); + const lastHeadSha = previousHeads[previousHeads.length - 1]; + return mongoose.models.PRAutomation.findOneAndUpdate( + { ...keyFilter(key), runId, state: 'fixing', round: claimed.round }, + { + ...BUMP, + $set: { + state: lastHeadSha == null ? 'idle' : 'waiting', + round: claimed.round - 1, + claimedHeads: previousHeads, + ...(lastHeadSha != null && { lastHeadSha }), + }, + $unset: { + runId: '', + ...(lastHeadSha == null && { startedAt: '', lastHeadSha: '' }), + }, + }, + { new: true, projection: { _id: 0, version: 1 } }, + ).lean<{ version: number }>(); + } + /** * Reports where the round that owns the record ended up. The transition only * applies while that round is still the current one, identified by its From c3f43bcf053e658c7e94d16fc1abe62575a6ea7f Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:43:02 +0200 Subject: [PATCH 20/25] feat: Configure the PR Automation Reaper Interval The reaper ran on a fixed hour. endpoints.agents.prAutomation.reaperIntervalMinutes sets it, from 5 to 1440 minutes with a default of 60, and both server entries pass the loaded base config to the reaper so the yaml value is the one that runs. --- api/server/experimental.js | 2 +- api/server/index.js | 2 +- packages/api/src/agents/automation.spec.ts | 15 +++++++++++++ packages/api/src/agents/automation.ts | 21 ++++++++++++------- packages/data-provider/src/config.ts | 15 ++++++++++++- .../data-provider/src/prAutomation.spec.ts | 9 ++++++++ .../data-provider/src/types/prAutomation.ts | 4 ++++ packages/data-schemas/src/app/service.spec.ts | 12 +++++++++++ 8 files changed, 69 insertions(+), 11 deletions(-) diff --git a/api/server/experimental.js b/api/server/experimental.js index 92b8fa9eb1e..194b01ebb71 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -459,7 +459,6 @@ if (cluster.isMaster) { await connectDb(); logger.info(`Worker ${process.pid}: Connected to MongoDB`); startCodeEnvironmentLifecycleReconciler({ mongoose }); - prAutomationReaper.start(); /** Background index sync (non-blocking) */ indexSync().catch((err) => { @@ -512,6 +511,7 @@ if (cluster.isMaster) { // principal) still merges DB `__base__` overrides, which must not drive which hook // modules load in every worker (matches api/server/index.js's baseOnly usage). const baseAppConfig = await getAppConfig({ baseOnly: true }); + prAutomationReaper.start(baseAppConfig); await configureSubagentTaskRouting(baseAppConfig?.endpoints?.agents?.subagentActivity); registerBackgroundTaskShutdown({ interruptGraceMs: baseAppConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs, diff --git a/api/server/index.js b/api/server/index.js index e61a3abc806..6307c561c8f 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -204,7 +204,6 @@ const startServer = async () => { logger.info('Connected to MongoDB'); startCodeEnvironmentLifecycleReconciler({ mongoose }); - startPRAutomationReaper({ reapPRAutomations }); indexSync().catch((err) => { logger.error('[indexSync] Background sync failed:', err); }); @@ -239,6 +238,7 @@ const startServer = async () => { logger.error('[sweepOrphanedPreviews] Background sweep failed:', err); }); const appConfig = await getAppConfig({ baseOnly: true }); + startPRAutomationReaper({ reapPRAutomations }, appConfig); await configureSubagentTaskRouting(appConfig?.endpoints?.agents?.subagentActivity); registerBackgroundTaskShutdown({ interruptGraceMs: appConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs, diff --git a/packages/api/src/agents/automation.spec.ts b/packages/api/src/agents/automation.spec.ts index 17c65afa767..6c5eb9b8239 100644 --- a/packages/api/src/agents/automation.spec.ts +++ b/packages/api/src/agents/automation.spec.ts @@ -1,3 +1,4 @@ +import { AppService } from '@librechat/data-schemas'; import { startPRAutomationReaper, reapOrphanedPRAutomations, @@ -45,6 +46,20 @@ describe('PR automation reaper', () => { expect(reapPRAutomations).toHaveBeenCalledTimes(1); }); + test('runs at the interval the yaml sets', async () => { + jest.useFakeTimers(); + jest.mocked(isLeader).mockResolvedValue(true); + const reapPRAutomations = emptyPage(); + const appConfig = await AppService({ + config: { endpoints: { agents: { prAutomation: { reaperIntervalMinutes: 15 } } } }, + }); + + stop = startPRAutomationReaper({ reapPRAutomations }, appConfig); + await jest.advanceTimersByTimeAsync(0); + await jest.advanceTimersByTimeAsync(15 * 60_000); + expect(reapPRAutomations).toHaveBeenCalledTimes(2); + }); + test('without Redis, waits in a worker until the primary assigns it', async () => { jest.useFakeTimers(); jest.mocked(isLeader).mockResolvedValue(true); diff --git a/packages/api/src/agents/automation.ts b/packages/api/src/agents/automation.ts index 8589311265c..e42b0c56e70 100644 --- a/packages/api/src/agents/automation.ts +++ b/packages/api/src/agents/automation.ts @@ -1,8 +1,8 @@ import { logger, runAsSystem } from '@librechat/data-schemas'; -import type { PRAutomationMethods } from '@librechat/data-schemas'; +import { DEFAULT_PR_AUTOMATION_REAPER_MINUTES } from 'librechat-data-provider'; +import type { AppConfig, PRAutomationMethods } from '@librechat/data-schemas'; import { isLeader } from '~/cluster'; -const REAP_INTERVAL_MS = 60 * 60_000; let reapTimer: NodeJS.Timeout | undefined; let reapInFlight: Promise | undefined; @@ -27,7 +27,7 @@ export async function reapOrphanedPRAutomations({ * elected replica periodically removes the PR automation records they leave behind. Returns a * function that stops the schedule. */ -export function startPRAutomationReaper(methods: ReaperMethods): () => void { +export function startPRAutomationReaper(methods: ReaperMethods, appConfig?: AppConfig): () => void { if (reapTimer != null) return stopPRAutomationReaper; const run = (): void => { if (reapInFlight != null) return; @@ -45,7 +45,10 @@ export function startPRAutomationReaper(methods: ReaperMethods): () => void { reapInFlight = current; }; run(); - reapTimer = setInterval(run, REAP_INTERVAL_MS); + const minutes = + appConfig?.endpoints?.agents?.prAutomation?.reaperIntervalMinutes ?? + DEFAULT_PR_AUTOMATION_REAPER_MINUTES; + reapTimer = setInterval(run, minutes * 60_000); reapTimer.unref(); return stopPRAutomationReaper; } @@ -58,21 +61,23 @@ function stopPRAutomationReaper(): void { /** * Starts the reaper in one worker of a clustered server. Leader election needs Redis; without * it every worker counts as the leader, so the worker the primary assigns the file-retention - * sweep to runs the reaper too. `start` marks the database ready, `assign` marks the worker. + * sweep to runs the reaper too. `start` passes the loaded config, `assign` marks the worker. */ export function createClusteredPRAutomationReaper( distributed: boolean, methods: ReaperMethods, -): { start: () => void; assign: () => void; stop: () => void } { +): { start: (appConfig?: AppConfig) => void; assign: () => void; stop: () => void } { let eligible = distributed; let ready = false; + let config: AppConfig | undefined; let stop: (() => void) | undefined; const maybeStart = () => { if (!eligible || !ready || stop != null) return; - stop = startPRAutomationReaper(methods); + stop = startPRAutomationReaper(methods, config); }; return { - start: () => { + start: (appConfig) => { + config = appConfig; ready = true; maybeStart(); }, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 2d1bf6697eb..aed97fc810b 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -70,9 +70,12 @@ import { import type { PullRequestProvider } from './types/pullRequest'; import { isActionTool } from './types/tools'; import { + MAX_PR_AUTOMATION_BOTS, PR_AUTOMATION_TRUST_LEVELS, DEFAULT_PR_AUTOMATION_BOTS, - MAX_PR_AUTOMATION_BOTS, + MIN_PR_AUTOMATION_REAPER_MINUTES, + MAX_PR_AUTOMATION_REAPER_MINUTES, + DEFAULT_PR_AUTOMATION_REAPER_MINUTES, } from './types/prAutomation'; import { apiBaseUrl } from './api-endpoints'; import { FileSources } from './types/files'; @@ -2039,6 +2042,16 @@ export const agentsEndpointSchema = baseEndpointSchema /** Widest author trust a user may select: `approvedBots` acts on the linked account and * allowlisted bots only, `collaborators` adds write-access users, `anyone` acts on all. */ maxTrust: z.enum(PR_AUTOMATION_TRUST_LEVELS).optional().default('approvedBots'), + /** Minutes between passes that remove records whose conversation or owner is gone, + * including conversations the retention index deleted. Runs whether or not `enabled` + * is set, so records from an earlier opt-in are still cleaned up. */ + reaperIntervalMinutes: z + .number() + .int() + .min(MIN_PR_AUTOMATION_REAPER_MINUTES) + .max(MAX_PR_AUTOMATION_REAPER_MINUTES) + .optional() + .default(DEFAULT_PR_AUTOMATION_REAPER_MINUTES), }) .optional(), /** Conversational background-task delivery policy. Automatic completion wakeups are diff --git a/packages/data-provider/src/prAutomation.spec.ts b/packages/data-provider/src/prAutomation.spec.ts index 2d825213876..5999f6c0255 100644 --- a/packages/data-provider/src/prAutomation.spec.ts +++ b/packages/data-provider/src/prAutomation.spec.ts @@ -45,9 +45,18 @@ describe('agent PR automation config', () => { maxMinutes: 120, maxBots: 20, maxTrust: 'approvedBots', + reaperIntervalMinutes: 60, }); }); + it.each([4, 1441, 1.5])('rejects a %s minute reaper interval', (reaperIntervalMinutes) => { + expect(parse({ reaperIntervalMinutes }).success).toBe(false); + }); + + it('accepts a shorter reaper interval', () => { + expect(parse({ reaperIntervalMinutes: 15 }).success).toBe(true); + }); + it('accepts an administrator raising the limits and the ceiling', () => { const result = parse({ enabled: true, diff --git a/packages/data-provider/src/types/prAutomation.ts b/packages/data-provider/src/types/prAutomation.ts index 90c2b8d70ac..967a8c78808 100644 --- a/packages/data-provider/src/types/prAutomation.ts +++ b/packages/data-provider/src/types/prAutomation.ts @@ -18,6 +18,10 @@ export const PR_AUTOMATION_STOP_CODES = [ /** Largest per-repository bot allowlist a deployment can configure. */ export const MAX_PR_AUTOMATION_BOTS = 100; export const DEFAULT_PR_AUTOMATION_BOTS = 20; +/** Minutes between passes that remove records whose conversation or owner is gone. */ +export const DEFAULT_PR_AUTOMATION_REAPER_MINUTES = 60; +export const MIN_PR_AUTOMATION_REAPER_MINUTES = 5; +export const MAX_PR_AUTOMATION_REAPER_MINUTES = 1440; /** Ordered from narrowest to widest: who may be acted on. */ export const PR_AUTOMATION_TRUST_LEVELS = ['approvedBots', 'collaborators', 'anyone'] as const; diff --git a/packages/data-schemas/src/app/service.spec.ts b/packages/data-schemas/src/app/service.spec.ts index 92d59a5aa76..3863b274529 100644 --- a/packages/data-schemas/src/app/service.spec.ts +++ b/packages/data-schemas/src/app/service.spec.ts @@ -220,6 +220,18 @@ describe('loadFiltersConfig', () => { }); }); +describe('AppService PR automation config', () => { + it('carries the yaml reaper interval onto the app config', async () => { + const config = { + endpoints: { [EModelEndpoint.agents]: { prAutomation: { reaperIntervalMinutes: 15 } } }, + } as DeepPartial; + + const result = await AppService({ config }); + + expect(result.endpoints?.[EModelEndpoint.agents]?.prAutomation?.reaperIntervalMinutes).toBe(15); + }); +}); + describe('AppService assistants config', () => { it('preserves configured Assistants API versions', async () => { const config = { From de1565c9c96ed02c00eeebc67191c27e3b0c8749 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:43:13 +0200 Subject: [PATCH 21/25] perf: Check PR Automation Records a Few at a Time When Reaping The reaper checked each record of a page one after another, two reads per record, so a large collection could take longer than the interval. It now runs up to eight liveness checks at once within a page, keeping the page order and the cursor. --- .../src/methods/prAutomation.spec.ts | 29 +++++++++++++++++++ .../data-schemas/src/methods/prAutomation.ts | 17 ++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 27f182cf12e..6cbd620a2c2 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -1356,6 +1356,35 @@ describe('reaping records the retention index left behind', () => { expect((await claim(1)).ok).toBe(true); }); + test('checks a few records at a time within a page', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `orphan-${index}`); + await PRAutomation.insertMany(ids.map((conversationId) => ({ user: userId, conversationId }))); + const Conversation = mongoose.models.Conversation; + const original = Conversation.aggregate.bind(Conversation); + let inFlight = 0; + let peak = 0; + const spy = jest.spyOn(Conversation, 'aggregate').mockImplementation((( + ...args: Parameters + ) => { + inFlight++; + peak = Math.max(peak, inFlight); + const query = original(...args); + const exec = query.exec.bind(query); + query.exec = (() => + exec().finally(() => { + inFlight--; + })) as typeof query.exec; + return query; + }) as typeof Conversation.aggregate); + try { + expect(await methods.reapPRAutomations()).toEqual({ scanned: 20, removed: 20 }); + } finally { + spy.mockRestore(); + } + expect(peak).toBeGreaterThan(1); + expect(peak).toBeLessThanOrEqual(8); + }); + test('pages through every record', async () => { const ids = Array.from({ length: 5 }, (_, index) => `orphan-${index}`); await PRAutomation.insertMany(ids.map((conversationId) => ({ user: userId, conversationId }))); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 35bf2155a9e..8996649f572 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -20,6 +20,8 @@ const DELETE_BATCH = 1000; /** A full SHA-1 or SHA-256 commit id; anything else is refused before it is stored. */ const COMMIT_SHA = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i; const DUPLICATE_KEY = 11000; +/** Liveness checks the reaper runs at once within a page. */ +const REAP_CONCURRENCY = 8; /** * `owner_inactive` is an account deletion in progress, which can still be cancelled, so the @@ -740,11 +742,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): version: number; }> >(); - let removed = 0; - for (const record of page) { + const reap = async (record: (typeof page)[number]): Promise => { const key = { userId: String(record.user), conversationId: record.conversationId }; if (!isGone(await checkLiveness(key, { tenantId: record.tenantId }))) { - continue; + return 0; } /** A record written after the check, a renewed enable included, has a new version. */ const result = await PRAutomation.deleteOne({ @@ -752,7 +753,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ...keyFilter(key), version: record.version, }); - removed += result.deletedCount; + return result.deletedCount; + }; + let removed = 0; + /** A few checks at a time, so a large page neither runs serially nor floods the database. */ + for (let start = 0; start < page.length; start += REAP_CONCURRENCY) { + const counts = await Promise.all(page.slice(start, start + REAP_CONCURRENCY).map(reap)); + for (const count of counts) { + removed += count; + } } const last = page.length === limit ? page[page.length - 1] : undefined; return { From c188e21143a1b76d7441b3b277dce8cf8db9ac14 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:21:41 +0200 Subject: [PATCH 22/25] fix: Bind PR Automation Records to Their Conversation Document A conversation deleted and saved again under the same id looked like the original, so its old record stayed claimable and a cascade delete could remove a record enabled for the replacement. Records now store conversationRef, the _id of the conversation they were enabled for. Claims and the reaper treat a different document as gone, an enable on an earlier conversation's record starts over for the new one, and the conversation cascade deletes only records of the documents it removed. --- .../src/methods/conversation.spec.ts | 37 ++++++ .../data-schemas/src/methods/conversation.ts | 10 +- .../src/methods/prAutomation.spec.ts | 46 ++++++- .../data-schemas/src/methods/prAutomation.ts | 125 +++++++++++++----- .../data-schemas/src/schema/prAutomation.ts | 2 + .../data-schemas/src/types/prAutomation.ts | 5 + 6 files changed, 184 insertions(+), 41 deletions(-) diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 9bd94db2f55..4e3bff36d1c 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -3743,6 +3743,43 @@ describe('Conversation Operations', () => { expect(await PRAutomation.countDocuments({ conversationId: keptConversationId })).toBe(1); }); + it('keeps a record enabled for the conversation saved again before the cleanup', async () => { + const conversationId = uuidv4(); + const PRAutomation = mongoose.models.PRAutomation; + const deleted = await Conversation.create({ + conversationId, + user: 'user123', + endpoint: 'agents', + }); + await PRAutomation.create({ + user: 'user123', + conversationId, + conversationRef: String(deleted._id), + }); + const original = Conversation.deleteMany.bind(Conversation); + const spy = jest.spyOn(Conversation, 'deleteMany').mockImplementationOnce((( + ...args: Parameters + ) => + original(...args).then(async (result) => { + const renewed = await Conversation.create({ + conversationId, + user: 'user123', + endpoint: 'agents', + }); + await PRAutomation.updateOne( + { user: 'user123', conversationId }, + { $set: { conversationRef: String(renewed._id) } }, + ); + return result; + })) as unknown as typeof Conversation.deleteMany); + try { + await deleteConvos('user123', { conversationId }); + } finally { + spy.mockRestore(); + } + expect(await PRAutomation.countDocuments({ user: 'user123', conversationId })).toBe(1); + }); + describe('when the conversation delete fails', () => { const user = 'user123'; const setup = async () => { diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index af1a55e2785..fe10bda6307 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -4027,7 +4027,7 @@ export function createConversationMethods( const userFilter = { ...filter, user }; type DeletionConversation = Pick< IConversation, - 'conversationId' | 'tenantId' | 'chatProjectId' | 'tags' + '_id' | 'conversationId' | 'tenantId' | 'chatProjectId' | 'tags' >; const retryCascadeOperation = async (operation: () => PromiseLike | T): Promise => { let lastError: unknown; @@ -4146,9 +4146,11 @@ export function createConversationMethods( /** Only after the delete committed, so a failed delete keeps the user's automation. * A claim refuses any record whose conversation is gone and removes it, so a cleanup * that still fails after its retry converges on the next claim. */ - await retryCascadeOperation(() => prAutomation.deletePRAutomations(user, waveIds)).catch( - (error) => - logger.warn('[deleteConvos] PR automation cleanup deferred to its next claim', error), + const waveRefs = wave.map((conversation) => String(conversation._id)); + await retryCascadeOperation(() => + prAutomation.deletePRAutomations(user, waveIds, waveRefs), + ).catch((error) => + logger.warn('[deleteConvos] PR automation cleanup deferred to its next claim', error), ); if (result.deletedCount > 0) { /** Result erasure is irreversible. Keep receipts intact when a diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 6cbd620a2c2..0ae19a8012a 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -474,6 +474,38 @@ describe('a deletion that lands while a claim is in flight', () => { }); }); +describe('a conversation deleted and saved again under the same id', () => { + const recreate = async () => { + await mongoose.models.Conversation.deleteMany({}); + await seedConversation(); + }; + + test('refuses a claim for the record of the earlier conversation', async () => { + await enable(pullOne); + await recreate(); + expect(await claim(1)).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + + test('starts a new run for the new conversation when enabled', async () => { + await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); + await addBot(7); + await recreate(); + const record = await enableWith({ ...key, binding: otherRepository }); + const conversation = await mongoose.models.Conversation.findOne({ + conversationId: key.conversationId, + }).lean<{ _id: unknown }>(); + expect(record).toMatchObject({ + ...otherRepository, + trust: 'approvedBots', + trustedBots: [], + round: 0, + conversationRef: String(conversation?._id), + }); + expect((await claim(1, { binding: otherRepository })).ok).toBe(true); + }); +}); + describe('enabling under a lowered trust ceiling', () => { test('restarts a stopped record at the current ceiling when no trust is requested', async () => { await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); @@ -1261,8 +1293,10 @@ describe('claiming for a conversation that no longer exists', () => { test('accepts a claim while the conversation retention date is still ahead', async () => { await enable(); - await mongoose.models.Conversation.deleteMany({}); - await seedConversation({ expiredAt: new Date(Date.now() + 3_600_000) }); + await mongoose.models.Conversation.updateOne( + { conversationId: key.conversationId }, + { $set: { expiredAt: new Date(Date.now() + 3_600_000) } }, + ); expect((await claim(1)).ok).toBe(true); }); @@ -1385,6 +1419,14 @@ describe('reaping records the retention index left behind', () => { expect(peak).toBeLessThanOrEqual(8); }); + test('removes a record of a conversation deleted and saved again under the same id', async () => { + await enable(); + await mongoose.models.Conversation.deleteMany({}); + await seedConversation(); + expect(await methods.reapPRAutomations()).toEqual({ scanned: 1, removed: 1 }); + expect(await methods.getPRAutomation(key)).toBeNull(); + }); + test('pages through every record', async () => { const ids = Array.from({ length: 5 }, (_, index) => `orphan-${index}`); await PRAutomation.insertMany(ids.map((conversationId) => ({ user: userId, conversationId }))); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 8996649f572..f43469261da 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -29,8 +29,13 @@ const REAP_CONCURRENCY = 8; */ type Liveness = 'live' | 'conversation_gone' | 'owner_gone' | 'owner_inactive'; type Gone = Extract; -/** The tenant a record belongs to; `null` or absent is a record stored without one. */ -type LivenessScope = { tenantId?: string | null }; +/** + * What a liveness check is judged against. `tenantId`, when the key is present, is the record's + * tenant (`null` for one stored without one). `conversationRef` is the `_id` of the conversation + * document the record was enabled for: a conversation deleted and saved again under the same id + * is a different document, so a record of the old one does not belong to it. + */ +type LivenessScope = { tenantId?: string | null; conversationRef?: string }; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; @@ -48,6 +53,7 @@ const BUMP = { $inc: { version: 1 } }; /** GitHub compares `owner/name` without case, so records store and match the lower case form. */ const canonicalRepository = (repository: string) => repository.toLowerCase(); const LINK_FIELDS = [ + '_id', 'conversationId', 'user', 'tenantId', @@ -69,7 +75,7 @@ function lineagePipeline( { userId, conversationId }: t.PRAutomationKey, scope?: LivenessScope, ): PipelineStage[] { - const tenant = scope == null ? {} : { tenantId: scope.tenantId ?? null }; + const tenant = scope != null && 'tenantId' in scope ? { tenantId: scope.tenantId ?? null } : {}; const stages: PipelineStage[] = [ { $match: { user: userId, conversationId, ...tenant } }, { $limit: 1 }, @@ -125,7 +131,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): fence: t.PRAutomationRunFence, ): Promise; }; - deletePRAutomations: (userId: string, conversationIds?: string[]) => Promise; + deletePRAutomations: ( + userId: string, + conversationIds?: string[], + conversationRefs?: string[], + ) => Promise; reapPRAutomations: (params?: t.ReapPRAutomationsParams) => Promise; setPRAutomationTrust: ( key: t.PRAutomationKey, @@ -180,11 +190,21 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * instead. */ async function checkLiveness(key: t.PRAutomationKey, scope?: LivenessScope): Promise { + return (await inspectLiveness(key, scope)).liveness; + } + + /** `checkLiveness`, also naming the conversation document a live check found. */ + async function inspectLiveness( + key: t.PRAutomationKey, + scope?: LivenessScope, + ): Promise<{ liveness: Liveness; conversationRef?: string }> { + const judged = (liveness: Liveness) => ({ liveness }); if (!isValidObjectIdString(key.userId)) { - return 'owner_inactive'; + return judged('owner_inactive'); } const Conversation = mongoose.models.Conversation; type Link = { + _id?: unknown; conversationId: string; user: string; tenantId?: string | null; @@ -198,10 +218,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): Conversation.aggregate>(lineagePipeline(key, scope)), ]); if (owner == null) { - return 'owner_gone'; + return judged('owner_gone'); } if (owner.agentTriggerDeletionStartedAt != null) { - return 'owner_inactive'; + return judged('owner_inactive'); } /** * Links of other users or other tenants never count: a lookup by `conversationId` is scoped @@ -222,20 +242,24 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } const conversation = links.get(key.conversationId); if (conversation == null) { - return 'conversation_gone'; + return judged('conversation_gone'); } let current: Link | undefined = conversation; for (let hops = 0; current?.subagentThread?.parentConversationId != null; hops++) { if (hops >= MAX_SUBAGENT_DEPTH) { - return 'conversation_gone'; + return judged('conversation_gone'); } current = links.get(current.subagentThread.parentConversationId); } const root = conversation.subagentThread?.rootConversationId; if (current == null || (root != null && current.conversationId !== root)) { - return 'conversation_gone'; + return judged('conversation_gone'); } - return 'live'; + const conversationRef = String(conversation._id); + if (scope?.conversationRef != null && scope.conversationRef !== conversationRef) { + return judged('conversation_gone'); + } + return { liveness: 'live', conversationRef }; } /** A missing owner reads as inactive to callers; the distinction only decides removal. */ @@ -327,7 +351,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ? undefined : { ...requested, repository: canonicalRepository(requested.repository) }; const trust = requestedTrust == null ? undefined : boundTrust(requestedTrust, maxTrust); - const before = await checkLiveness(key); + const { liveness: before, conversationRef } = await inspectLiveness(key); if (before !== 'live') { return { ok: false, error: { code: isGone(before) ? goneCode(before) : before } }; } @@ -348,44 +372,63 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const observed = await PRAutomation.findOne(filter).lean(); let written: number; let restartedRun = true; + const fresh = (version: number) => ({ + ...filter, + state: 'idle', + round: 0, + trustedBots: [], + claimedHeads: [], + epoch: attempt, + pending: attempt, + version, + conversationRef, + trust: clampPRAutomationTrust(trust, maxTrust), + ...binding, + }); + /** A record of an earlier conversation saved under the same id starts over for this one. */ + const replaced = + observed?.conversationRef != null && observed.conversationRef !== conversationRef; if (observed == null) { written = 1; try { - await PRAutomation.create({ - ...filter, - state: 'idle', - round: 0, - trustedBots: [], - claimedHeads: [], - epoch: attempt, - pending: attempt, - version: written, - trust: clampPRAutomationTrust(trust, maxTrust), - ...binding, - }); + await PRAutomation.create(fresh(written)); } catch (error) { if ((error as { code?: number }).code === DUPLICATE_KEY) { return { ok: false, error: { code: 'conflict' } }; } throw error; } + } else if (replaced) { + written = observed.version + 1; + const applied = await PRAutomation.replaceOne( + { ...filter, version: observed.version }, + { + ...fresh(written), + ...(observed.tenantId != null && { tenantId: observed.tenantId }), + }, + { runValidators: true }, + ); + if (applied.matchedCount === 0) { + return { ok: false, error: { code: 'conflict' } }; + } } else { written = observed.version + 1; const restart = enableRestart(observed, attempt, binding, trust ?? observed.trust, maxTrust); restartedRun = restart != null; /** An enable also narrows a stored level the current ceiling no longer allows. */ const narrowed = clampPRAutomationTrust(observed.trust, maxTrust); - const update = restart ?? (narrowed === observed.trust ? {} : { $set: { trust: narrowed } }); + const update: { $set?: Record; $unset?: Record } = + restart ?? (narrowed === observed.trust ? {} : { $set: { trust: narrowed } }); const applied = await PRAutomation.updateOne( { ...filter, version: observed.version }, - { ...update, ...BUMP }, + { ...update, $set: { ...update.$set, conversationRef }, ...BUMP }, { runValidators: true }, ); if (applied.matchedCount === 0) { return { ok: false, error: { code: 'conflict' } }; } } - const after = await checkLiveness(key); + const after = await checkLiveness(key, { conversationRef }); if (isGone(after)) { await PRAutomation.deleteOne({ ...filter, version: written }); return { ok: false, error: { code: goneCode(after) } }; @@ -512,9 +555,9 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * `deleteConvos`, so liveness is checked here rather than trusted to a cascade. */ const seen = await PRAutomation.findOne(keyFilter(key)) - .select({ _id: 0, version: 1 }) - .lean<{ version: number }>(); - const before = await checkLiveness(key); + .select({ _id: 0, version: 1, conversationRef: 1 }) + .lean<{ version: number; conversationRef?: string }>(); + const before = await checkLiveness(key, { conversationRef: seen?.conversationRef }); if (before !== 'live') { return refuseClaim(key, before, seen?.version); } @@ -554,7 +597,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): */ let after: Liveness; try { - after = await checkLiveness(key); + after = await checkLiveness(key, { conversationRef: claimed.conversationRef }); } catch (error) { /** A check that fails leaves no answer to act on, so the round is handed back. */ await releaseClaim(key, claimed, runId).catch(() => undefined); @@ -699,9 +742,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * Removes every record of a user, or of the listed conversations, in bounded batches. * Cleanup only: it creates nothing, so a deployment that never used the feature pays one - * indexed delete that matches no document. + * indexed delete that matches no document. `conversationRefs`, aligned with + * `conversationIds`, names the conversation documents that were deleted, so a record enabled + * for a conversation saved again under the same id meanwhile is kept. */ - async function deletePRAutomations(userId: string, conversationIds?: string[]): Promise { + async function deletePRAutomations( + userId: string, + conversationIds?: string[], + conversationRefs?: string[], + ): Promise { const PRAutomation = mongoose.models.PRAutomation; if (PRAutomation == null || conversationIds?.length === 0) { return; @@ -711,9 +760,13 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return; } for (let start = 0; start < conversationIds.length; start += DELETE_BATCH) { + const refs = conversationRefs?.slice(start, start + DELETE_BATCH); await PRAutomation.deleteMany({ user: userId, conversationId: { $in: conversationIds.slice(start, start + DELETE_BATCH) }, + ...(refs != null && { + $or: [{ conversationRef: { $in: refs } }, { conversationRef: { $exists: false } }], + }), }); } } @@ -732,19 +785,21 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const page = await PRAutomation.find(after == null ? {} : { _id: { $gt: after } }) .sort({ _id: 1 }) .limit(limit) - .select({ _id: 1, user: 1, tenantId: 1, conversationId: 1, version: 1 }) + .select({ _id: 1, user: 1, tenantId: 1, conversationId: 1, conversationRef: 1, version: 1 }) .lean< Array<{ _id: Types.ObjectId; user: string; tenantId?: string; conversationId: string; + conversationRef?: string; version: number; }> >(); const reap = async (record: (typeof page)[number]): Promise => { const key = { userId: String(record.user), conversationId: record.conversationId }; - if (!isGone(await checkLiveness(key, { tenantId: record.tenantId }))) { + const scope = { tenantId: record.tenantId, conversationRef: record.conversationRef }; + if (!isGone(await checkLiveness(key, scope))) { return 0; } /** A record written after the check, a renewed enable included, has a new version. */ diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index 8724debc2fd..ecd8b24ded0 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -36,6 +36,8 @@ const prAutomationSchema: Schema = new Schema( }, /** Unbounded, like the conversation schema it mirrors, so every stored conversation can enable. */ conversationId: { type: String, required: true }, + /** The `_id` of the conversation document the record was enabled for. */ + conversationRef: { type: String, maxlength: 64 }, repository: { type: String, maxlength: 256 }, pullNumber: { type: Number, diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 705c1729afb..8a214b8c739 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -14,6 +14,11 @@ export interface IPRAutomation { user: string; tenantId?: string; conversationId: string; + /** + * The `_id` of the conversation document the record was enabled for. A conversation deleted + * and saved again under the same id is a different document, and the record does not carry over. + */ + conversationRef?: string; /** `owner/name` of the repository the pull request belongs to. */ repository?: string; pullNumber?: number; From 272686249b2bec409bcb1d7d2efa1a58bfec6fbd Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:33:25 +0200 Subject: [PATCH 23/25] fix: Fence PR Automation Deletes on the Record as Well as Its Version A record deleted and enabled again starts its version count over, so a claim refusal or an enable cleanup that named only the version could remove the new record. Those deletes, the enable commit and its rollback now name the document _id they saw along with the version. --- .../src/methods/prAutomation.spec.ts | 25 ++++++++++ .../data-schemas/src/methods/prAutomation.ts | 46 +++++++++++-------- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 0ae19a8012a..f9917704c7e 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -487,6 +487,31 @@ describe('a conversation deleted and saved again under the same id', () => { expect(await methods.getPRAutomation(key)).toBeNull(); }); + test('keeps the new record when a claim for the old one refuses after a re-enable', async () => { + await enable(pullOne); + const before = await methods.getPRAutomation(key); + const spy = interleave( + async () => { + await recreate(); + await PRAutomation.deleteMany({}); + await enable(pullOne); + }, + PRAutomation, + 'findOne', + ); + try { + expect(await claim(1, { epoch: before?.epoch })).toEqual({ + ok: false, + error: { code: 'not_found' }, + }); + } finally { + spy.mockRestore(); + } + const renewed = await methods.getPRAutomation(key); + expect(renewed?.version).toBe(before?.version); + expect(renewed?.epoch).not.toBe(before?.epoch); + }); + test('starts a new run for the new conversation when enabled', async () => { await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); await addBot(7); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index f43469261da..59ae00eab01 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -50,6 +50,11 @@ const runReset = (epoch: string, fields: Record = {}) => ({ * included, makes it miss instead of being overwritten. */ const BUMP = { $inc: { version: 1 } }; +/** + * One record as a write saw it. The version alone is not an identity: a record deleted and + * enabled again starts its count over, so a fence names the document `_id` as well. + */ +type RecordFence = { _id: unknown; version: number }; /** GitHub compares `owner/name` without case, so records store and match the lower case form. */ const canonicalRepository = (repository: string) => repository.toLowerCase(); const LINK_FIELDS = [ @@ -269,21 +274,25 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): /** * A record whose conversation or owner is gone can never run again, so it is removed. This * also converges a cleanup that failed after the delete committed. The delete names the - * version read before the liveness check, so a record renewed after the check is kept; without + * record and version read before the liveness check, so a record renewed after the check is kept; without * one nothing is removed and the reaper decides later. */ async function refuseClaim( key: t.PRAutomationKey, liveness: Exclude, - version: number | undefined, + fence: RecordFence | null | undefined, ): Promise { if (!isGone(liveness)) { return { ok: false, error: { code: liveness } }; } const removed = - version == null + fence == null ? { deletedCount: 0 } - : await mongoose.models.PRAutomation.deleteOne({ ...keyFilter(key), version }); + : await mongoose.models.PRAutomation.deleteOne({ + ...keyFilter(key), + _id: fence._id, + version: fence.version, + }); return { ok: false, error: { code: removed.deletedCount > 0 ? goneCode(liveness) : 'not_found' }, @@ -371,6 +380,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const attempt = randomUUID(); const observed = await PRAutomation.findOne(filter).lean(); let written: number; + let recordId: unknown = observed?._id; let restartedRun = true; const fresh = (version: number) => ({ ...filter, @@ -391,7 +401,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (observed == null) { written = 1; try { - await PRAutomation.create(fresh(written)); + recordId = (await PRAutomation.create(fresh(written)))._id; } catch (error) { if ((error as { code?: number }).code === DUPLICATE_KEY) { return { ok: false, error: { code: 'conflict' } }; @@ -430,18 +440,18 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } const after = await checkLiveness(key, { conversationRef }); if (isGone(after)) { - await PRAutomation.deleteOne({ ...filter, version: written }); + await PRAutomation.deleteOne({ ...filter, _id: recordId, version: written }); return { ok: false, error: { code: goneCode(after) } }; } if (after !== 'live') { if (restartedRun) { - await undoEnable(filter, written, observed); + await undoEnable(filter, { _id: recordId, version: written }, observed); } return { ok: false, error: { code: after } }; } if (restartedRun) { const committed = await PRAutomation.updateOne( - { ...filter, version: written, pending: attempt }, + { ...filter, _id: recordId, version: written, pending: attempt }, { $unset: { pending: '' }, ...BUMP }, ); if (committed.matchedCount === 0) { @@ -494,16 +504,16 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): */ async function undoEnable( key: ReturnType, - written: number, + written: RecordFence, previous: (t.IPRAutomation & { _id: unknown }) | null, ): Promise { const PRAutomation = mongoose.models.PRAutomation; - const guard = { ...key, version: written }; + const guard = { ...key, _id: written._id, version: written.version }; if (previous == null) { await PRAutomation.deleteOne(guard); return; } - await PRAutomation.replaceOne(guard, { ...previous, version: written + 1 }); + await PRAutomation.replaceOne(guard, { ...previous, version: written.version + 1 }); } async function disablePRAutomation(key: t.PRAutomationKey): Promise<{ removed: boolean }> { @@ -555,11 +565,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * `deleteConvos`, so liveness is checked here rather than trusted to a cascade. */ const seen = await PRAutomation.findOne(keyFilter(key)) - .select({ _id: 0, version: 1, conversationRef: 1 }) - .lean<{ version: number; conversationRef?: string }>(); + .select({ _id: 1, version: 1, conversationRef: 1 }) + .lean(); const before = await checkLiveness(key, { conversationRef: seen?.conversationRef }); if (before !== 'live') { - return refuseClaim(key, before, seen?.version); + return refuseClaim(key, before, seen); } /** @@ -608,7 +618,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } const released = await releaseClaim(key, claimed, runId); /** Only a claim that undid itself knows the version it left; otherwise the reaper decides. */ - return refuseClaim(key, after, released?.version); + return refuseClaim(key, after, released); } const current = await getPRAutomation(key); @@ -652,7 +662,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): key: t.PRAutomationKey, claimed: t.IPRAutomation, runId: string, - ): Promise<{ version: number } | null> { + ): Promise { const previousHeads = claimed.claimedHeads.slice(0, -1); const lastHeadSha = previousHeads[previousHeads.length - 1]; return mongoose.models.PRAutomation.findOneAndUpdate( @@ -670,8 +680,8 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ...(lastHeadSha == null && { startedAt: '', lastHeadSha: '' }), }, }, - { new: true, projection: { _id: 0, version: 1 } }, - ).lean<{ version: number }>(); + { new: true, projection: { _id: 1, version: 1 } }, + ).lean(); } /** From d680b1490a39fdc2c28c1faa3e3f8dd36fcc68d3 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:44:57 +0200 Subject: [PATCH 24/25] fix: Fence PR Automation Enable Writes and Recovery Cleanup on the Record An enable fenced its write on the version it read but not on the record, so a record disabled and enabled again to the same version was overwritten and left pending. Its write now names the record it read. The deletion retry that cleans up after a root a previous attempt already removed deleted by conversation id alone; it now reaps those records through the liveness check, so a record of a conversation saved again under the same id is kept. --- .../src/methods/conversation.spec.ts | 8 +- .../data-schemas/src/methods/conversation.ts | 3 +- .../src/methods/prAutomation.spec.ts | 37 ++++++ .../data-schemas/src/methods/prAutomation.ts | 111 ++++++++++++------ 4 files changed, 117 insertions(+), 42 deletions(-) diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 4e3bff36d1c..04175ea32ac 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -3850,12 +3850,14 @@ describe('Conversation Operations', () => { }); it('removes the PR automation record of a root that a previous attempt already deleted', async () => { + /** The reaper judges the owner too, so the record names a real account id. */ + const owner = new mongoose.Types.ObjectId().toString(); const rootId = uuidv4(); const childId = uuidv4(); const PRAutomation = mongoose.models.PRAutomation; await Conversation.create({ conversationId: childId, - user: 'user123', + user: owner, endpoint: EModelEndpoint.agents, subagentThread: { rootConversationId: rootId, @@ -3867,9 +3869,9 @@ describe('Conversation Operations', () => { depth: 1, }, }); - await PRAutomation.create({ user: 'user123', conversationId: rootId }); + await PRAutomation.create({ user: owner, conversationId: rootId }); - await deleteConvos('user123', { conversationId: rootId }, { allowEmpty: true }); + await deleteConvos(owner, { conversationId: rootId }, { allowEmpty: true }); expect(await PRAutomation.countDocuments({ conversationId: rootId })).toBe(0); }); diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index fe10bda6307..809a483af04 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -4185,8 +4185,9 @@ export function createConversationMethods( ]; if (recoveryConversationIds.length > 0) { + /** The deleted documents are unknown here, so only records whose conversation is gone go. */ await retryCascadeOperation(() => - prAutomation.deletePRAutomations(user, recoveryConversationIds), + prAutomation.reapPRAutomationsOf(user, recoveryConversationIds), ).catch((error) => logger.warn('[deleteConvos] PR automation cleanup deferred to its next claim', error), ); diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index f9917704c7e..418c201f888 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -512,6 +512,17 @@ describe('a conversation deleted and saved again under the same id', () => { expect(renewed?.epoch).not.toBe(before?.epoch); }); + test('reaps only the listed records whose conversation is gone', async () => { + await enable(pullOne); + const orphan = { userId, conversationId: 'orphan-1' }; + await PRAutomation.create({ user: userId, conversationId: orphan.conversationId }); + expect( + await methods.reapPRAutomationsOf(userId, [key.conversationId, orphan.conversationId]), + ).toBe(1); + expect(await methods.getPRAutomation(key)).not.toBeNull(); + expect(await methods.getPRAutomation(orphan)).toBeNull(); + }); + test('starts a new run for the new conversation when enabled', async () => { await enableWith({ ...key, binding: pullOne, trust: 'anyone' }); await addBot(7); @@ -736,6 +747,32 @@ describe('enabling for a conversation or owner that no longer exists', () => { ); }); + test('does not write over a record disabled and enabled again after its read', async () => { + await enable(pullOne); + const before = await methods.getPRAutomation(key); + const spy = interleave( + async () => { + await methods.disablePRAutomation(key); + await enable(pullOne); + }, + PRAutomation, + 'findOne', + ); + try { + expect(await enableOpen({ ...key, binding: pullTwo })).toEqual({ + ok: false, + error: { code: 'conflict' }, + }); + } finally { + spy.mockRestore(); + } + const renewed = await methods.getPRAutomation(key); + expect(renewed?.version).toBe(before?.version); + expect(renewed).toMatchObject(pullOne); + expect(renewed?.pending).toBeUndefined(); + expect((await claim(1)).ok).toBe(true); + }); + test('keeps a user stop that lands while a rejected enable rolls back', async () => { await enable(pullOne); const result = await enableCrossedBy(() => methods.stopPRAutomation(key, 'user_stopped')); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 59ae00eab01..2f5873f9519 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -9,7 +9,7 @@ import { import type { PRAutomationTrustLevel } from 'librechat-data-provider'; import type { PRAutomationStopCode } from 'librechat-data-provider'; import type { PRAutomationState } from 'librechat-data-provider'; -import type { PipelineStage, Types } from 'mongoose'; +import type { PipelineStage } from 'mongoose'; import type * as t from '~/types/prAutomation'; import { isValidObjectIdString } from '~/utils/objectId'; import { createIndexesWithRetry } from '~/utils/retry'; @@ -55,6 +55,20 @@ const BUMP = { $inc: { version: 1 } }; * enabled again starts its count over, so a fence names the document `_id` as well. */ type RecordFence = { _id: unknown; version: number }; +const REAP_FIELDS = { + _id: 1, + user: 1, + tenantId: 1, + conversationId: 1, + conversationRef: 1, + version: 1, +}; +type ReapCandidate = RecordFence & { + user: string; + tenantId?: string; + conversationId: string; + conversationRef?: string; +}; /** GitHub compares `owner/name` without case, so records store and match the lower case form. */ const canonicalRepository = (repository: string) => repository.toLowerCase(); const LINK_FIELDS = [ @@ -142,6 +156,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): conversationRefs?: string[], ) => Promise; reapPRAutomations: (params?: t.ReapPRAutomationsParams) => Promise; + reapPRAutomationsOf: (userId: string, conversationIds: string[]) => Promise; setPRAutomationTrust: ( key: t.PRAutomationKey, trust: PRAutomationTrustLevel, @@ -411,7 +426,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): } else if (replaced) { written = observed.version + 1; const applied = await PRAutomation.replaceOne( - { ...filter, version: observed.version }, + { ...filter, _id: observed._id, version: observed.version }, { ...fresh(written), ...(observed.tenantId != null && { tenantId: observed.tenantId }), @@ -430,7 +445,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): const update: { $set?: Record; $unset?: Record } = restart ?? (narrowed === observed.trust ? {} : { $set: { trust: narrowed } }); const applied = await PRAutomation.updateOne( - { ...filter, version: observed.version }, + { ...filter, _id: observed._id, version: observed.version }, { ...update, $set: { ...update.$set, conversationRef }, ...BUMP }, { runValidators: true }, ); @@ -791,43 +806,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): after, limit = 100, }: t.ReapPRAutomationsParams = {}): Promise { - const PRAutomation = mongoose.models.PRAutomation; - const page = await PRAutomation.find(after == null ? {} : { _id: { $gt: after } }) + const page = await mongoose.models.PRAutomation.find( + after == null ? {} : { _id: { $gt: after } }, + ) .sort({ _id: 1 }) .limit(limit) - .select({ _id: 1, user: 1, tenantId: 1, conversationId: 1, conversationRef: 1, version: 1 }) - .lean< - Array<{ - _id: Types.ObjectId; - user: string; - tenantId?: string; - conversationId: string; - conversationRef?: string; - version: number; - }> - >(); - const reap = async (record: (typeof page)[number]): Promise => { - const key = { userId: String(record.user), conversationId: record.conversationId }; - const scope = { tenantId: record.tenantId, conversationRef: record.conversationRef }; - if (!isGone(await checkLiveness(key, scope))) { - return 0; - } - /** A record written after the check, a renewed enable included, has a new version. */ - const result = await PRAutomation.deleteOne({ - _id: record._id, - ...keyFilter(key), - version: record.version, - }); - return result.deletedCount; - }; - let removed = 0; - /** A few checks at a time, so a large page neither runs serially nor floods the database. */ - for (let start = 0; start < page.length; start += REAP_CONCURRENCY) { - const counts = await Promise.all(page.slice(start, start + REAP_CONCURRENCY).map(reap)); - for (const count of counts) { - removed += count; - } - } + .select(REAP_FIELDS) + .lean(); + const removed = await reapCandidates(page); const last = page.length === limit ? page[page.length - 1] : undefined; return { scanned: page.length, @@ -836,6 +822,54 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): }; } + /** + * Reaps the records of the listed conversations only, for a cleanup that knows the ids but + * not the conversation documents they belonged to: a record whose conversation was saved again + * under the same id belongs to the new one, is live, and is kept. + */ + async function reapPRAutomationsOf(userId: string, conversationIds: string[]): Promise { + let removed = 0; + for (let start = 0; start < conversationIds.length; start += DELETE_BATCH) { + const records = await mongoose.models.PRAutomation.find({ + user: userId, + conversationId: { $in: conversationIds.slice(start, start + DELETE_BATCH) }, + }) + .select(REAP_FIELDS) + .lean(); + removed += await reapCandidates(records); + } + return removed; + } + + /** Checks a few records at a time, so a large batch neither runs serially nor floods the database. */ + async function reapCandidates(records: ReapCandidate[]): Promise { + let removed = 0; + for (let start = 0; start < records.length; start += REAP_CONCURRENCY) { + const counts = await Promise.all( + records.slice(start, start + REAP_CONCURRENCY).map(reapCandidate), + ); + for (const count of counts) { + removed += count; + } + } + return removed; + } + + async function reapCandidate(record: ReapCandidate): Promise { + const key = { userId: String(record.user), conversationId: record.conversationId }; + const scope = { tenantId: record.tenantId, conversationRef: record.conversationRef }; + if (!isGone(await checkLiveness(key, scope))) { + return 0; + } + /** A record written after the check, a renewed enable included, has a new version. */ + const result = await mongoose.models.PRAutomation.deleteOne({ + _id: record._id, + ...keyFilter(key), + version: record.version, + }); + return result.deletedCount; + } + /** `maxTrust` is the administrator ceiling; a wider request is stored as the ceiling. */ async function setPRAutomationTrust( key: t.PRAutomationKey, @@ -924,6 +958,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): stopPRAutomation, deletePRAutomations, reapPRAutomations, + reapPRAutomationsOf, setPRAutomationTrust, addPRAutomationBot, removePRAutomationBot, From 95a82fb00f3210020985e5eb8c0e6f6dd56c03f7 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:29:24 +0200 Subject: [PATCH 25/25] fix: Bind PR Automation Records to Their Ancestor Conversations A subagent thread's ancestors were matched by conversation id, so a root or parent deleted and saved again under the same id still controlled the old record. Records now store lineageRefs, the _id of each ancestor in walk order, and the liveness check treats a different document at any position as gone. An enable on a record whose ancestors changed starts over. --- .../src/methods/prAutomation.spec.ts | 59 +++++++++++++++++++ .../data-schemas/src/methods/prAutomation.ts | 54 +++++++++++++---- .../data-schemas/src/schema/prAutomation.ts | 2 + .../data-schemas/src/types/prAutomation.ts | 2 + 4 files changed, 104 insertions(+), 13 deletions(-) diff --git a/packages/data-schemas/src/methods/prAutomation.spec.ts b/packages/data-schemas/src/methods/prAutomation.spec.ts index 418c201f888..a8bc9445918 100644 --- a/packages/data-schemas/src/methods/prAutomation.spec.ts +++ b/packages/data-schemas/src/methods/prAutomation.spec.ts @@ -914,6 +914,65 @@ describe('claiming for a subagent thread whose root conversation is gone', () => headSha: head(1), }); + const recreateRoot = async () => { + await mongoose.models.Conversation.deleteOne({ conversationId: key.conversationId }); + await seedConversation(); + }; + + test('refuses and reaps a child whose root was deleted and saved again', async () => { + await seedChild(); + await enableOpen({ ...childKey, binding: pullOne }); + await recreateRoot(); + expect(await methods.reapPRAutomations()).toEqual({ scanned: 1, removed: 1 }); + await enableOpen({ ...childKey, binding: pullOne }); + await recreateRoot(); + expect(await claimChild()).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + }); + + test('refuses a grandchild whose middle ancestor was deleted and saved again', async () => { + const leafKey = { userId, conversationId: 'leaf-1' }; + const thread = (parent: string) => ({ + rootConversationId: key.conversationId, + parentConversationId: parent, + parentMessageId: 'message-1', + parentToolCallId: 'call-1', + subagentType: 'agent-child', + subagentKind: 'agent', + depth: 1, + }); + await seedChild(); + await mongoose.models.Conversation.create({ + conversationId: leafKey.conversationId, + user: userId, + endpoint: 'agents', + subagentThread: { ...thread(childKey.conversationId), depth: 2 }, + }); + await enableOpen({ ...leafKey, binding: pullOne }); + await mongoose.models.Conversation.deleteOne({ conversationId: childKey.conversationId }); + await seedChild(); + expect( + await methods.claimPRAutomationRound({ + ...leafKey, + ...limits, + binding: { ...pullOne, epoch: await currentEpoch(leafKey) }, + headSha: head(1), + }), + ).toEqual({ ok: false, error: { code: 'conversation_gone' } }); + }); + + test('starts a child over for the root saved again when it is enabled', async () => { + await seedChild(); + await enableOpen({ ...childKey, binding: pullOne, trust: 'anyone' }); + const before = await methods.getPRAutomation(childKey); + await recreateRoot(); + await enableOpen({ ...childKey, binding: pullOne }); + const after = await methods.getPRAutomation(childKey); + expect(after?.epoch).not.toBe(before?.epoch); + expect(after?.lineageRefs).not.toEqual(before?.lineageRefs); + expect(after).toMatchObject({ trust: 'approvedBots', round: 0 }); + expect((await claimChild()).ok).toBe(true); + }); + test('rejects a claim for a child once its root was deleted', async () => { await seedChild(); await enableOpen({ ...childKey, binding: pullOne }); diff --git a/packages/data-schemas/src/methods/prAutomation.ts b/packages/data-schemas/src/methods/prAutomation.ts index 2f5873f9519..1ab20edd520 100644 --- a/packages/data-schemas/src/methods/prAutomation.ts +++ b/packages/data-schemas/src/methods/prAutomation.ts @@ -33,9 +33,12 @@ type Gone = Extract; * What a liveness check is judged against. `tenantId`, when the key is present, is the record's * tenant (`null` for one stored without one). `conversationRef` is the `_id` of the conversation * document the record was enabled for: a conversation deleted and saved again under the same id - * is a different document, so a record of the old one does not belong to it. + * is a different document, so a record of the old one does not belong to it. `lineageRefs` + * does the same for a subagent thread's ancestors, parent first and root last. */ -type LivenessScope = { tenantId?: string | null; conversationRef?: string }; +type LivenessScope = { tenantId?: string | null; conversationRef?: string; lineageRefs?: string[] }; +/** The conversation documents a live check found: the record's own and its ancestors'. */ +type Lineage = { conversationRef: string; lineageRefs: string[] }; const CLAIMABLE_STATES: PRAutomationState[] = ['idle', 'waiting']; const SETTLEABLE_STATES: PRAutomationState[] = ['fixing', 'needs_user']; const SETTLED_STATES: PRAutomationState[] = ['waiting', 'needs_user']; @@ -61,6 +64,7 @@ const REAP_FIELDS = { tenantId: 1, conversationId: 1, conversationRef: 1, + lineageRefs: 1, version: 1, }; type ReapCandidate = RecordFence & { @@ -68,6 +72,7 @@ type ReapCandidate = RecordFence & { tenantId?: string; conversationId: string; conversationRef?: string; + lineageRefs?: string[]; }; /** GitHub compares `owner/name` without case, so records store and match the lower case form. */ const canonicalRepository = (repository: string) => repository.toLowerCase(); @@ -129,6 +134,9 @@ function lineagePipeline( return stages; } +const sameRefs = (a: string[], b: string[]) => + a.length === b.length && a.every((ref, index) => ref === b[index]); + const isGone = (liveness: Liveness): liveness is Gone => liveness === 'conversation_gone' || liveness === 'owner_gone'; @@ -217,7 +225,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): async function inspectLiveness( key: t.PRAutomationKey, scope?: LivenessScope, - ): Promise<{ liveness: Liveness; conversationRef?: string }> { + ): Promise<{ liveness: Liveness } & Partial> { const judged = (liveness: Liveness) => ({ liveness }); if (!isValidObjectIdString(key.userId)) { return judged('owner_inactive'); @@ -265,11 +273,15 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): return judged('conversation_gone'); } let current: Link | undefined = conversation; + const lineageRefs: string[] = []; for (let hops = 0; current?.subagentThread?.parentConversationId != null; hops++) { if (hops >= MAX_SUBAGENT_DEPTH) { return judged('conversation_gone'); } current = links.get(current.subagentThread.parentConversationId); + if (current != null) { + lineageRefs.push(String(current._id)); + } } const root = conversation.subagentThread?.rootConversationId; if (current == null || (root != null && current.conversationId !== root)) { @@ -279,7 +291,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): if (scope?.conversationRef != null && scope.conversationRef !== conversationRef) { return judged('conversation_gone'); } - return { liveness: 'live', conversationRef }; + /** An ancestor deleted and saved again under the same id no longer controls the record. */ + if (scope?.lineageRefs != null && !sameRefs(scope.lineageRefs, lineageRefs)) { + return judged('conversation_gone'); + } + return { liveness: 'live', conversationRef, lineageRefs }; } /** A missing owner reads as inactive to callers; the distinction only decides removal. */ @@ -375,7 +391,7 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): ? undefined : { ...requested, repository: canonicalRepository(requested.repository) }; const trust = requestedTrust == null ? undefined : boundTrust(requestedTrust, maxTrust); - const { liveness: before, conversationRef } = await inspectLiveness(key); + const { liveness: before, conversationRef, lineageRefs } = await inspectLiveness(key); if (before !== 'live') { return { ok: false, error: { code: isGone(before) ? goneCode(before) : before } }; } @@ -407,12 +423,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): pending: attempt, version, conversationRef, + lineageRefs, trust: clampPRAutomationTrust(trust, maxTrust), ...binding, }); /** A record of an earlier conversation saved under the same id starts over for this one. */ const replaced = - observed?.conversationRef != null && observed.conversationRef !== conversationRef; + (observed?.conversationRef != null && observed.conversationRef !== conversationRef) || + (observed?.lineageRefs != null && !sameRefs(observed.lineageRefs, lineageRefs ?? [])); if (observed == null) { written = 1; try { @@ -446,14 +464,14 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): restart ?? (narrowed === observed.trust ? {} : { $set: { trust: narrowed } }); const applied = await PRAutomation.updateOne( { ...filter, _id: observed._id, version: observed.version }, - { ...update, $set: { ...update.$set, conversationRef }, ...BUMP }, + { ...update, $set: { ...update.$set, conversationRef, lineageRefs }, ...BUMP }, { runValidators: true }, ); if (applied.matchedCount === 0) { return { ok: false, error: { code: 'conflict' } }; } } - const after = await checkLiveness(key, { conversationRef }); + const after = await checkLiveness(key, { conversationRef, lineageRefs }); if (isGone(after)) { await PRAutomation.deleteOne({ ...filter, _id: recordId, version: written }); return { ok: false, error: { code: goneCode(after) } }; @@ -580,9 +598,12 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): * `deleteConvos`, so liveness is checked here rather than trusted to a cascade. */ const seen = await PRAutomation.findOne(keyFilter(key)) - .select({ _id: 1, version: 1, conversationRef: 1 }) - .lean(); - const before = await checkLiveness(key, { conversationRef: seen?.conversationRef }); + .select({ _id: 1, version: 1, conversationRef: 1, lineageRefs: 1 }) + .lean(); + const before = await checkLiveness(key, { + conversationRef: seen?.conversationRef, + lineageRefs: seen?.lineageRefs, + }); if (before !== 'live') { return refuseClaim(key, before, seen); } @@ -622,7 +643,10 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): */ let after: Liveness; try { - after = await checkLiveness(key, { conversationRef: claimed.conversationRef }); + after = await checkLiveness(key, { + conversationRef: claimed.conversationRef, + lineageRefs: claimed.lineageRefs, + }); } catch (error) { /** A check that fails leaves no answer to act on, so the round is handed back. */ await releaseClaim(key, claimed, runId).catch(() => undefined); @@ -857,7 +881,11 @@ export function createPRAutomationMethods(mongoose: typeof import('mongoose')): async function reapCandidate(record: ReapCandidate): Promise { const key = { userId: String(record.user), conversationId: record.conversationId }; - const scope = { tenantId: record.tenantId, conversationRef: record.conversationRef }; + const scope = { + tenantId: record.tenantId, + conversationRef: record.conversationRef, + lineageRefs: record.lineageRefs, + }; if (!isGone(await checkLiveness(key, scope))) { return 0; } diff --git a/packages/data-schemas/src/schema/prAutomation.ts b/packages/data-schemas/src/schema/prAutomation.ts index ecd8b24ded0..cbf2c704bc6 100644 --- a/packages/data-schemas/src/schema/prAutomation.ts +++ b/packages/data-schemas/src/schema/prAutomation.ts @@ -38,6 +38,8 @@ const prAutomationSchema: Schema = new Schema( conversationId: { type: String, required: true }, /** The `_id` of the conversation document the record was enabled for. */ conversationRef: { type: String, maxlength: 64 }, + /** The `_id` of each ancestor of a subagent thread, parent first and root last. */ + lineageRefs: { type: [{ type: String, maxlength: 64 }], default: undefined }, repository: { type: String, maxlength: 256 }, pullNumber: { type: Number, diff --git a/packages/data-schemas/src/types/prAutomation.ts b/packages/data-schemas/src/types/prAutomation.ts index 8a214b8c739..2f24f86393e 100644 --- a/packages/data-schemas/src/types/prAutomation.ts +++ b/packages/data-schemas/src/types/prAutomation.ts @@ -19,6 +19,8 @@ export interface IPRAutomation { * and saved again under the same id is a different document, and the record does not carry over. */ conversationRef?: string; + /** The same for each ancestor of a subagent thread, parent first and root last. */ + lineageRefs?: string[]; /** `owner/name` of the repository the pull request belongs to. */ repository?: string; pullNumber?: number;