diff --git a/apps/backend/src/index.ts b/apps/backend/src/index.ts index da74ed5..0d09869 100644 --- a/apps/backend/src/index.ts +++ b/apps/backend/src/index.ts @@ -1,4 +1,5 @@ import express from 'express' +import type { NextFunction, Request, Response } from 'express' import cors from 'cors' import projectRouter from './modules/project'; import runRouter from './modules/run'; @@ -12,6 +13,7 @@ import { createBullBoard } from '@bull-board/api'; import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'; import { ExpressAdapter } from '@bull-board/express'; import { runQueue } from './modules/worker'; +import { logger } from './modules/utils'; const app = express(); app.use(cors()) @@ -33,6 +35,23 @@ createBullBoard({ }); app.use("/admin/queues", bullBoardAdapter.getRouter()); +// Express 5 forwards rejected async handlers here; without this they'd be +// answered by the default handler, which logs nothing and can leak stacks. +app.use((err: unknown, req: Request, res: Response, next: NextFunction) => { + logger.error(`Unhandled error on ${req.method} ${req.originalUrl}: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + if (res.headersSent) { + return next(err); + } + return res.status(500).json({ success: false, message: "Internal server error" }); +}); + +process.on("unhandledRejection", (reason) => { + logger.error(`Unhandled promise rejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}`); +}); +process.on("uncaughtException", (err) => { + logger.error(`Uncaught exception: ${err.stack ?? err.message}`); +}); + console.table(expressListEndpoints(questionRouter)); app.listen(3000, () =>{ console.log("Server is running on port 3000") diff --git a/apps/backend/src/modules/chat.ts b/apps/backend/src/modules/chat.ts index a2ae96b..71bea4f 100644 --- a/apps/backend/src/modules/chat.ts +++ b/apps/backend/src/modules/chat.ts @@ -26,6 +26,26 @@ GET /chat/:projectId/history → all past runs' events, for reload chatRouter.post('/', auth, createRun) chatRouter.post('/:projectId', auth, createRun) +// Event rows are written by the agent worker; a corrupt/truncated one must not +// take down the whole state response. +function parseEventContent(content: string | null, runId: string): unknown { + if(!content) return null + try{ + return JSON.parse(content) + } catch(e){ + logger.error(`Failed to parse stored event content for run ${runId}: ${e}`) + return null + } +} + +async function markRunFailed(runId: string){ + try{ + await prisma.run.update({where: {id: runId}, data: {status: 'FAILED', endedAt: new Date()}}) + } catch(e){ + logger.error(`Failed to mark run ${runId} as FAILED: ${e}`) + } +} + async function createRun(req: Request, res: Response){ const userId = req.headers.userid let projectId = req.params?.projectId @@ -78,6 +98,9 @@ async function createRun(req: Request, res: Response){ logger.info(`Added to run queue`) } catch(e){ logger.error(`Failed to enqueue run ${run.id}: ${e}`) + // The Run row already exists — leaving it IN_PROGRESS would have the + // frontend waiting on a stream for a job that was never queued. + await markRunFailed(run.id) return res.status(500).json({success: false, message: `Failed to start run`}) } @@ -108,7 +131,7 @@ chatRouter.get('/:runId/state', auth, async (req: Request, res: Response) => { where: { runId, type: run.status === 'CLARIFICATION_NEEDED' ? 'clarification_needed' : 'select_design' }, orderBy: { createdAt: 'desc' }, }) - pauseEvent = event?.content ? JSON.parse(event.content) : null + pauseEvent = parseEventContent(event?.content ?? null, runId) } let completedEvent: unknown = null @@ -117,7 +140,7 @@ chatRouter.get('/:runId/state', auth, async (req: Request, res: Response) => { where: { runId, type: 'run_completed' }, orderBy: { createdAt: 'desc' }, }) - completedEvent = event?.content ? JSON.parse(event.content) : null + completedEvent = parseEventContent(event?.content ?? null, runId) } let failedEvent: unknown = null @@ -126,7 +149,7 @@ chatRouter.get('/:runId/state', auth, async (req: Request, res: Response) => { where: { runId, type: 'run_failed' }, orderBy: { createdAt: 'desc' }, }) - failedEvent = event?.content ? JSON.parse(event.content) : null + failedEvent = parseEventContent(event?.content ?? null, runId) } return res.status(200).json({ @@ -185,6 +208,9 @@ chatRouter.post('/:projectId/:runId/continue', auth, async (req: Request, res: R logger.info(`Continuing run ${run.id}`) } catch(e){ logger.error(`Failed to re-enqueue run ${run.id}: ${e}`) + // Status was flipped to IN_PROGRESS just above, so it has to be undone + // or the run is stuck awaiting a job that doesn't exist. + await markRunFailed(run.id) return res.status(500).json({success: false, message: `Failed to continue run`}) } diff --git a/apps/backend/src/modules/design.ts b/apps/backend/src/modules/design.ts index 51e22cb..d0cc5ff 100644 --- a/apps/backend/src/modules/design.ts +++ b/apps/backend/src/modules/design.ts @@ -27,9 +27,15 @@ designRouter.post("/:projectId", internalAuth, async( req: Request, res: Respons message: "Invalid projectId", }); } + if (!Array.isArray(designs) || designs.some((d) => typeof d !== "string")) { + return res.status(400).json({ + success: false, + message: "designs must be an array of html strings", + }); + } let result try{ - console.log(`Saving designs to the db`) + logger.info(`Saving designs to the db`) result = await Promise.all(designs.map((design: string) => prisma.design.create({ data:{ @@ -66,12 +72,13 @@ designRouter.get("/:projectId/getDesigns", auth, async (req: Request, res: Respo projectId: projectId, }, }); - logger.info(`Designs are: ${designs}`) + logger.info(`Found ${designs.length} design(s) for project ${projectId}`) return res.status(200).json({ success: true, data: designs, }); } catch (e) { + logger.error(`Failed to list designs for project ${projectId}: ${e}`); return res.status(500).json({ success: false, message: "Internal server error", @@ -110,6 +117,7 @@ designRouter.get("/:projectId/selectedDesign", auth, async (req: Request, res: R data: design, }); } catch (e) { + logger.error(`Failed to fetch selected design for project ${projectId}: ${e}`); return res.status(500).json({ success: false, message: "Internal server error", @@ -170,6 +178,7 @@ designRouter.patch("/:projectId/designs/:designId", auth, async (req: Request, r data: selectedDesign, }); } catch (e) { + logger.error(`Failed to select design ${designId} for project ${projectId}: ${e}`); return res.status(500).json({ success: false, message: "Internal server error", diff --git a/apps/backend/src/modules/project.ts b/apps/backend/src/modules/project.ts index 20d22db..7714395 100644 --- a/apps/backend/src/modules/project.ts +++ b/apps/backend/src/modules/project.ts @@ -17,18 +17,19 @@ const projectRouter = Router(); const r2 = new R2(); projectRouter.get("/", auth, async (req: AuthRequest, res: Response) => { - const userId = req.user.id + const userId = req.user?.id if(!userId){ - - res.status(401).json({success: false, message: `UserId not given`}) + return res.status(401).json({success: false, message: `UserId not given`}) } - await prisma.project.findMany({where: {userId: userId}}) - const projects = await prisma.project.findMany({where: {userId: userId}}) - if(!projects){ - res.status(404).json({success: false, message: `Projects not found`}) + try{ + const projects = await prisma.project.findMany({where: {userId: userId}}) + return res.status(200).json({success: true, data: projects}) + } + catch(e){ + logger.error(`Failed to list projects for user ${userId}: ${e}`) + return res.status(500).json({success: false, message: `Internal server error`}) } - res.status(200).json({success: true, data: projects}) }); projectRouter.post("/", async (req: Request, res: Response) => { @@ -53,24 +54,30 @@ projectRouter.post("/", async (req: Request, res: Response) => { } return res.status(201).json({success: true, message: `project created`, data: saveIntoDB}) }catch(e){ - return res.json(500).json({message: `Internal server error`}) + logger.error(`Failed to create project for user ${userId}: ${e}`) + return res.status(500).json({success: false, message: `Internal server error`}) } }) projectRouter.get("/:projectId", auth, async (req: Request, res: Response) => { const projectId = req.params.projectId - if(!projectId){ - - res.status(401).json({success: false, message: `UserId not given`}) - } - if (typeof projectId !== "string") { + if (typeof projectId !== "string" || !projectId) { return res.status(400).json({ success: false, message: "Invalid projectId", }); } - const projects = await prisma.project.findUniqueOrThrow({where: {id: projectId}}) - res.status(200).json({success: true, data: projects}) + try{ + const project = await prisma.project.findUnique({where: {id: projectId}}) + if(!project){ + return res.status(404).json({success: false, message: `Project not found`}) + } + return res.status(200).json({success: true, data: project}) + } + catch(e){ + logger.error(`Failed to fetch project ${projectId}: ${e}`) + return res.status(500).json({success: false, message: `Internal server error`}) + } }); projectRouter.patch('/:projectId', auth, async (req: Request, res: Response) =>{ const projectId = req.params.projectId @@ -106,36 +113,37 @@ projectRouter.patch('/:projectId', auth, async (req: Request, res: Response) =>{ }); } try{ - const project = await prisma.project.findUniqueOrThrow({where: {id: projectId}}) + const project = await prisma.project.findUnique({where: {id: projectId}}) if(!project){ - return res.status(404).json({message: `Project not found`}) + return res.status(404).json({success: false, message: `Project not found`}) } const dbUpdate = await prisma.project.update({where: {id: projectId}, data: data}) return res.status(200).json({success: true, data: dbUpdate}) } catch(e){ - return res.status(500).json({message: `Internal Server error`}) + logger.error(`Failed to update project ${projectId}: ${e}`) + return res.status(500).json({success: false, message: `Internal Server error`}) } }) projectRouter.delete('/:projectId', auth, async (req: Request, res: Response) =>{ const projectId = req.params.projectId - if(!projectId){ - - res.status(401).json({success: false, message: `UserId not given`}) - } - if (typeof projectId !== "string") { + if (typeof projectId !== "string" || !projectId) { return res.status(400).json({ success: false, message: "Invalid projectId", }); } - const dbUpdate = await prisma.project.delete({where: {id: projectId}}) - if(!dbUpdate){ - return res.send(500).json({success: false, message: `Failed to update DB`}) + + try{ + await prisma.project.delete({where: {id: projectId}}) + return res.status(200).json({success: true}) + } + catch(e){ + logger.error(`Failed to delete project ${projectId}: ${e}`) + return res.status(500).json({success: false, message: `Failed to delete project`}) } - res.status(200).json({success: true}) }); // Files are synced sandbox -> R2 by E2BSandbox.SyncR2() during the run, so this diff --git a/apps/backend/src/modules/question.ts b/apps/backend/src/modules/question.ts index 9f07038..45f3fde 100644 --- a/apps/backend/src/modules/question.ts +++ b/apps/backend/src/modules/question.ts @@ -27,6 +27,7 @@ questionRouter.get('/:projectId/getQuestions', auth, async (req: Request, res: R return res.status(200).json({success: true, data: questions}) } catch(e){ + logger.error(`Failed to fetch questions for project ${projectId}: ${e}`) return res.status(500).json({ success: false, message: "Internal server error", @@ -42,24 +43,34 @@ questionRouter.post('/:projectId/:runId', internalAuth, async (req: Request, res if(typeof projectId !== 'string' || typeof runId !== 'string'){ return res.status(400).json({success: false, message: `Bad types of the params`}) } - const result = await Promise.all( - questionsObj.map((question: IncomingQuestion) => - prisma.question.create({ - data: { - id: randomUUIDv7(), - runId, - projectId, - question: question.question, - options: question.option, - createdAt: new Date(), - }, - }) - ) - ); - return res.status(201).json({ - success: true, - data: result - }); + if(!Array.isArray(questionsObj) || questionsObj.length === 0){ + return res.status(400).json({success: false, message: `questionsObj must be a non-empty array`}) + } + + try{ + const result = await Promise.all( + questionsObj.map((question: IncomingQuestion) => + prisma.question.create({ + data: { + id: randomUUIDv7(), + runId, + projectId, + question: question.question, + options: question.option, + createdAt: new Date(), + }, + }) + ) + ); + return res.status(201).json({ + success: true, + data: result + }); + } + catch(e){ + logger.error(`Failed to save questions for run ${runId}: ${e}`) + return res.status(500).json({success: false, message: `Internal server error`}) + } }) questionRouter.post('/:projectId/:runId/answers', auth, async (req: Request, res: Response) =>{ diff --git a/apps/backend/src/modules/run.ts b/apps/backend/src/modules/run.ts index e65fd3a..1a41fec 100644 --- a/apps/backend/src/modules/run.ts +++ b/apps/backend/src/modules/run.ts @@ -49,6 +49,7 @@ runRouter.get("/:projectId/runs", auth, async (req: Request, res: Response) => { data: runs, }); } catch (e) { + logger.error(`Failed to list runs for project ${projectId}: ${e}`); return res.status(500).json({ success: false, message: "Internal server error", @@ -90,6 +91,7 @@ runRouter.get("/:projectId/runs/:runId", auth, async (req: Request, res: Respons data: run, }); } catch (e) { + logger.error(`Failed to fetch run ${runId}: ${e}`); return res.status(500).json({ success: false, message: "Internal server error", @@ -142,6 +144,7 @@ runRouter.get("/:projectId/:runId/todos", auth, async (req: Request, res: Respon data: todos, }); } catch (e) { + logger.error(`Failed to fetch todos for run ${runId}: ${e}`); return res.status(500).json({ success: false, message: "Internal server error", @@ -204,6 +207,7 @@ runRouter.get("/:projectId/:runId/summaries", auth, async (req: Request, res: Re data: summaries, }); } catch (e) { + logger.error(`Failed to fetch summaries for run ${runId}: ${e}`); return res.status(500).json({ success: false, message: "Internal server error", diff --git a/apps/backend/src/modules/user.ts b/apps/backend/src/modules/user.ts index e3bdfdc..54d3229 100644 --- a/apps/backend/src/modules/user.ts +++ b/apps/backend/src/modules/user.ts @@ -24,7 +24,7 @@ userRouter.post("/signup", async (req: Request, res: Response) => { return res.status(400).json({ success: false, message: "Invalid email" }); } if (!isValidPassword(password)) { - logger.error(`Password is not valid: ${password}`); + logger.warn(`Rejected signup for ${email}: password does not meet requirements`); return res.status(400).json({ success: false, message: "Password must be at least 8 characters", @@ -50,7 +50,8 @@ userRouter.post("/signup", async (req: Request, res: Response) => { console.log("Token created"); return res.status(201).json({ success: true, data: { token, user: toPublicUser(user) } }); } catch (e) { - return res.status(500).json({ success: false, message: `Internal server error: ${e}` }); + logger.error(`Signup failed for ${email}: ${e}`); + return res.status(500).json({ success: false, message: "Internal server error" }); } }); @@ -83,6 +84,7 @@ userRouter.post("/login", async (req: Request, res: Response) => { .status(200) .json({ success: true, data: { token, user: toPublicUser(user) } }); } catch (e) { + logger.error(`Login failed for ${email}: ${e}`); return res .status(500) .json({ success: false, message: "Internal server error" }); @@ -98,9 +100,17 @@ userRouter.post("/google", async (req: Request, res: Response) => { .json({ success: false, message: "Missing idToken" }); } + let profile: Awaited>; try { - const profile = await verifyGoogleIdToken(idToken); + profile = await verifyGoogleIdToken(idToken); + } catch (e) { + logger.warn(`Google id token verification failed: ${e}`); + return res + .status(401) + .json({ success: false, message: "Invalid Google token" }); + } + try { let user = await prisma.user.findUnique({ where: { googleId: profile.googleId }, }); @@ -132,10 +142,13 @@ userRouter.post("/google", async (req: Request, res: Response) => { .status(200) .json({ success: true, data: { token, user: toPublicUser(user) } }); } catch (e) { - console.error("Google sign-in failed:", e); + // The token itself already verified above, so anything here is our + // problem, not a bad credential — a 401 sent the user back to the login + // screen for what was really a database failure. + logger.error(`Google sign-in failed after token verification: ${e}`); return res - .status(401) - .json({ success: false, message: "Invalid Google token" }); + .status(500) + .json({ success: false, message: "Internal server error" }); } }); @@ -155,7 +168,7 @@ userRouter.get("/me", auth, async (req: AuthRequest, res: Response) => { console.log(toPublicUser(user)); return res.status(200).json({ success: true, data: toPublicUser(user) }); } catch (e) { - console.error(`Error getting user: ${e}`); + logger.error(`Error getting user: ${e}`); return res .status(500) .json({ success: false, message: "Internal server error" }); diff --git a/apps/backend/src/modules/worker.ts b/apps/backend/src/modules/worker.ts index 0065c49..0fa9f41 100644 --- a/apps/backend/src/modules/worker.ts +++ b/apps/backend/src/modules/worker.ts @@ -33,6 +33,26 @@ const worker = new Worker("run-agent", async (job) => { },{connection: redis, lockDuration: 60_000, stalledInterval: 30_000, maxStalledCount: 1, concurrency: 5} ); -worker.on("failed", (job, err) => { - logger.error(`Job ${job?.id} failed: ${err.message}`); +worker.on("failed", async (job, err) => { + logger.error(`Job ${job?.id} failed: ${err.stack ?? err.message}`); + + // A job can die before the agent ever runs (sandbox boot, bad payload), in + // which case no run_failed event was emitted and the run would sit + // IN_PROGRESS forever. + const runId: unknown = job?.data?.runId; + if (typeof runId !== "string") return; + if (job && job.attemptsMade < (job.opts.attempts ?? 1)) return; + + try { + await prisma.run.update({ + where: { id: runId }, + data: { status: "FAILED", endedAt: new Date() }, + }); + } catch (e) { + logger.error(`Failed to mark run ${runId} as FAILED after job failure: ${e}`); + } +}); + +worker.on("error", (err) => { + logger.error(`Run worker error: ${err instanceof Error ? err.message : String(err)}`); }); \ No newline at end of file diff --git a/apps/frontend/src/lib/run.tsx b/apps/frontend/src/lib/run.tsx index 0c7d160..0fe8681 100644 --- a/apps/frontend/src/lib/run.tsx +++ b/apps/frontend/src/lib/run.tsx @@ -94,7 +94,10 @@ export function RunProvider({ children }: { children: ReactNode }) { let event: OrchestratorEvent; try { event = JSON.parse(raw); - } catch { + } catch (err) { + // Skipping one malformed frame is fine, silently dropping every + // frame of a broken stream is not. + console.error("Discarded unparseable run event", { runId, raw, err }); return; } @@ -231,8 +234,18 @@ export function RunProvider({ children }: { children: ReactNode }) { setState({ status: "failed", runId, projectId, error: failedEvent?.error ?? "This run failed." }); return true; } + // A run we can't map to a UI state (e.g. still PENDING) isn't a + // failure, but callers can only see "false" so leave a trace. + console.warn(`No resumable UI state for run ${runId} (status ${status})`); return false; - } catch { + } catch (err) { + console.error(`Failed to resume run ${runId}`, err); + setState({ + status: "failed", + runId, + projectId: "", + error: messageFor(err, "Couldn't load this build."), + }); return false; } }, diff --git a/apps/frontend/src/pages/Projects.tsx b/apps/frontend/src/pages/Projects.tsx index 45fdde7..8b35003 100644 --- a/apps/frontend/src/pages/Projects.tsx +++ b/apps/frontend/src/pages/Projects.tsx @@ -40,11 +40,13 @@ export function Projects() { ); try { await api.patch(`/api/project/${project.id}`, { starred: !project.isStarred }); - } catch { - // revert on failure + } catch (err) { + // revert on failure, and say so — the star silently snapping back looked + // like a UI bug setProjects((prev) => prev?.map((p) => (p.id === project.id ? { ...p, isStarred: project.isStarred } : p)) ?? null, ); + setOpenError(err instanceof ApiError ? err.message : "Couldn't update this project."); } }; diff --git a/packages/agents/agent/agent.ts b/packages/agents/agent/agent.ts index 8511d08..2d2e176 100644 --- a/packages/agents/agent/agent.ts +++ b/packages/agents/agent/agent.ts @@ -1,7 +1,9 @@ import type { OrchestratorResponse, OrchestratorSSE, Project, User, Answers, BootstrapResponse, DesignOption } from "../types/agentTypes" import { E2BSandbox } from "./utils/sandbox" import { b } from "../baml_client" -import {type ComplexityLevel, type Error, type Question, type PlannerTodo, type ToolResult} from '../baml_client/types' +// Aliased: baml's `Error` model would otherwise shadow the global Error +// constructor in this file. +import {type ComplexityLevel, type Error as TaskError, type Question, type PlannerTodo, type ToolResult} from '../baml_client/types' import { COMPLEXITY_CHECKER_AND_QUESTION_GENERATOR_PROMPT, ORCHESTRATOR_SUMMARY_PROMPT, PLAN_TASK_SYSTEM_PROMPT} from "./config/sysPrompts" import { DAG } from "./services/dag" import { Screen } from "@google/stitch-sdk" @@ -37,10 +39,10 @@ type OrchestratorContext = { type OrchestratorState = { screenId: string | null // last most scrreen screenIdByTaskId: Map // taskId (uiExpert) -> screenId, for dependency-specific lookups - lastTestErrors: Error[] + lastTestErrors: TaskError[] lastToolResult: ToolResult | null - lastError: Error | null - errorsByTaskId: Map // taskId (tester) -> errors, if debugger needs a specific tester's output + lastError: TaskError | null + errorsByTaskId: Map // taskId (tester) -> errors, if debugger needs a specific tester's output } @@ -287,14 +289,28 @@ export class OrchestratorAgent{ async Orchestrate(userPrompt: string, answers?: Answers[], selectedDesignId?: string): Promise{ logger.info(`Running orchestrator`) if(selectedDesignId){ - await axios.patch(`${BACKEND_URL}/api/design/${this.projectId}/designs/${selectedDesignId}`, {}, { headers: internalAuthHeader() }) + try{ + await axios.patch(`${BACKEND_URL}/api/design/${this.projectId}/designs/${selectedDesignId}`, {}, { headers: internalAuthHeader() }) + } + catch(e){ + // Without a persisted selection Bootstrap would just ask for a + // design again, so fail the run loudly instead of looping. + const reason = `Failed to persist design selection ${selectedDesignId}: ${e instanceof Error ? e.message : String(e)}` + logger.error(reason) + await this.emitter.emit({ type: 'run_failed', error: reason }) + return { + status: 'error', + reason + } + } } var data; try{ data = await this.Bootstrap(userPrompt, answers); } catch(e){ - const reason = `Bootstrap failed with error ${e}` + const reason = `Bootstrap failed with error ${e instanceof Error ? e.message : String(e)}` + logger.error(`Bootstrap failed for run ${this.runId}: ${e instanceof Error ? e.stack ?? e.message : String(e)}`) await this.emitter.emit({ type: 'run_failed', error: reason }) return { status: 'error', @@ -305,7 +321,20 @@ export class OrchestratorAgent{ if(data.status === 'clarification_needed'){ if(!data.alreadySaved){ logger.info(`LLM generated questions, saving them`) - await axios.post(`${BACKEND_URL}/api/question/${this.projectId}/${this.runId}`, { questionsObj: data.questions }, { headers: internalAuthHeader() }) + try{ + await axios.post(`${BACKEND_URL}/api/question/${this.projectId}/${this.runId}`, { questionsObj: data.questions }, { headers: internalAuthHeader() }) + } + catch(e){ + // Unsaved questions can't be answered later (the answers + // endpoint resolves them by id), so this can't be a warning. + const reason = `Failed to save clarifying questions: ${e instanceof Error ? e.message : String(e)}` + logger.error(reason) + await this.emitter.emit({ type: 'run_failed', error: reason }) + return { + status: 'error', + reason + } + } } await this.emitter.emit({ type: 'clarification_needed', questions: data.questions }) return { @@ -337,6 +366,10 @@ export class OrchestratorAgent{ const mainResult = await mainAgent.runLoop() if(!mainResult.success){ + // Without this the run stays IN_PROGRESS forever: run status is + // only persisted off emitted events. + logger.error(`Main agent failed for run ${this.runId}: ${mainResult.summary}`) + await this.emitter.emit({ type: 'run_failed', error: mainResult.summary }) return { status: 'error', reason: mainResult.summary @@ -388,13 +421,21 @@ export class OrchestratorAgent{ } let testsPassing: boolean | null = null; - let lastErrors = null + let lastErrors: TaskError | null = null let testResults if (agentType === 'coder') { // #TODO: Make this below loop as batch testing of dependent DAG tasks logger.info(`Starting tester debugger loop`) testsPassing = false; testResults = await this.TesterDebuggerLoop(this.semanticMem) if(testResults.success) testsPassing = true + else { + // A coder task whose code never builds isn't a success — + // carry that (and the last error) into the orchestrator + // context so later tasks and the summary see it. + lastErrors = testResults.lastError ?? null + logger.error(`Tester/debugger loop could not get task ${todo.id} building: ${JSON.stringify(lastErrors)}`) + summaries.push(...testResults.summaries) + } } // this.shouldBatchTest() @@ -402,8 +443,10 @@ export class OrchestratorAgent{ taskId: todo.id, task: todo.task, agentAssigned: agentType, - summary: result.summary, - success: result.success + summary: testsPassing === false + ? `${result.summary}\nTests/build still failing: ${lastErrors ? `${lastErrors.fileName}: ${lastErrors.error}` : 'unknown error'}` + : result.summary, + success: result.success && testsPassing !== false }); } // FIX: this.state/context in place of summaries. => done, kept subagents summary short and avoided LLM call. @@ -425,8 +468,10 @@ export class OrchestratorAgent{ return result; } catch(e){ + const reason = `Failed to expose preview url: ${e instanceof Error ? e.message : String(e)}` logger.error(`Error occurred while hosting ${e}`) - throw new Error + await this.emitter.emit({ type: 'run_failed', error: reason }) + throw new Error(reason, { cause: e }) } // Deploy if only user says this explictily // #TEST: replace with appropriate path of project directory @@ -460,7 +505,7 @@ export class OrchestratorAgent{ return await b.OrchestratorSummary(ORCHESTRATOR_SUMMARY_PROMPT, summaries) } // that tester <-> debugger loop - async TesterDebuggerLoop(semanticMem: string, ): Promise<{success: true | false, summaries: string[], lastError?: Error}>{ + async TesterDebuggerLoop(semanticMem: string, ): Promise<{success: true | false, summaries: string[], lastError?: TaskError}>{ let loopCount = 0; let summaries: string[] = [] let lastError @@ -481,9 +526,20 @@ export class OrchestratorAgent{ const tester = new TesterAgent(this.userId, this.projectId, this.sandbox) const testerRes: TesterResponse = await tester.testCodebase(testerContext) - const error: Error = { - fileName: testerRes.errorRes!.file, - error: testerRes.errorRes!.error + testerRes.errorRes!.line + if(!testerRes.errorRes){ + // Nothing for the debugger to act on: dereferencing errorRes + // here used to throw a TypeError that the catch below turned + // into a bare `success: false`, hiding why the loop ended. + logger.warn(`Tester returned success=${testerRes.success} with no error payload while build check still failing, ending tester/debugger loop`) + return { + success: false, + summaries: summaries, + lastError: lastError ?? { fileName: 'TESTER_NO_ERROR', error: 'Build check failed but tester reported no error' } + } + } + const error: TaskError = { + fileName: testerRes.errorRes.file, + error: testerRes.errorRes.error + testerRes.errorRes.line } // #CRITICAL: halt only after the debugger has had 2 attempts at the same // error signature with no progress, not on the first repeat. @@ -528,17 +584,23 @@ export class OrchestratorAgent{ deployReady = await this.preDeployCheck() loopCount++; } + if(!deployReady){ + logger.error(`Tester/debugger loop exhausted ${TESTER_DEBUGGER_LOOP_MAX_ITERATIONS} iteration(s) with the build still failing`) + } return { - success: true, + // Falling out of the loop on the iteration cap is a failure, not + // a pass — only a green pre-deploy check counts as success. + success: deployReady, summaries: summaries, lastError: lastError } } catch(e){ - logger.error(`TesterDebuggerLoop failed: ${e}`) + logger.error(`TesterDebuggerLoop failed: ${e instanceof Error ? e.stack ?? e.message : String(e)}`) return{ success: false, summaries, + lastError: lastError ?? { fileName: 'TESTER_DEBUGGER_LOOP', error: e instanceof Error ? e.message : String(e) } } } } diff --git a/packages/agents/agent/events.ts b/packages/agents/agent/events.ts index 9f6533c..fa237cc 100644 --- a/packages/agents/agent/events.ts +++ b/packages/agents/agent/events.ts @@ -3,6 +3,7 @@ import type { Question } from '../baml_client/types'; import axios from 'axios'; import { BACKEND_URL, REDIS_HOST, REDIS_PORT } from './config/systemConfig'; import IORedis from "ioredis"; +import { logger } from "./utils/logger"; export type OrchestratorEvent = MainAgentEvents | { type: "orchestrator_agent_started"; } | { type: "clarification_needed"; questions: Question[] } @@ -37,8 +38,10 @@ export function createBackendEmitter(runId: string): EventEmitter{ timeout: 5000, }) } catch(err){ - console.error(`Failed to emit event for run ${runId}:`, err) - + // Event persistence is best-effort — losing a progress event + // must not abort the run — but it also drives run status, so it + // has to be loud. + logger.error(`Failed to emit ${event.type} event for run ${runId}: ${err instanceof Error ? err.message : String(err)}`) } } } @@ -50,13 +53,19 @@ const redisPublisher = new IORedis({ maxRetriesPerRequest: null }); +// Without a listener ioredis' 'error' events become unhandled and take the +// whole worker process down. +redisPublisher.on("error", (err) => { + logger.error(`Redis publisher error: ${err instanceof Error ? err.message : String(err)}`) +}); + export function createRedisEmitter(runId: string): EventEmitter{ return { async emit(event: OrchestratorEvent){ try{ await redisPublisher.publish(`run:${runId}`, JSON.stringify(event)) } catch(err){ - console.error(`Failed to publish event for run ${runId}:`, err) + logger.error(`Failed to publish ${event.type} event for run ${runId}: ${err instanceof Error ? err.message : String(err)}`) } } } diff --git a/packages/agents/agent/index.ts b/packages/agents/agent/index.ts index 8d5d48c..b8bf477 100644 --- a/packages/agents/agent/index.ts +++ b/packages/agents/agent/index.ts @@ -3,6 +3,7 @@ import type { Answers, OrchestratorResponse } from "../types/agentTypes" import { OrchestratorAgent } from "./agent" import { createRunEmitter, type EventEmitter } from "./events"; import { E2BSandbox } from "./utils/sandbox" +import { logger } from "./utils/logger" // export async function SpinUpSandbox(userId: string, projectId: string): Promise{ @@ -20,7 +21,7 @@ export async function AgentCall( semanticMem: string, answers?: Answers[], selectedDesignId?: string -): Promise { +): Promise { const orchestrator: OrchestratorAgent = new OrchestratorAgent(userId, projectId, sandbox, runId, semanticMem) @@ -28,7 +29,11 @@ export async function AgentCall( const result = await orchestrator.Orchestrate(userPrompt, answers, selectedDesignId) return result } catch (err) { - await createRunEmitter(runId).emit({ type: "run_failed", error: String(err) }) + // The event is only for the UI — the job itself must still fail so BullMQ + // marks it failed instead of completed with an undefined result. + logger.error(`AgentCall failed for run ${runId}: ${err instanceof Error ? err.stack ?? err.message : String(err)}`) + await createRunEmitter(runId).emit({ type: "run_failed", error: err instanceof Error ? err.message : String(err) }) + throw err } } diff --git a/packages/agents/agent/mainAgent.ts b/packages/agents/agent/mainAgent.ts index b2e9830..76411f6 100644 --- a/packages/agents/agent/mainAgent.ts +++ b/packages/agents/agent/mainAgent.ts @@ -52,6 +52,8 @@ export class MainAgent{ async runLoop(): Promise{ logger.info(`[MainAgent:${this.runId}] runLoop starting, maxIterations=${MAIN_AGENT_MAX_ITERATIONS}`) + let completed = false + let aborted = false try{ const updatedSystemPrompt = MAIN_AGENT_SYSTEM_PROMPT + await this.buildSystemPrompt() while(this.iterations < MAIN_AGENT_MAX_ITERATIONS){ @@ -76,6 +78,7 @@ export class MainAgent{ timestamp: new Date().toISOString() }) shouldBreak = true + completed = true } if(response.stopReason === 'aborted'){ logger.warn(`[MainAgent:${this.runId}] LLM call aborted at iteration ${this.iterations}`) @@ -85,6 +88,7 @@ export class MainAgent{ timestamp: new Date().toISOString() }) shouldBreak = true + aborted = true } // if(response.stopReason === 'QnA'){ @@ -160,7 +164,7 @@ export class MainAgent{ logger.info(`[MainAgent:${this.runId}] runLoop breaking after iteration ${this.iterations}`) break } - this.saveSessionState() // write to Postgres — failure recovery + void this.saveSessionState() // write to Postgres — failure recovery this.iterations++ } } @@ -172,9 +176,20 @@ export class MainAgent{ } } logger.info(`[MainAgent:${this.runId}] runLoop finished after ${this.iterations} iterations, building summary`) + const summary = await this.BuildSummary() + if(aborted){ + logger.error(`[MainAgent:${this.runId}] runLoop ended on an aborted LLM call`) + return { success: false, summary: `Main agent aborted: ${summary}` } + } + if(!completed){ + // Ran out of iterations without the LLM ever signalling completion — + // reporting that as success hid unfinished builds. + logger.error(`[MainAgent:${this.runId}] runLoop hit the ${MAIN_AGENT_MAX_ITERATIONS} iteration cap without completing`) + return { success: false, summary: `Main agent hit the ${MAIN_AGENT_MAX_ITERATIONS} iteration cap without completing: ${summary}` } + } return { success: true, - summary: await this.BuildSummary() + summary } } diff --git a/packages/agents/agent/subagents/coder.ts b/packages/agents/agent/subagents/coder.ts index c53e5cf..4018919 100644 --- a/packages/agents/agent/subagents/coder.ts +++ b/packages/agents/agent/subagents/coder.ts @@ -89,11 +89,11 @@ export class CoderAgent extends BaseAgent { @@ -104,7 +104,8 @@ export class DebuggerAgent extends BaseAgent logger.warn(`Failed to kill dev server after tester error: ${killErr}`)) throw e } diff --git a/packages/agents/agent/utils/sandbox.ts b/packages/agents/agent/utils/sandbox.ts index 4c463ee..bed8abb 100644 --- a/packages/agents/agent/utils/sandbox.ts +++ b/packages/agents/agent/utils/sandbox.ts @@ -38,6 +38,7 @@ export class E2BSandbox{ sandbox = await Sandbox.connect(sandboxId) await sandbox.setTimeout(SANDBOX_TIMEOUT_MS) } catch (e) { + logger.warn(`Could not reconnect to sandbox ${sandboxId}, creating a new one: ${e instanceof Error ? e.message : String(e)}`) sandbox = null } } @@ -110,7 +111,7 @@ export class E2BSandbox{ } catch(e){ logger.error(`Failed to generate repo tree: ${e}`) - throw new Error(`Error occurred while generating repository tree`) + throw new Error(`Failed to generate repository tree: ${e instanceof Error ? e.message : String(e)}`, { cause: e }) } } @@ -128,7 +129,7 @@ export class E2BSandbox{ } catch(e){ logger.error(`Failed to read ${path}: ${e}`) - throw new Error("Error occured while reading from sandbox file") + throw new Error(`Failed to read ${path} from sandbox: ${e instanceof Error ? e.message : String(e)}`, { cause: e }) } } else if(payload.action === 'writeFile'){ @@ -143,7 +144,7 @@ export class E2BSandbox{ } catch(e){ logger.error(`Failed to write ${path}: ${e}`) - throw new Error("Error occurred while executing write sandbox file") + throw new Error(`Failed to write ${path} in sandbox: ${e instanceof Error ? e.message : String(e)}`, { cause: e }) } } else if(payload.action === 'editFile'){ @@ -161,7 +162,7 @@ export class E2BSandbox{ } catch(e){ logger.error(`Failed to delete ${path}: ${e}`) - throw new Error("Error occurred while executing deleting sandbox file") + throw new Error(`Failed to delete ${path} in sandbox: ${e instanceof Error ? e.message : String(e)}`, { cause: e }) } } else if(payload.action === 'runCommand'){ @@ -198,13 +199,12 @@ export class E2BSandbox{ } } logger.error(`Failed to run command "${payload.command}" (cwd: ${cwd}): ${e}`) - throw new Error("Error occurred while executing sandbox cmd") + throw new Error(`Failed to run "${payload.command}" in sandbox: ${e instanceof Error ? e.message : String(e)}`, { cause: e }) } } - return { - success: false, - content: "Unknown error occurred" - } + // Every known action returns above, so reaching here means the caller + // sent an action this sandbox doesn't implement. + throw new Error(`Unsupported sandbox action: ${(payload as {action?: string}).action}`) } /* Steps: @@ -268,8 +268,15 @@ export class E2BSandbox{ throw new Error(`Error occurred while starting the preview server: ${e instanceof Error ? e.message : String(e)}`) } } - Release(){ - this.sandbox.kill() + async Release(){ + try{ + await this.sandbox.kill() + } + catch(e){ + // Worth knowing about (it leaks a paid sandbox) but never worth + // failing a finished run over. + logger.error(`Failed to kill sandbox ${this.sandboxId}: ${e instanceof Error ? e.message : String(e)}`) + } } }