From 317aec35e2d86f710f2fb7b13c66c3a7f39ffd32 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Wed, 28 Jan 2026 09:40:44 -0500 Subject: [PATCH 01/14] Re-enable windows builds (#5893) * Re-enable windows builds The incident has ended: https://www.githubstatus.com/incidents/90hj03y5tj3c * Run prettier --- .github/workflows/general.yml | 5 +++-- .github/workflows/python-client-build.yml | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/general.yml b/.github/workflows/general.yml index 5c64d6ba3ac..4de7152ac18 100644 --- a/.github/workflows/general.yml +++ b/.github/workflows/general.yml @@ -1419,7 +1419,8 @@ jobs: check-all-general-jobs-passed: permissions: {} if: always() && github.repository == 'tensorzero/tensorzero' - needs: [ + needs: + [ check-version-consistency, check-production-deployment-docker-compose, check-if-edited-then-edit, @@ -1428,7 +1429,7 @@ jobs: check-python-client-build, check-node-bindings, check-python-schemas, - # TODO (#5873): Add build-windows back when re-enabled + build-windows, build-ui-container, build-gateway-container, build-gateway-e2e-container, diff --git a/.github/workflows/python-client-build.yml b/.github/workflows/python-client-build.yml index eba2b958742..a7622893d80 100644 --- a/.github/workflows/python-client-build.yml +++ b/.github/workflows/python-client-build.yml @@ -120,8 +120,7 @@ jobs: windows: runs-on: ${{ matrix.platform.runner }} - # TODO (#5873): Temporarily disabled - re-enable when ready - if: false && github.repository == 'tensorzero/tensorzero' + if: github.repository == 'tensorzero/tensorzero' strategy: matrix: platform: @@ -255,8 +254,7 @@ jobs: uv run twine check ./clients/python/dist/tensorzero-*.tar.gz check-artifacts: - # TODO (#5873): Add windows back when re-enabled - needs: [linux, musllinux, macos, sdist] + needs: [linux, musllinux, windows, macos, sdist] runs-on: ubuntu-latest if: github.repository == 'tensorzero/tensorzero' steps: @@ -268,5 +266,4 @@ jobs: ls -lh ./wheels-*/* # Count the number of files named 'tensorzero-*.whl', and verify that it equals 5 # If we add new platforms / python versions, update this number - # TODO (#5873): Update to 6 when windows is re-enabled - ls -lh ./wheels-*/* | grep -c 'tensorzero-.*\.whl' | grep -q 5 + ls -lh ./wheels-*/* | grep -c 'tensorzero-.*\.whl' | grep -q 6 From a7bc928ea48533499e030a0e499e557d241b7405 Mon Sep 17 00:00:00 2001 From: Gabriel Bianconi <1275491+GabrielBianconi@users.noreply.github.com> Date: Wed, 28 Jan 2026 10:26:20 -0500 Subject: [PATCH 02/14] Distroless provider proxy - Phase 1 (#5928) * Distroless provider proxy * Distroless provider proxy * Fix * Fix --- provider-proxy/Dockerfile | 5 +++ provider-proxy/src/lib.rs | 20 ++++----- provider-proxy/src/main.rs | 45 ++++++++++++++++--- .../tests/e2e/docker-compose.live.yml | 2 - 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/provider-proxy/Dockerfile b/provider-proxy/Dockerfile index a11ffb47d9c..ed4b44a3424 100644 --- a/provider-proxy/Dockerfile +++ b/provider-proxy/Dockerfile @@ -28,8 +28,13 @@ COPY --from=builder /release/provider-proxy /usr/local/bin/provider-proxy WORKDIR /app +# Port 3003: proxy +# Port 3004: health check EXPOSE 3003 3004 +# Usage: provider-proxy health-check [--port XXXX] +HEALTHCHECK CMD ["provider-proxy", "health-check"] + ENTRYPOINT ["provider-proxy"] CMD ["--help"] diff --git a/provider-proxy/src/lib.rs b/provider-proxy/src/lib.rs index e8d96a63733..a00ad2ed033 100644 --- a/provider-proxy/src/lib.rs +++ b/provider-proxy/src/lib.rs @@ -493,16 +493,6 @@ pub async fn run_server(args: Args, server_started: oneshot::Sender) std::fs::create_dir_all(&args.cache_path).expect("Failed to create cache directory"); - // Start health check server - let health_port = args.health_port; - // TODO(https://github.com/tensorzero/tensorzero/issues/3983): Audit this callsite - #[expect(clippy::disallowed_methods)] - tokio::spawn(async move { - if let Err(e) = run_health_server(health_port).await { - tracing::error!("Health check server failed: {:?}", e); - } - }); - let _ = rustls::crypto::ring::default_provider() .install_default() .inspect_err(|e| tracing::error!("Failed to install rustls ring provider: {e:?}")); @@ -592,6 +582,16 @@ pub async fn run_server(args: Args, server_started: oneshot::Sender) .await .unwrap(); + // Start health check server + let health_port = args.health_port; + // TODO(https://github.com/tensorzero/tensorzero/issues/3983): Audit this callsite + #[expect(clippy::disallowed_methods)] + tokio::spawn(async move { + if let Err(e) = run_health_server(health_port).await { + tracing::error!("Health check server failed: {:?}", e); + } + }); + tracing::info!(?args, "HTTP Proxy is listening on http://{server_addr}"); server_started .send(server_addr) diff --git a/provider-proxy/src/main.rs b/provider-proxy/src/main.rs index d7f69887244..133e5b707a4 100644 --- a/provider-proxy/src/main.rs +++ b/provider-proxy/src/main.rs @@ -1,10 +1,45 @@ -use clap::Parser; +use clap::{Parser, Subcommand}; use provider_proxy::{Args, run_server}; +use std::process::ExitCode; use tokio::sync::oneshot; +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Cli { + #[command(subcommand)] + command: Option, + #[command(flatten)] + args: Args, +} + +#[derive(Subcommand, Debug)] +enum Commands { + /// Check if the health endpoint is responding (for container health checks) + HealthCheck { + /// Health check port + #[arg(long, default_value = "3004")] + port: u16, + }, +} + +async fn run_health_check(port: u16) -> ExitCode { + let url = format!("http://127.0.0.1:{port}/health"); + match reqwest::get(&url).await { + Ok(response) if response.status().is_success() => ExitCode::SUCCESS, + _ => ExitCode::FAILURE, + } +} + #[tokio::main] -async fn main() { - let args = Args::parse(); - let (server_started_tx, _server_started_rx) = oneshot::channel(); - run_server(args, server_started_tx).await; +async fn main() -> ExitCode { + let cli = Cli::parse(); + + match cli.command { + Some(Commands::HealthCheck { port }) => run_health_check(port).await, + None => { + let (server_started_tx, _server_started_rx) = oneshot::channel(); + run_server(cli.args, server_started_tx).await; + ExitCode::SUCCESS + } + } } diff --git a/tensorzero-core/tests/e2e/docker-compose.live.yml b/tensorzero-core/tests/e2e/docker-compose.live.yml index 231fd1f37dd..20e70f170b9 100644 --- a/tensorzero-core/tests/e2e/docker-compose.live.yml +++ b/tensorzero-core/tests/e2e/docker-compose.live.yml @@ -30,8 +30,6 @@ services: "${PROVIDER_PROXY_CACHE_MODE:-read-old-write-new}", ] healthcheck: - test: - ["CMD", "wget", "--spider", "--tries=1", "http://localhost:3004/health"] start_period: 10s start_interval: 1s timeout: 1s From 9e09f0e87d596ca9bf456d09eb7e85da5129c400 Mon Sep 17 00:00:00 2001 From: Viraj Mehta Date: Wed, 28 Jan 2026 10:56:39 -0500 Subject: [PATCH 03/14] gave the client the ability to cancel sessions (#5923) * gave the client the ability to cancel sessions * added ui handling for cancelling sessions * improved error handling, added gateway route * made this a stop button * cancel client-side tool execution when the cancel route is called * changed cancel->interrupt everywhere * fixed issue with toast in wrong session * added feature flag to ui for interruption * forgot a file * improved error handling --- gateway/src/routes/internal.rs | 4 ++ internal/autopilot-client/src/client.rs | 37 +++++++++++ internal/durable-tools-spawn/src/client.rs | 42 ++++++++++++ .../src/endpoints/internal/autopilot.rs | 42 ++++++++++++ ui/app/components/autopilot/ChatInput.tsx | 65 +++++++++++++------ ui/app/routes.ts | 4 ++ .../$session_id/actions/interrupt.route.ts | 37 +++++++++++ .../autopilot/sessions/$session_id/route.tsx | 50 ++++++++++++++ ui/app/utils/feature_flags.ts | 14 +++- ui/app/utils/tensorzero/autopilot-client.ts | 12 ++++ 10 files changed, 286 insertions(+), 21 deletions(-) create mode 100644 ui/app/routes/api/autopilot/sessions/$session_id/actions/interrupt.route.ts diff --git a/gateway/src/routes/internal.rs b/gateway/src/routes/internal.rs index 48188e4f30d..59a828b734a 100644 --- a/gateway/src/routes/internal.rs +++ b/gateway/src/routes/internal.rs @@ -257,6 +257,10 @@ pub fn build_internal_non_otel_enabled_routes() -> Router { "/internal/autopilot/v1/sessions/{session_id}/actions/approve_all", post(endpoints::internal::autopilot::approve_all_tool_calls_handler), ) + .route( + "/internal/autopilot/v1/sessions/{session_id}/actions/interrupt", + post(endpoints::internal::autopilot::interrupt_session_handler), + ) // Other Autopilot endpoints .route( "/internal/autopilot/status", diff --git a/internal/autopilot-client/src/client.rs b/internal/autopilot-client/src/client.rs index fdac07646d0..f1200882f65 100644 --- a/internal/autopilot-client/src/client.rs +++ b/internal/autopilot-client/src/client.rs @@ -397,6 +397,43 @@ impl AutopilotClient { Ok(body) } + /// Interrupts a session. + pub async fn interrupt_session(&self, session_id: Uuid) -> Result<(), AutopilotError> { + let url = self + .base_url + .join(&format!("/v1/sessions/{session_id}/actions/interrupt"))?; + let response = self + .http_client + .post(url) + .headers(self.auth_headers()) + .send() + .await?; + self.check_response(response).await?; + Ok(()) + } + + /// Interrupt all durable tasks associated with a session ID. + /// + /// This interrupts any running durable tasks (and their recursive children) + /// that were spawned for the given session. + /// + /// # Returns + /// + /// Returns the number of tasks that were interrupted. + /// + /// # Errors + /// + /// Returns an error if the interruption fails. + pub async fn interrupt_tasks_for_session( + &self, + session_id: Uuid, + ) -> Result { + self.spawn_client + .interrupt_tasks_by_session_id(session_id) + .await + .map_err(AutopilotError::from) + } + // ------------------------------------------------------------------------- // Event Endpoints // ------------------------------------------------------------------------- diff --git a/internal/durable-tools-spawn/src/client.rs b/internal/durable-tools-spawn/src/client.rs index fe2acb02e56..ab929b4c22a 100644 --- a/internal/durable-tools-spawn/src/client.rs +++ b/internal/durable-tools-spawn/src/client.rs @@ -154,6 +154,48 @@ impl SpawnClient { pub fn pool(&self) -> &PgPool { self.durable.pool() } + + /// Interrupt all durable tasks associated with a session ID. + /// + /// This queries for non-terminal tasks where `params->'side_info'->>'session_id'` + /// matches the provided session ID, then interrupts each one. The durable framework's + /// `cancel_task` function automatically cascades interruption to child tasks. + /// + /// # Returns + /// + /// Returns the number of tasks that were interrupted. + /// + /// # Errors + /// + /// Returns an error if any database operation fails. + pub async fn interrupt_tasks_by_session_id(&self, session_id: Uuid) -> Result { + let queue_name = self.queue_name(); + + // Find all non-terminal tasks with this session_id + // Using QueryBuilder for dynamic table name (queue_name is a trusted internal value) + let mut query_builder: sqlx::QueryBuilder = + sqlx::QueryBuilder::new("SELECT task_id FROM durable.t_"); + query_builder.push(queue_name); + query_builder.push(" WHERE params->'side_info'->>'session_id' = "); + query_builder.push_bind(session_id.to_string()); + query_builder.push(" AND state NOT IN ('completed', 'failed', 'cancelled')"); + + let task_ids: Vec<(Uuid,)> = query_builder + .build_query_as() + .fetch_all(self.pool()) + .await?; + + // Interrupt each task (cascade to children handled by durable.cancel_task) + for (task_id,) in &task_ids { + sqlx::query("SELECT durable.cancel_task($1, $2)") + .bind(queue_name) + .bind(task_id) + .execute(self.pool()) + .await?; + } + + Ok(task_ids.len() as u64) + } } /// Builder for creating a [`SpawnClient`]. diff --git a/tensorzero-core/src/endpoints/internal/autopilot.rs b/tensorzero-core/src/endpoints/internal/autopilot.rs index 016e2abe879..64da7316d78 100644 --- a/tensorzero-core/src/endpoints/internal/autopilot.rs +++ b/tensorzero-core/src/endpoints/internal/autopilot.rs @@ -136,6 +136,35 @@ pub async fn approve_all_tool_calls( .map_err(Error::from) } +/// Interrupt an autopilot session via the Autopilot API. +/// +/// This interrupts all durable tasks associated with the session (best effort), +/// then interrupts the session via the Autopilot API. +/// +/// This is the core function called by both the HTTP handler and embedded client. +pub async fn interrupt_session( + autopilot_client: &AutopilotClient, + session_id: Uuid, +) -> Result<(), Error> { + // Interrupt durable tasks first (best effort - log warning on failure) + if let Err(e) = autopilot_client + .interrupt_tasks_for_session(session_id) + .await + { + tracing::warn!( + session_id = %session_id, + error = %e, + "Failed to interrupt durable tasks for session" + ); + } + + // Then interrupt the session via autopilot API + autopilot_client + .interrupt_session(session_id) + .await + .map_err(Error::from) +} + // ============================================================================= // HTTP Handlers // ============================================================================= @@ -235,6 +264,19 @@ pub async fn approve_all_tool_calls_handler( Ok(Json(response)) } +/// Handler for `POST /internal/autopilot/v1/sessions/{session_id}/actions/interrupt` +/// +/// Interrupts an autopilot session via the Autopilot API. +#[axum::debug_handler(state = AppStateData)] +#[instrument(name = "autopilot.interrupt_session", skip_all, fields(session_id = %session_id))] +pub async fn interrupt_session_handler( + State(app_state): AppState, + Path(session_id): Path, +) -> Result<(), Error> { + let client = get_autopilot_client(&app_state)?; + interrupt_session(&client, session_id).await +} + /// Handler for `GET /internal/autopilot/status` /// /// Returns whether autopilot is configured (i.e., whether `TENSORZERO_AUTOPILOT_API_KEY` is set). diff --git a/ui/app/components/autopilot/ChatInput.tsx b/ui/app/components/autopilot/ChatInput.tsx index 3d16c475f62..707b7ed9441 100644 --- a/ui/app/components/autopilot/ChatInput.tsx +++ b/ui/app/components/autopilot/ChatInput.tsx @@ -1,4 +1,4 @@ -import { SendHorizontal, Loader2 } from "lucide-react"; +import { SendHorizontal, Loader2, StopCircle } from "lucide-react"; import { useState, useRef, @@ -34,6 +34,9 @@ type ChatInputProps = { submitDisabled?: boolean; className?: string; isNewSession?: boolean; + isInterruptible?: boolean; + isInterrupting?: boolean; + onInterrupt?: () => void; }; export function ChatInput({ @@ -44,6 +47,9 @@ export function ChatInput({ submitDisabled = false, className, isNewSession = false, + isInterruptible = false, + isInterrupting = false, + onInterrupt, }: ChatInputProps) { const [text, setText] = useState(""); const fetcher = useFetcher(); @@ -155,24 +161,45 @@ export function ChatInput({ style={{ minHeight: MIN_HEIGHT, maxHeight: MAX_HEIGHT }} rows={1} /> - + {submitDisabled && isInterruptible ? ( + + ) : ( + + )} ); } diff --git a/ui/app/routes.ts b/ui/app/routes.ts index 964971c06a6..3a9fdaa6f3b 100644 --- a/ui/app/routes.ts +++ b/ui/app/routes.ts @@ -69,6 +69,10 @@ export default [ "autopilot/sessions/:session_id/events/message", "routes/api/autopilot/sessions/$session_id/events/message.route.ts", ), + route( + "autopilot/sessions/:session_id/actions/interrupt", + "routes/api/autopilot/sessions/$session_id/actions/interrupt.route.ts", + ), ]), // Datasets diff --git a/ui/app/routes/api/autopilot/sessions/$session_id/actions/interrupt.route.ts b/ui/app/routes/api/autopilot/sessions/$session_id/actions/interrupt.route.ts new file mode 100644 index 00000000000..ccfba01993f --- /dev/null +++ b/ui/app/routes/api/autopilot/sessions/$session_id/actions/interrupt.route.ts @@ -0,0 +1,37 @@ +import type { ActionFunctionArgs } from "react-router"; +import { getAutopilotClient } from "~/utils/tensorzero.server"; +import { logger } from "~/utils/logger"; + +/** + * API route for interrupting an autopilot session. + * + * Route: POST /api/autopilot/sessions/:session_id/actions/interrupt + */ +export async function action({ params, request }: ActionFunctionArgs) { + const sessionId = params.session_id; + if (!sessionId) { + return Response.json( + { success: false, error: "Session ID is required" }, + { status: 400 }, + ); + } + + if (request.method !== "POST") { + return Response.json( + { success: false, error: "Method not allowed" }, + { status: 405 }, + ); + } + + const client = getAutopilotClient(); + + try { + await client.interruptAutopilotSession(sessionId); + return Response.json({ success: true }); + } catch (error) { + logger.error("Failed to interrupt session:", error); + const message = + error instanceof Error ? error.message : "Failed to interrupt session"; + return Response.json({ success: false, error: message }, { status: 500 }); + } +} diff --git a/ui/app/routes/autopilot/sessions/$session_id/route.tsx b/ui/app/routes/autopilot/sessions/$session_id/route.tsx index f93c3c9c7c7..0f4495a99dd 100644 --- a/ui/app/routes/autopilot/sessions/$session_id/route.tsx +++ b/ui/app/routes/autopilot/sessions/$session_id/route.tsx @@ -12,6 +12,7 @@ import { data, isRouteErrorResponse, Link, + useFetcher, useNavigate, type RouteHandle, } from "react-router"; @@ -27,6 +28,7 @@ import { getAutopilotClient } from "~/utils/tensorzero.server"; import { useAutopilotEventStream } from "~/hooks/useAutopilotEventStream"; import type { AutopilotStatus, GatewayEvent } from "~/types/tensorzero"; import { useToast } from "~/hooks/use-toast"; +import { getFeatureFlags } from "~/utils/feature_flags"; // Nil UUID for creating new sessions const NIL_UUID = "00000000-0000-0000-0000-000000000000"; @@ -522,6 +524,10 @@ export default function AutopilotSessionEventsPage({ const { sessionId, eventsData, isNewSession } = loaderData; const navigate = useNavigate(); const { toast } = useToast(); + const interruptFetcher = useFetcher(); + + // Track which session the interrupt was initiated for to prevent cross-session toast + const interruptedSessionRef = useRef(null); // Lift optimistic messages state to parent so ChatInput can work outside Suspense const [optimisticMessages, setOptimisticMessages] = useState< @@ -558,6 +564,47 @@ export default function AutopilotSessionEventsPage({ setAutopilotStatus(status); }, []); + // Handle interrupt session + const handleInterruptSession = useCallback(() => { + interruptedSessionRef.current = sessionId; + interruptFetcher.submit(null, { + method: "POST", + action: `/api/autopilot/sessions/${encodeURIComponent(sessionId)}/actions/interrupt`, + }); + }, [interruptFetcher, sessionId]); + + // Show toast on interrupt result (only if still on the same session) + useEffect(() => { + if (interruptFetcher.state === "idle" && interruptFetcher.data) { + // Only show toast if we're still on the session that was interrupted + if (interruptedSessionRef.current !== sessionId) { + return; + } + const data = interruptFetcher.data as { + success: boolean; + error?: string; + }; + if (data.success) { + toast.success({ + title: "Session interrupted", + description: "The autopilot session has been interrupted.", + }); + } else if (data.error) { + toast.error({ + title: "Failed to interrupt session", + description: data.error, + }); + } + } + }, [interruptFetcher.state, interruptFetcher.data, toast, sessionId]); + + // Interruptible when actively processing (not idle or failed) and feature flag is enabled + const { FF_INTERRUPT_SESSION } = getFeatureFlags(); + const isInterruptible = + FF_INTERRUPT_SESSION && + autopilotStatus.status !== "idle" && + autopilotStatus.status !== "failed"; + // Disable submit unless status is idle or failed const submitDisabled = autopilotStatus.status !== "idle" && autopilotStatus.status !== "failed"; @@ -660,6 +707,9 @@ export default function AutopilotSessionEventsPage({ isNewSession={isNewSession} disabled={isEventsLoading} submitDisabled={submitDisabled} + isInterruptible={isInterruptible} + isInterrupting={interruptFetcher.state !== "idle"} + onInterrupt={handleInterruptSession} /> ); diff --git a/ui/app/utils/feature_flags.ts b/ui/app/utils/feature_flags.ts index f52cc18a291..2ae9e0cff76 100644 --- a/ui/app/utils/feature_flags.ts +++ b/ui/app/utils/feature_flags.ts @@ -7,6 +7,9 @@ interface FeatureFlags { /// from regen-fixtures without trampling existing entries, and then to use the cached /// entries from the normal ui e2e tests FORCE_CACHE_ON: boolean; + /// When set, enables the interrupt session button in autopilot sessions + /// This allows users to interrupt a running autopilot session + FF_INTERRUPT_SESSION: boolean; } /** @@ -15,12 +18,19 @@ interface FeatureFlags { * @returns FeatureFlags */ export function getFeatureFlags(): FeatureFlags { - const envValue = canUseDOM + const forceCacheEnv = canUseDOM ? import.meta.env.VITE_TENSORZERO_FORCE_CACHE_ON : process.env.VITE_TENSORZERO_FORCE_CACHE_ON; - const FORCE_CACHE_ON = envValue === "1"; + const FORCE_CACHE_ON = forceCacheEnv === "1"; + + const interruptSessionEnv = canUseDOM + ? import.meta.env.VITE_TENSORZERO_FF_INTERRUPT_SESSION + : process.env.VITE_TENSORZERO_FF_INTERRUPT_SESSION; + const FF_INTERRUPT_SESSION = interruptSessionEnv === "1"; + return { FORCE_CACHE_ON, + FF_INTERRUPT_SESSION, }; } diff --git a/ui/app/utils/tensorzero/autopilot-client.ts b/ui/app/utils/tensorzero/autopilot-client.ts index 56652166ed9..d7c6692e2c1 100644 --- a/ui/app/utils/tensorzero/autopilot-client.ts +++ b/ui/app/utils/tensorzero/autopilot-client.ts @@ -36,6 +36,18 @@ export class AutopilotClient extends BaseTensorZeroClient { return (await response.json()) as ListSessionsResponse; } + /** + * Interrupts an autopilot session. + */ + async interruptAutopilotSession(sessionId: string): Promise { + const endpoint = `/internal/autopilot/v1/sessions/${encodeURIComponent(sessionId)}/actions/interrupt`; + const response = await this.fetch(endpoint, { method: "POST" }); + if (!response.ok) { + const message = await this.getErrorText(response); + this.handleHttpError({ message, response }); + } + } + /** * Lists events for an autopilot session with optional pagination. */ From 57444fd2d158e10f122ca892e41f195b6ced177c Mon Sep 17 00:00:00 2001 From: Shuyang Li Date: Wed, 28 Jan 2026 11:37:27 -0500 Subject: [PATCH 04/14] Disable fail-fast for live-tests on scheduled CI runs (#5841) The live-tests job was missing the fail-fast configuration, causing it to terminate early when one matrix job fails during scheduled runs. This matches the existing behavior of the client-tests job, which already had this setting to ensure scheduled runs complete all jobs for better visibility into provider flakiness. Co-authored-by: Claude --- .github/workflows/merge-queue.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/merge-queue.yml index c5eade469a8..5a82aef1a1f 100644 --- a/.github/workflows/merge-queue.yml +++ b/.github/workflows/merge-queue.yml @@ -112,6 +112,8 @@ jobs: strategy: matrix: batch_writes: [true, false] + # Don't fail-fast for manual/cron runs, so that we get the full picture of what broke + fail-fast: ${{ github.event_name == 'merge_group' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 From ef298d625ce3b54d34aa9b48bd3f798ac01e1b1e Mon Sep 17 00:00:00 2001 From: Viraj Mehta Date: Wed, 28 Jan 2026 11:45:13 -0500 Subject: [PATCH 05/14] added better defaults for durable worker on client side gateway (#5932) --- gateway/src/main.rs | 9 +++++++-- internal/autopilot-worker/src/worker.rs | 10 +++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/gateway/src/main.rs b/gateway/src/main.rs index e03e18f20d3..40b1a94b9df 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -15,7 +15,7 @@ use tokio::signal; use tokio_stream::wrappers::IntervalStream; use autopilot_worker::{AutopilotWorkerConfig, AutopilotWorkerHandle, spawn_autopilot_worker}; -use durable_tools::EmbeddedClient; +use durable_tools::{EmbeddedClient, WorkerOptions}; use tensorzero_auth::constants::{DEFAULT_ORGANIZATION, DEFAULT_WORKSPACE}; use tensorzero_core::config::{Config, ConfigFileGlob}; use tensorzero_core::db::clickhouse::migration_manager::manual_run_clickhouse_migrations; @@ -564,7 +564,12 @@ async fn spawn_autopilot_worker_if_configured( // TODO: decide how we want to do autopilot config. let default_max_attempts = 5; - let config = AutopilotWorkerConfig::new(pool, t0_client, default_max_attempts); + let worker_options = WorkerOptions { + poll_interval: Duration::from_secs(1), + concurrency: 8, + ..Default::default() + }; + let config = AutopilotWorkerConfig::new(pool, t0_client, default_max_attempts, worker_options); Ok(Some( spawn_autopilot_worker( diff --git a/internal/autopilot-worker/src/worker.rs b/internal/autopilot-worker/src/worker.rs index 56a1ca96ff9..0fb46365e6c 100644 --- a/internal/autopilot-worker/src/worker.rs +++ b/internal/autopilot-worker/src/worker.rs @@ -28,6 +28,8 @@ pub struct AutopilotWorkerConfig { pub t0_client: Arc, /// Default max attempts for a task in the worker pub default_max_attempts: u32, + /// Options for the durable worker (poll interval, claim timeout, etc.) + pub worker_options: WorkerOptions, } impl AutopilotWorkerConfig { @@ -37,6 +39,8 @@ impl AutopilotWorkerConfig { /// /// * `pool` - Database pool for the durable task queue /// * `t0_client` - TensorZero client for inference and autopilot operations + /// * `default_max_attempts` - Default max attempts for a task + /// * `worker_options` - Options for the durable worker (poll interval, etc.) /// /// Environment variables: /// - `TENSORZERO_AUTOPILOT_QUEUE_NAME`: Queue name (default: "autopilot") @@ -44,6 +48,7 @@ impl AutopilotWorkerConfig { pool: PgPool, t0_client: Arc, default_max_attempts: u32, + worker_options: WorkerOptions, ) -> Self { let mut queue_name = autopilot_client::DEFAULT_SPAWN_QUEUE_NAME.to_string(); if cfg!(feature = "e2e_tests") @@ -57,6 +62,7 @@ impl AutopilotWorkerConfig { queue_name, t0_client, default_max_attempts, + worker_options, } } } @@ -64,6 +70,7 @@ impl AutopilotWorkerConfig { /// The autopilot worker that executes client tools. pub struct AutopilotWorker { executor: Arc, + worker_options: WorkerOptions, } impl AutopilotWorker { @@ -83,6 +90,7 @@ impl AutopilotWorker { Ok(Self { executor: Arc::new(executor), + worker_options: config.worker_options, }) } @@ -121,7 +129,7 @@ impl AutopilotWorker { /// migrations not applied, database connection issues). pub async fn start(&self) -> Result { self.executor - .start_worker(WorkerOptions::default()) + .start_worker(self.worker_options.clone()) .await .map_err(Into::into) } From 7c0de0ffebf8527603cfed0130b27f4eafdb0b3a Mon Sep 17 00:00:00 2001 From: Gabriel Bianconi <1275491+GabrielBianconi@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:16:22 -0500 Subject: [PATCH 06/14] Clean up legacy gitkeep (#5934) * Clean up legacy gitkeep * Clean up legacy gitkeep --- ui/fixtures/{s3-fixtures => large-fixtures}/.gitkeep | 0 ui/fixtures/small-fixtures/.gitkeep | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ui/fixtures/{s3-fixtures => large-fixtures}/.gitkeep (100%) create mode 100644 ui/fixtures/small-fixtures/.gitkeep diff --git a/ui/fixtures/s3-fixtures/.gitkeep b/ui/fixtures/large-fixtures/.gitkeep similarity index 100% rename from ui/fixtures/s3-fixtures/.gitkeep rename to ui/fixtures/large-fixtures/.gitkeep diff --git a/ui/fixtures/small-fixtures/.gitkeep b/ui/fixtures/small-fixtures/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d From 2de816c6fdf229e1fc9f21d02c0c434436ae12a8 Mon Sep 17 00:00:00 2001 From: simeonlee <15081451+simeonlee@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:17:21 -0500 Subject: [PATCH 07/14] Add graceful error handling to autopilot session page (#5901) * Add graceful error handling to autopilot session page - Use pattern for initial load errors, showing inline error state while preserving header and chat input - Add EventErrorBoundary to wrap individual events, preventing a single malformed event from crashing the entire chat - Existing error handling (SSE retry, pagination catch) unchanged * Revert cosmetic changes Keep prop name as eventsData and restore original comment * Fix chat input staying disabled on load error When initial load fails, call onError to clear isEventsLoading so ChatInput becomes usable again. * Simplify: remove loading-based input disable ChatInput no longer disabled during loading - users can type immediately. If submit fails (e.g. session doesn't exist), onMessageFailed shows toast. Removes onLoaded callback and isEventsLoading state tracking. * Disable ChatInput when initial load fails Tracks hasLoadError state to prevent users from sending messages when the event stream failed to load (avoids invisible optimistic messages). * Allow typing but block submit on load error * Use disabled instead of submitDisabled for load error * Restore loading state tracking, remove unnecessary comments - Disable ChatInput during loading AND error states - Remove redundant comments (code is self-documenting) - Consistent callback pattern for onLoaded and onError * Restore original comments --- ui/app/components/autopilot/EventStream.tsx | 73 +++++++++++-- .../autopilot/sessions/$session_id/route.tsx | 103 ++++++++++++------ 2 files changed, 138 insertions(+), 38 deletions(-) diff --git a/ui/app/components/autopilot/EventStream.tsx b/ui/app/components/autopilot/EventStream.tsx index 0e1f13505d0..f25cc9eba8f 100644 --- a/ui/app/components/autopilot/EventStream.tsx +++ b/ui/app/components/autopilot/EventStream.tsx @@ -1,6 +1,12 @@ -import { AlertTriangle, ChevronRight, Loader2 } from "lucide-react"; -import { type RefObject, useState } from "react"; +import { + AlertCircle, + AlertTriangle, + ChevronRight, + Loader2, +} from "lucide-react"; +import { Component, type RefObject, useState } from "react"; import { Skeleton } from "~/components/ui/skeleton"; +import { logger } from "~/utils/logger"; import { TableItemTime } from "~/components/ui/TableItems"; import { Tooltip, @@ -318,6 +324,58 @@ function renderEventTitle(event: GatewayEvent) { } } +/** + * Error boundary for individual event items. + * Prevents a single malformed event from crashing the entire chat. + */ +interface EventErrorBoundaryState { + hasError: boolean; +} + +interface EventErrorBoundaryProps { + eventId: string; + children: React.ReactNode; +} + +class EventErrorBoundary extends Component< + EventErrorBoundaryProps, + EventErrorBoundaryState +> { + constructor(props: EventErrorBoundaryProps) { + super(props); + this.state = { hasError: false }; + } + + static getDerivedStateFromError(): EventErrorBoundaryState { + return { hasError: true }; + } + + componentDidCatch(error: Error, errorInfo: React.ErrorInfo) { + logger.error( + `Event ${this.props.eventId} failed to render:`, + error, + errorInfo, + ); + } + + render() { + if (this.state.hasError) { + return ( +
+
+ + + Failed to display event. The event data may be corrupted. + +
+
+ ); + } + + return this.props.children; + } +} + function EventItem({ event, isPending = false, @@ -507,11 +565,12 @@ export default function EventStream({ {isLoadingOlder && } {events.map((event) => ( - + + + ))} {/* Optimistic messages at the end */} diff --git a/ui/app/routes/autopilot/sessions/$session_id/route.tsx b/ui/app/routes/autopilot/sessions/$session_id/route.tsx index 0f4495a99dd..4dd472b7540 100644 --- a/ui/app/routes/autopilot/sessions/$session_id/route.tsx +++ b/ui/app/routes/autopilot/sessions/$session_id/route.tsx @@ -1,7 +1,6 @@ import type { Route } from "./+types/route"; import { Suspense, - use, useCallback, useEffect, useMemo, @@ -9,14 +8,16 @@ import { useState, } from "react"; import { + Await, data, isRouteErrorResponse, Link, + useAsyncError, useFetcher, useNavigate, type RouteHandle, } from "react-router"; -import { Loader2, Plus } from "lucide-react"; +import { AlertCircle, Loader2, Plus } from "lucide-react"; import { PageHeader, Breadcrumbs } from "~/components/layout/PageLayout"; import EventStream, { type OptimisticMessage, @@ -28,6 +29,7 @@ import { getAutopilotClient } from "~/utils/tensorzero.server"; import { useAutopilotEventStream } from "~/hooks/useAutopilotEventStream"; import type { AutopilotStatus, GatewayEvent } from "~/types/tensorzero"; import { useToast } from "~/hooks/use-toast"; +import { SectionErrorNotice } from "~/components/ui/error/ErrorContentPrimitives"; import { getFeatureFlags } from "~/utils/feature_flags"; // Nil UUID for creating new sessions @@ -131,7 +133,32 @@ function EventStreamSkeleton() { ); } -// Main content component that resolves the promise and renders the event stream with SSE +/** + * Error state shown when initial event stream load fails. + * Preserves the chat container layout so the page doesn't completely break. + */ +function EventStreamLoadError({ onError }: { onError: () => void }) { + const error = useAsyncError(); + const message = + error instanceof Error ? error.message : "Failed to load session events"; + + // Notify parent that we're in error state (disables ChatInput) + useEffect(() => { + onError(); + }, [onError]); + + return ( +
+ +
+ ); +} + +// Main content component that renders the event stream with SSE function EventStreamContent({ sessionId, eventsData, @@ -143,7 +170,7 @@ function EventStreamContent({ onStatusChange, }: { sessionId: string; - eventsData: EventsData | Promise; + eventsData: EventsData; isNewSession: boolean; optimisticMessages: OptimisticMessage[]; onOptimisticMessagesChange: (messages: OptimisticMessage[]) => void; @@ -151,13 +178,12 @@ function EventStreamContent({ onLoaded: () => void; onStatusChange: (status: AutopilotStatus) => void; }) { - // Resolve promise (or use direct data for new session) const { events: initialEvents, hasMoreEvents: initialHasMore, pendingToolCalls: initialPendingToolCalls, status: initialStatus, - } = eventsData instanceof Promise ? use(eventsData) : eventsData; + } = eventsData; // Signal that loading is complete (this runs after promise resolves) useEffect(() => { @@ -539,17 +565,6 @@ export default function AutopilotSessionEventsPage({ setOptimisticMessages([]); }, [sessionId]); - // Track if events are still loading (for disabling chat input) - // New sessions have direct data (not a promise), so they're not loading - const [isEventsLoading, setIsEventsLoading] = useState( - !isNewSession && eventsData instanceof Promise, - ); - - // Reset loading state when session changes (useState initial value only applies on first mount) - useEffect(() => { - setIsEventsLoading(!isNewSession && eventsData instanceof Promise); - }, [sessionId, isNewSession, eventsData]); - // Track autopilot status for disabling submit const [autopilotStatus, setAutopilotStatus] = useState({ status: "idle", @@ -609,6 +624,26 @@ export default function AutopilotSessionEventsPage({ const submitDisabled = autopilotStatus.status !== "idle" && autopilotStatus.status !== "failed"; + // Track loading/error state for ChatInput - disabled until events resolve + const [isEventsLoading, setIsEventsLoading] = useState( + !isNewSession && eventsData instanceof Promise, + ); + const [hasLoadError, setHasLoadError] = useState(false); + + useEffect(() => { + setIsEventsLoading(!isNewSession && eventsData instanceof Promise); + setHasLoadError(false); + }, [sessionId, isNewSession, eventsData]); + + const handleEventsLoaded = useCallback(() => { + setIsEventsLoading(false); + }, []); + + const handleLoadError = useCallback(() => { + setIsEventsLoading(false); + setHasLoadError(true); + }, []); + // Ref for scroll container - shared between parent and EventStreamContent const scrollContainerRef = useRef(null); @@ -684,18 +719,24 @@ export default function AutopilotSessionEventsPage({ } /> - }> - setIsEventsLoading(false)} - onStatusChange={handleStatusChange} - /> + }> + } + > + {(resolvedData) => ( + + )} + {/* Chat input - always visible outside Suspense, disabled while loading */} @@ -705,7 +746,7 @@ export default function AutopilotSessionEventsPage({ onMessageFailed={handleMessageFailed} className="mt-4" isNewSession={isNewSession} - disabled={isEventsLoading} + disabled={isEventsLoading || hasLoadError} submitDisabled={submitDisabled} isInterruptible={isInterruptible} isInterrupting={interruptFetcher.state !== "idle"} @@ -727,7 +768,7 @@ function EventStreamContentWrapper({ onStatusChange, }: { sessionId: string; - eventsData: EventsData | Promise; + eventsData: EventsData; isNewSession: boolean; optimisticMessages: OptimisticMessage[]; onOptimisticMessagesChange: (messages: OptimisticMessage[]) => void; From 26c6a9b36938c0248549d590f20702a7ab7f7ec6 Mon Sep 17 00:00:00 2001 From: simeonlee <15081451+simeonlee@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:17:51 -0500 Subject: [PATCH 08/14] Polish autopilot chat input styling (#5881) * Polish autopilot chat input styling - Remove focus ring outline on textarea - Move send button inside input with absolute positioning - Make input fully rounded (pill shape) for single line - Revert to rounded-md when multiline - Center text vertically with adjusted padding - Update placeholder to "Send a message..." * Fix chat input text size to be consistent across screen widths * Add debounced resize handler for textarea height adjustment * Simplify message labels from 'User Message' to 'User' * Shorten breadcrumb label from 'Autopilot Sessions' to 'Autopilot' * Restore focus to textarea after sending message * Restore focus to textarea after sending message * Darken textarea border on focus * Use consistent rounded-md border radius for chat input Remove dynamic pill-to-rectangular border radius transition that was jarring when the textarea expanded. Always use rounded-md for cleaner UX. --- ui/app/components/autopilot/ChatInput.tsx | 49 ++++++++++++++----- ui/app/components/autopilot/EventStream.tsx | 4 +- .../autopilot/sessions/$session_id/route.tsx | 4 +- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/ui/app/components/autopilot/ChatInput.tsx b/ui/app/components/autopilot/ChatInput.tsx index 707b7ed9441..0a907c3f94c 100644 --- a/ui/app/components/autopilot/ChatInput.tsx +++ b/ui/app/components/autopilot/ChatInput.tsx @@ -75,7 +75,7 @@ export function ChatInput({ () => isNewSession ? PLACEHOLDERS[Math.floor(Math.random() * PLACEHOLDERS.length)] - : "Type a message...", + : "Send a message...", [isNewSession], ); @@ -98,6 +98,20 @@ export function ChatInput({ adjustTextareaHeight(); }, [text, adjustTextareaHeight]); + // Debounced resize handler for window width changes + useEffect(() => { + let timeoutId: ReturnType; + const handleResize = () => { + clearTimeout(timeoutId); + timeoutId = setTimeout(adjustTextareaHeight, 100); + }; + window.addEventListener("resize", handleResize); + return () => { + clearTimeout(timeoutId); + window.removeEventListener("resize", handleResize); + }; + }, [adjustTextareaHeight]); + // Handle fetcher response useEffect(() => { if (fetcher.state === "idle" && fetcher.data) { @@ -108,6 +122,11 @@ export function ChatInput({ previousUserMessageEventIdRef.current = data.event_id; setText(""); onMessageSentRef.current?.(data, pendingTextRef.current); + requestAnimationFrame(() => { + requestAnimationFrame(() => { + textareaRef.current?.focus(); + }); + }); } } }, [fetcher.state, fetcher.data]); @@ -149,15 +168,19 @@ export function ChatInput({ ); return ( -
+