From 5fc0ddb8a474bcaebc4a5a46daf6bd824501944d Mon Sep 17 00:00:00 2001 From: simeonlee <15081451+simeonlee@users.noreply.github.com> Date: Tue, 3 Feb 2026 20:50:12 -0500 Subject: [PATCH 1/5] Add Suspense boundary to function detail page (#6078) Add streaming support for the function detail page: - Header shows immediately using route params + config context - Sections load asynchronously with skeleton loading state - Error state handles data fetch failures gracefully - Uses location.key for proper Suspense cache busting on navigation This is the first PR in a series to add progressive loading to the function detail page. --- .../functions/$function_name/route.tsx | 403 +++++++++++++----- 1 file changed, 308 insertions(+), 95 deletions(-) diff --git a/ui/app/routes/observability/functions/$function_name/route.tsx b/ui/app/routes/observability/functions/$function_name/route.tsx index d5d516c1a38..18d0155ff03 100644 --- a/ui/app/routes/observability/functions/$function_name/route.tsx +++ b/ui/app/routes/observability/functions/$function_name/route.tsx @@ -1,6 +1,13 @@ import { countInferencesForFunction } from "~/utils/clickhouse/inference.server"; import type { Route } from "./+types/route"; -import { data, useNavigate, useSearchParams } from "react-router"; +import { + Await, + data, + useAsyncError, + useLocation, + useNavigate, + useSearchParams, +} from "react-router"; import PageButtons from "~/components/utils/PageButtons"; import { getConfig, getFunctionConfig } from "~/utils/config/index.server"; import FunctionInferenceTable from "./FunctionInferenceTable"; @@ -9,7 +16,7 @@ import FunctionSchema from "./FunctionSchema"; import { FunctionExperimentation } from "./FunctionExperimentation"; import { useFunctionConfig } from "~/context/config"; import { MetricSelector } from "~/components/function/variant/MetricSelector"; -import { useMemo } from "react"; +import { Suspense, useMemo } from "react"; import { VariantPerformance } from "~/components/function/variant/VariantPerformance"; import { VariantThroughput } from "~/components/function/variant/VariantThroughput"; import FunctionVariantTable from "./FunctionVariantTable"; @@ -23,34 +30,175 @@ import { } from "~/components/layout/PageLayout"; import { FunctionTypeBadge } from "~/components/function/FunctionSelector"; import { DEFAULT_FUNCTION } from "~/utils/constants"; -import type { TimeWindow } from "~/types/tensorzero"; +import type { FunctionConfig, TimeWindow } from "~/types/tensorzero"; import { getTensorZeroClient } from "~/utils/tensorzero.server"; import { applyPaginationLogic } from "~/utils/pagination"; +import { Skeleton } from "~/components/ui/skeleton"; +import { PageErrorContent } from "~/components/ui/error"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "~/components/ui/table"; -export async function loader({ request, params }: Route.LoaderArgs) { - const { function_name } = params; - const url = new URL(request.url); - const config = await getConfig(); - const beforeInference = url.searchParams.get("beforeInference"); - const afterInference = url.searchParams.get("afterInference"); - const limit = Number(url.searchParams.get("limit")) || 10; - const metric_name = url.searchParams.get("metric_name") || undefined; - const time_granularity = (url.searchParams.get("time_granularity") || - "week") as TimeWindow; - const throughput_time_granularity = (url.searchParams.get( - "throughput_time_granularity", - ) || "week") as TimeWindow; - const feedback_time_granularity = (url.searchParams.get( - "cumulative_feedback_time_granularity", - ) || "week") as TimeWindow; - if (limit > 100) { - throw data("Limit cannot exceed 100", { status: 400 }); - } +export type FunctionDetailData = Awaited< + ReturnType +>; + +function FunctionDetailPageHeader({ + functionName, + functionConfig, +}: { + functionName: string; + functionConfig: FunctionConfig | null; +}) { + return ( + + } + name={functionName} + tag={ + functionConfig ? ( + + ) : undefined + } + > + {functionConfig && } + + ); +} + +function SectionsSkeleton() { + return ( + + + + + + + Name + Type + Weight + Inferences + + + + {[1, 2, 3].map((i) => ( + + + + + + + + + + + + + + + ))} + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + ID + Variant + Time + + + + {[1, 2, 3, 4, 5].map((i) => ( + + + + + + + + + + + + ))} + +
+ + + ); +} + +function SectionsErrorState() { + const error = useAsyncError(); + return ( + + + + + + ); +} + +type FetchParams = { + function_name: string; + function_config: FunctionConfig; + config: Awaited>; + beforeInference: string | null; + afterInference: string | null; + limit: number; + metric_name: string | undefined; + time_granularity: TimeWindow; + throughput_time_granularity: TimeWindow; + feedback_time_granularity: TimeWindow; +}; + +async function fetchFunctionDetailData(params: FetchParams) { + const { + function_name, + function_config, + config, + beforeInference, + afterInference, + limit, + metric_name, + time_granularity, + throughput_time_granularity, + feedback_time_granularity, + } = params; - const function_config = await getFunctionConfig(function_name, config); - if (!function_config) { - throw data(`Function ${function_name} not found`, { status: 404 }); - } const client = getTensorZeroClient(); const inferencePromise = client.listInferenceMetadata({ function_name, @@ -201,9 +349,58 @@ export async function loader({ request, params }: Route.LoaderArgs) { }; } -export default function InferencesPage({ loaderData }: Route.ComponentProps) { - const { +export async function loader({ request, params }: Route.LoaderArgs) { + const { function_name } = params; + const url = new URL(request.url); + const config = await getConfig(); + const beforeInference = url.searchParams.get("beforeInference"); + const afterInference = url.searchParams.get("afterInference"); + const limit = Number(url.searchParams.get("limit")) || 10; + const metric_name = url.searchParams.get("metric_name") || undefined; + const time_granularity = (url.searchParams.get("time_granularity") || + "week") as TimeWindow; + const throughput_time_granularity = (url.searchParams.get( + "throughput_time_granularity", + ) || "week") as TimeWindow; + const feedback_time_granularity = (url.searchParams.get( + "cumulative_feedback_time_granularity", + ) || "week") as TimeWindow; + if (limit > 100) { + throw data("Limit cannot exceed 100", { status: 400 }); + } + + const function_config = await getFunctionConfig(function_name, config); + if (!function_config) { + throw data(`Function ${function_name} not found`, { status: 404 }); + } + + return { function_name, + functionDetailData: fetchFunctionDetailData({ + function_name, + function_config, + config, + beforeInference, + afterInference, + limit, + metric_name, + time_granularity, + throughput_time_granularity, + feedback_time_granularity, + }), + }; +} + +function SectionsContent({ + data, + functionName, + functionConfig, +}: { + data: FunctionDetailData; + functionName: string; + functionConfig: FunctionConfig; +}) { + const { inferences, hasNextInferencePage, hasPreviousInferencePage, @@ -214,14 +411,10 @@ export default function InferencesPage({ loaderData }: Route.ComponentProps) { variant_counts, feedback_timeseries, variant_sampling_probabilities, - } = loaderData; + } = data; const navigate = useNavigate(); const [searchParams] = useSearchParams(); - const function_config = useFunctionConfig(function_name); - if (!function_config) { - throw data(`Function ${function_name} not found`, { status: 404 }); - } // Only get top/bottom inferences if array is not empty const topInference = inferences.length > 0 ? inferences[0] : null; @@ -262,78 +455,98 @@ export default function InferencesPage({ loaderData }: Route.ComponentProps) { ); return ( - - - } - name={function_name} - tag={} - > - - + + + + + - + {functionName !== DEFAULT_FUNCTION && ( - - + + )} - {function_name !== DEFAULT_FUNCTION && ( - - - - + + + + + + + + + {variant_performances && ( + )} + - - - - + + + + - - - - {variant_performances && ( - - )} - + + + + + + + ); +} - - - - +export default function FunctionDetailPage({ + loaderData, +}: Route.ComponentProps) { + const { function_name, functionDetailData } = loaderData; + const location = useLocation(); + const function_config = useFunctionConfig(function_name); - - - - - - + if (!function_config) { + throw data(`Function ${function_name} not found`, { status: 404 }); + } + + return ( + + + + }> + } + > + {(data) => ( + + )} + + ); } From 9a4c8f3bc2a0fd5874eaf16f62acdd7c90824348 Mon Sep 17 00:00:00 2001 From: Shuyang Li Date: Tue, 3 Feb 2026 21:24:34 -0500 Subject: [PATCH 2/5] Query building unit tests for Postgres (#6004) --- tensorzero-core/src/db/postgres/feedback.rs | 1485 +++++++++++++---- .../src/db/postgres/inference_queries.rs | 1115 +++++++++++-- 2 files changed, 2143 insertions(+), 457 deletions(-) diff --git a/tensorzero-core/src/db/postgres/feedback.rs b/tensorzero-core/src/db/postgres/feedback.rs index ae742b053ad..e39b0a6f93c 100644 --- a/tensorzero-core/src/db/postgres/feedback.rs +++ b/tensorzero-core/src/db/postgres/feedback.rs @@ -27,60 +27,53 @@ use crate::function::FunctionConfig; use super::PostgresConnectionInfo; // ===================================================================== -// FeedbackQueries trait implementation +// Query builder functions (for unit testing) // ===================================================================== -#[async_trait] -impl FeedbackQueries for PostgresConnectionInfo { - async fn get_feedback_by_variant( - &self, - metric_name: &str, - function_name: &str, - variant_names: Option<&Vec>, - ) -> Result, Error> { - let pool = self.get_pool_result()?; - // Handle empty variant_names - if let Some(names) = variant_names - && names.is_empty() - { - return Ok(vec![]); - } - - let mut qb = QueryBuilder::new( - r" +/// Builds a query to get feedback statistics grouped by variant. +/// +/// If `variant_names` is empty, no variant filter is applied (all variants are included). +/// If `variant_names` is non-empty, only the specified variants are included. +fn build_feedback_by_variant_query( + metric_name: &str, + function_name: &str, + variant_names: &[String], +) -> QueryBuilder { + let mut qb = QueryBuilder::new( + r" WITH feedback AS ( SELECT target_id, value::INT::DOUBLE PRECISION as value FROM tensorzero.boolean_metric_feedback WHERE metric_name = ", - ); - qb.push_bind(metric_name); - qb.push( - r" + ); + qb.push_bind(metric_name); + qb.push( + r" UNION ALL SELECT target_id, value FROM tensorzero.float_metric_feedback WHERE metric_name = ", - ); - qb.push_bind(metric_name); - qb.push( - r" + ); + qb.push_bind(metric_name); + qb.push( + r" ), inferences AS ( SELECT id, variant_name FROM tensorzero.chat_inferences WHERE function_name = ", - ); - qb.push_bind(function_name); - qb.push( - r" + ); + qb.push_bind(function_name); + qb.push( + r" UNION ALL SELECT id, variant_name FROM tensorzero.json_inferences WHERE function_name = ", - ); - qb.push_bind(function_name); - qb.push( - r" + ); + qb.push_bind(function_name); + qb.push( + r" ) SELECT i.variant_name, @@ -90,15 +83,309 @@ impl FeedbackQueries for PostgresConnectionInfo { FROM feedback f JOIN inferences i ON f.target_id = i.id WHERE 1=1", - ); + ); + + if !variant_names.is_empty() { + qb.push(" AND i.variant_name = ANY("); + qb.push_bind(variant_names); + qb.push(")"); + } + + qb.push(" GROUP BY i.variant_name"); + + qb +} + +/// Builds a query for cumulative feedback time series. +/// +/// The `interval_str` parameter must be a trusted value from `TimeWindow::to_postgres_time_unit()`. +/// +/// If `variant_names` is empty, no variant filter is applied (all variants are included). +/// If `variant_names` is non-empty, only the specified variants are included. +fn build_cumulative_feedback_timeseries_query( + metric_name: &str, + function_name: &str, + variant_names: &[String], + interval_str: &str, + max_periods: i32, +) -> QueryBuilder { + let mut qb = QueryBuilder::new("WITH feedback_with_variant AS (SELECT date_trunc('"); + qb.push(interval_str); + qb.push("', f.created_at) + INTERVAL '1 "); + qb.push(interval_str); + qb.push( + r"' AS period_end, + i.variant_name, + f.value + FROM ( + SELECT target_id, value::INT::DOUBLE PRECISION as value, created_at + FROM tensorzero.boolean_metric_feedback WHERE metric_name = ", + ); + qb.push_bind(metric_name); + qb.push( + r" + UNION ALL + SELECT target_id, value, created_at + FROM tensorzero.float_metric_feedback WHERE metric_name = ", + ); + qb.push_bind(metric_name); + qb.push( + r" + ) f + JOIN ( + SELECT id, variant_name FROM tensorzero.chat_inferences WHERE function_name = ", + ); + qb.push_bind(function_name); + qb.push( + r" + UNION ALL + SELECT id, variant_name FROM tensorzero.json_inferences WHERE function_name = ", + ); + qb.push_bind(function_name); + qb.push(") i ON f.target_id = i.id WHERE 1=1"); + + if !variant_names.is_empty() { + qb.push(" AND i.variant_name = ANY("); + qb.push_bind(variant_names); + qb.push(")"); + } + + qb.push( + r" + ), + period_stats AS ( + SELECT + period_end, + variant_name, + COUNT(*) as period_count, + SUM(value) as period_sum, + SUM(value * value) as period_sum_sq + FROM feedback_with_variant + GROUP BY period_end, variant_name + ), + cumulative_stats AS ( + SELECT + period_end, + variant_name, + SUM(period_count) OVER w as count, + SUM(period_sum) OVER w as sum_val, + SUM(period_sum_sq) OVER w as sum_sq + FROM period_stats + WINDOW w AS (PARTITION BY variant_name ORDER BY period_end ROWS UNBOUNDED PRECEDING) + ), + max_period AS ( + SELECT MAX(period_end) as max_end FROM cumulative_stats + ) + SELECT + period_end, + variant_name, + (sum_val / count)::REAL as mean, + CASE WHEN count > 1 THEN ((sum_sq - sum_val * sum_val / count) / (count - 1))::REAL END as variance, + count::BIGINT + FROM cumulative_stats + WHERE period_end >= (SELECT max_end FROM max_period) - (", + ); + qb.push_bind(max_periods); + qb.push(" || ' "); + qb.push(interval_str); + qb.push("')::INTERVAL ORDER BY period_end, variant_name"); + + qb +} + +/// Builds a query to get metrics that have feedback for a function. +fn build_metrics_with_feedback_query( + function_name: &str, + table: &str, + variant_name: Option<&str>, +) -> QueryBuilder { + let mut qb = QueryBuilder::new("WITH inference_ids AS (SELECT id FROM "); + qb.push(table); + qb.push(" WHERE function_name = "); + qb.push_bind(function_name); + + if let Some(vn) = variant_name { + qb.push(" AND variant_name = "); + qb.push_bind(vn); + } + + qb.push( + r") + SELECT metric_name, feedback_count::INT FROM ( + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.boolean_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.float_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT 'demonstration' as metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.demonstration_feedback + WHERE inference_id IN (SELECT id FROM inference_ids) + ) combined + WHERE feedback_count > 0 + ORDER BY metric_name", + ); + + qb +} + +/// Parameters for building a variant performances query. +struct VariantPerformancesQueryParams<'a> { + metric_name: &'a str, + function_name: &'a str, + inference_table: &'a str, + metric_table: &'a str, + value_cast: &'a str, + time_bucket_expr: &'a str, + metric_level: MetricConfigLevel, + variant_name: Option<&'a str>, +} + +/// Builds a query for variant performances. +fn build_variant_performances_query( + params: &VariantPerformancesQueryParams<'_>, +) -> QueryBuilder { + match params.metric_level { + MetricConfigLevel::Episode => build_variant_performances_episode_query(params), + MetricConfigLevel::Inference => build_variant_performances_inference_query(params), + } +} + +fn build_variant_performances_episode_query( + params: &VariantPerformancesQueryParams<'_>, +) -> QueryBuilder { + let mut qb = QueryBuilder::new( + "WITH feedback AS ( + SELECT DISTINCT ON (target_id) target_id, ", + ); + qb.push(params.value_cast); + qb.push( + " as value + FROM ", + ); + qb.push(params.metric_table); + qb.push(" WHERE metric_name = "); + qb.push_bind(params.metric_name); + qb.push( + " ORDER BY target_id, created_at DESC + ), + per_episode AS ( + SELECT ", + ); + qb.push(params.time_bucket_expr); + qb.push( + " AS period_start, + i.variant_name, + i.episode_id, + f.value + FROM ", + ); + qb.push(params.inference_table); + qb.push( + " i + JOIN feedback f ON f.target_id = i.episode_id + WHERE i.function_name = ", + ); + qb.push_bind(params.function_name); - if let Some(names) = variant_names { - qb.push(" AND i.variant_name = ANY("); - qb.push_bind(names); - qb.push(")"); + if let Some(vn) = params.variant_name { + qb.push(" AND i.variant_name = "); + qb.push_bind(vn); + } + + qb.push( + " GROUP BY period_start, i.variant_name, i.episode_id, f.value + ) + SELECT + period_start, + variant_name, + COUNT(*)::INT as count, + AVG(value) as avg_metric, + STDDEV_SAMP(value) as stdev, + CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(value) / SQRT(COUNT(*))) END as ci_error + FROM per_episode + GROUP BY period_start, variant_name + ORDER BY period_start ASC, variant_name ASC", + ); + + qb +} + +fn build_variant_performances_inference_query( + params: &VariantPerformancesQueryParams<'_>, +) -> QueryBuilder { + let mut qb = QueryBuilder::new( + "WITH feedback AS ( + SELECT DISTINCT ON (target_id) target_id, ", + ); + qb.push(params.value_cast); + qb.push( + " as value, created_at + FROM ", + ); + qb.push(params.metric_table); + qb.push(" WHERE metric_name = "); + qb.push_bind(params.metric_name); + qb.push( + " ORDER BY target_id, created_at DESC + ) + SELECT ", + ); + qb.push(params.time_bucket_expr); + qb.push( + " as period_start, + i.variant_name, + COUNT(*)::INT as count, + AVG(f.value) as avg_metric, + STDDEV_SAMP(f.value) as stdev, + CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(f.value) / SQRT(COUNT(*))) END as ci_error + FROM ", + ); + qb.push(params.inference_table); + qb.push( + " i + JOIN feedback f ON f.target_id = i.id + WHERE i.function_name = ", + ); + qb.push_bind(params.function_name); + + if let Some(vn) = params.variant_name { + qb.push(" AND i.variant_name = "); + qb.push_bind(vn); + } + + qb.push(" GROUP BY period_start, i.variant_name ORDER BY period_start ASC, i.variant_name ASC"); + + qb +} + +// ===================================================================== +// FeedbackQueries trait implementation +// ===================================================================== + +#[async_trait] +impl FeedbackQueries for PostgresConnectionInfo { + async fn get_feedback_by_variant( + &self, + metric_name: &str, + function_name: &str, + variant_names: Option<&Vec>, + ) -> Result, Error> { + let pool = self.get_pool_result()?; + // Handle empty variant_names - return early to avoid unnecessary query + if let Some(names) = variant_names + && names.is_empty() + { + return Ok(vec![]); } - qb.push(" GROUP BY i.variant_name"); + // Pass empty slice for None, or the actual slice for Some + let variant_slice = variant_names.map(|v| v.as_slice()).unwrap_or(&[]); + let mut qb = build_feedback_by_variant_query(metric_name, function_name, variant_slice); let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; @@ -144,91 +431,15 @@ impl FeedbackQueries for PostgresConnectionInfo { let interval_str = time_window.to_postgres_time_unit(); - // For this complex query with date_trunc expressions that need the interval as a literal, - // we need to build separate queries for each time window type. - // The interval string is trusted (from our enum match above). - // Build the base CTE query using QueryBuilder for value bindings - // Note: interval_str comes from a trusted enum match, so it's safe to include directly - let mut qb = QueryBuilder::new("WITH feedback_with_variant AS (SELECT date_trunc('"); - qb.push(interval_str); - qb.push("', f.created_at) + INTERVAL '1 "); - qb.push(interval_str); - qb.push( - r"' AS period_end, - i.variant_name, - f.value - FROM ( - SELECT target_id, value::INT::DOUBLE PRECISION as value, created_at - FROM tensorzero.boolean_metric_feedback WHERE metric_name = ", + // Pass empty slice for None, or the actual slice for Some + let variant_slice = variant_names.as_deref().unwrap_or(&[]); + let mut qb = build_cumulative_feedback_timeseries_query( + &metric_name, + &function_name, + variant_slice, + interval_str, + max_periods as i32, ); - qb.push_bind(&metric_name); - qb.push( - r" - UNION ALL - SELECT target_id, value, created_at - FROM tensorzero.float_metric_feedback WHERE metric_name = ", - ); - qb.push_bind(&metric_name); - qb.push( - r" - ) f - JOIN ( - SELECT id, variant_name FROM tensorzero.chat_inferences WHERE function_name = ", - ); - qb.push_bind(&function_name); - qb.push( - r" - UNION ALL - SELECT id, variant_name FROM tensorzero.json_inferences WHERE function_name = ", - ); - qb.push_bind(&function_name); - qb.push(") i ON f.target_id = i.id WHERE 1=1"); - - if let Some(names) = variant_names { - qb.push(" AND i.variant_name = ANY("); - qb.push_bind(names.as_slice()); - qb.push(")"); - } - - qb.push( - r" - ), - period_stats AS ( - SELECT - period_end, - variant_name, - COUNT(*) as period_count, - SUM(value) as period_sum, - SUM(value * value) as period_sum_sq - FROM feedback_with_variant - GROUP BY period_end, variant_name - ), - cumulative_stats AS ( - SELECT - period_end, - variant_name, - SUM(period_count) OVER w as count, - SUM(period_sum) OVER w as sum_val, - SUM(period_sum_sq) OVER w as sum_sq - FROM period_stats - WINDOW w AS (PARTITION BY variant_name ORDER BY period_end ROWS UNBOUNDED PRECEDING) - ), - max_period AS ( - SELECT MAX(period_end) as max_end FROM cumulative_stats - ) - SELECT - period_end, - variant_name, - (sum_val / count)::REAL as mean, - CASE WHEN count > 1 THEN ((sum_sq - sum_val * sum_val / count) / (count - 1))::REAL END as variance, - count::BIGINT - FROM cumulative_stats - WHERE period_end >= (SELECT max_end FROM max_period) - (", - ); - qb.push_bind(max_periods as i32); - qb.push(" || ' "); - qb.push(interval_str); - qb.push("')::INTERVAL ORDER BY period_end, variant_name"); let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; @@ -387,37 +598,7 @@ impl FeedbackQueries for PostgresConnectionInfo { let pool = self.get_pool_result()?; let table = function_config.postgres_table_name(); - // Build the query using QueryBuilder - let mut qb = QueryBuilder::new("WITH inference_ids AS (SELECT id FROM "); - qb.push(table); - qb.push(" WHERE function_name = "); - qb.push_bind(function_name); - - if let Some(vn) = variant_name { - qb.push(" AND variant_name = "); - qb.push_bind(vn); - } - - qb.push( - r") - SELECT metric_name, feedback_count::INT FROM ( - SELECT metric_name, COUNT(*)::INT as feedback_count - FROM tensorzero.boolean_metric_feedback - WHERE target_id IN (SELECT id FROM inference_ids) - GROUP BY metric_name - UNION ALL - SELECT metric_name, COUNT(*)::INT as feedback_count - FROM tensorzero.float_metric_feedback - WHERE target_id IN (SELECT id FROM inference_ids) - GROUP BY metric_name - UNION ALL - SELECT 'demonstration' as metric_name, COUNT(*)::INT as feedback_count - FROM tensorzero.demonstration_feedback - WHERE inference_id IN (SELECT id FROM inference_ids) - ) combined - WHERE feedback_count > 0 - ORDER BY metric_name", - ); + let mut qb = build_metrics_with_feedback_query(function_name, table, variant_name); let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; @@ -488,7 +669,7 @@ impl FeedbackQueries for PostgresConnectionInfo { }; // Build time bucket expression based on time window - let time_bucket_i = match params.time_window { + let time_bucket_expr = match params.time_window { TimeWindow::Minute => "date_trunc('minute', i.created_at)", TimeWindow::Hour => "date_trunc('hour', i.created_at)", TimeWindow::Day => "date_trunc('day', i.created_at)", @@ -497,124 +678,20 @@ impl FeedbackQueries for PostgresConnectionInfo { TimeWindow::Cumulative => "'1970-01-01 00:00:00'::TIMESTAMPTZ", }; - // For both episode-level and inference-level metrics, we deduplicate feedback - // by target_id to use only the latest feedback when multiple feedback entries - // exist for the same target. This matches the ClickHouse implementation. - // - // For episode-level metrics, we additionally group by episode_id to avoid - // counting the same episode multiple times when there are multiple inferences - // per episode. - let rows = match metric_level { - MetricConfigLevel::Episode => { - // Episode-level: deduplicate by grouping on episode_id in subquery - let mut qb = QueryBuilder::new( - "WITH feedback AS ( - SELECT DISTINCT ON (target_id) target_id, ", - ); - qb.push(value_cast); - qb.push( - " as value - FROM ", - ); - qb.push(metric_table); - qb.push(" WHERE metric_name = "); - qb.push_bind(params.metric_name); - qb.push( - " ORDER BY target_id, created_at DESC - ), - per_episode AS ( - SELECT ", - ); - qb.push(time_bucket_i); - qb.push( - " AS period_start, - i.variant_name, - i.episode_id, - f.value - FROM ", - ); - qb.push(inference_table); - qb.push( - " i - JOIN feedback f ON f.target_id = i.episode_id - WHERE i.function_name = ", - ); - qb.push_bind(params.function_name); - - if let Some(vn) = params.variant_name { - qb.push(" AND i.variant_name = "); - qb.push_bind(vn); - } - - qb.push( - " GROUP BY period_start, i.variant_name, i.episode_id, f.value - ) - SELECT - period_start, - variant_name, - COUNT(*)::INT as count, - AVG(value) as avg_metric, - STDDEV_SAMP(value) as stdev, - CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(value) / SQRT(COUNT(*))) END as ci_error - FROM per_episode - GROUP BY period_start, variant_name - ORDER BY period_start ASC, variant_name ASC", - ); - - qb.build().fetch_all(pool).await.map_err(Error::from)? - } - MetricConfigLevel::Inference => { - // Inference-level: SELECT DISTINCT ON (target_id) keeps the latest feedback by target_id. - // (First result with SELECT DISTINCT ON target_id, ORDER BY created_at DESC) - // This matches ClickHouse implementation. - let mut qb = QueryBuilder::new( - "WITH feedback AS ( - SELECT DISTINCT ON (target_id) target_id, ", - ); - qb.push(value_cast); - qb.push( - " as value, created_at - FROM ", - ); - qb.push(metric_table); - qb.push(" WHERE metric_name = "); - qb.push_bind(params.metric_name); - qb.push( - " ORDER BY target_id, created_at DESC - ) - SELECT ", - ); - qb.push(time_bucket_i); - qb.push( - " as period_start, - i.variant_name, - COUNT(*)::INT as count, - AVG(f.value) as avg_metric, - STDDEV_SAMP(f.value) as stdev, - CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(f.value) / SQRT(COUNT(*))) END as ci_error - FROM ", - ); - qb.push(inference_table); - qb.push( - " i - JOIN feedback f ON f.target_id = i.id - WHERE i.function_name = ", - ); - qb.push_bind(params.function_name); - - if let Some(vn) = params.variant_name { - qb.push(" AND i.variant_name = "); - qb.push_bind(vn); - } - - qb.push( - " GROUP BY period_start, i.variant_name ORDER BY period_start ASC, i.variant_name ASC", - ); - - qb.build().fetch_all(pool).await.map_err(Error::from)? - } + let query_params = VariantPerformancesQueryParams { + metric_name: params.metric_name, + function_name: params.function_name, + inference_table, + metric_table, + value_cast, + time_bucket_expr, + metric_level, + variant_name: params.variant_name, }; + let mut qb = build_variant_performances_query(&query_params); + let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; + Ok(rows .into_iter() .map(|row| VariantPerformanceRow { @@ -795,13 +872,13 @@ impl FeedbackQueries for PostgresConnectionInfo { } } -async fn query_boolean_feedback( - pool: &PgPool, +/// Builds a query to fetch boolean metric feedback for a target. +fn build_boolean_feedback_query( target_id: Uuid, before: Option, after: Option, limit: i64, -) -> Result, Error> { +) -> Result, Error> { let mut qb = QueryBuilder::new( "SELECT id, target_id, metric_name, value, tags, created_at FROM tensorzero.boolean_metric_feedback WHERE target_id = ", ); @@ -812,6 +889,18 @@ async fn query_boolean_feedback( qb.push(" LIMIT "); qb.push_bind(limit); + Ok(qb) +} + +async fn query_boolean_feedback( + pool: &PgPool, + target_id: Uuid, + before: Option, + after: Option, + limit: i64, +) -> Result, Error> { + let mut qb = build_boolean_feedback_query(target_id, before, after, limit)?; + let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; rows.into_iter() @@ -831,13 +920,13 @@ async fn query_boolean_feedback( .collect() } -async fn query_float_feedback( - pool: &PgPool, +/// Builds a query to fetch float metric feedback for a target. +fn build_float_feedback_query( target_id: Uuid, before: Option, after: Option, limit: i64, -) -> Result, Error> { +) -> Result, Error> { let mut qb = QueryBuilder::new( "SELECT id, target_id, metric_name, value, tags, created_at FROM tensorzero.float_metric_feedback WHERE target_id = ", ); @@ -848,6 +937,18 @@ async fn query_float_feedback( qb.push(" LIMIT "); qb.push_bind(limit); + Ok(qb) +} + +async fn query_float_feedback( + pool: &PgPool, + target_id: Uuid, + before: Option, + after: Option, + limit: i64, +) -> Result, Error> { + let mut qb = build_float_feedback_query(target_id, before, after, limit)?; + let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; rows.into_iter() @@ -867,13 +968,13 @@ async fn query_float_feedback( .collect() } -async fn query_comment_feedback( - pool: &PgPool, +/// Builds a query to fetch comment feedback for a target. +fn build_comment_feedback_query( target_id: Uuid, before: Option, after: Option, limit: i64, -) -> Result, Error> { +) -> Result, Error> { let mut qb = QueryBuilder::new( "SELECT id, target_id, target_type, value, tags, created_at FROM tensorzero.comment_feedback WHERE target_id = ", ); @@ -884,6 +985,18 @@ async fn query_comment_feedback( qb.push(" LIMIT "); qb.push_bind(limit); + Ok(qb) +} + +async fn query_comment_feedback( + pool: &PgPool, + target_id: Uuid, + before: Option, + after: Option, + limit: i64, +) -> Result, Error> { + let mut qb = build_comment_feedback_query(target_id, before, after, limit)?; + let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; rows.into_iter() @@ -909,13 +1022,13 @@ async fn query_comment_feedback( .collect() } -async fn query_demonstration_feedback( - pool: &PgPool, +/// Builds a query to fetch demonstration feedback for an inference. +fn build_demonstration_feedback_query( inference_id: Uuid, before: Option, after: Option, limit: i64, -) -> Result, Error> { +) -> Result, Error> { let mut qb = QueryBuilder::new( "SELECT id, inference_id, value, tags, created_at FROM tensorzero.demonstration_feedback WHERE inference_id = ", ); @@ -926,6 +1039,18 @@ async fn query_demonstration_feedback( qb.push(" LIMIT "); qb.push_bind(limit); + Ok(qb) +} + +async fn query_demonstration_feedback( + pool: &PgPool, + inference_id: Uuid, + before: Option, + after: Option, + limit: i64, +) -> Result, Error> { + let mut qb = build_demonstration_feedback_query(inference_id, before, after, limit)?; + let rows = qb.build().fetch_all(pool).await.map_err(Error::from)?; rows.into_iter() @@ -1065,3 +1190,793 @@ async fn query_demonstration_bounds(pool: &PgPool, target_id: Uuid) -> Result $2 ORDER BY id ASC LIMIT $3 + ", + ); + } + + // ===== build_float_feedback_query tests ===== + + #[test] + fn test_build_float_feedback_query_no_pagination() { + let target_id = Uuid::now_v7(); + let qb = build_float_feedback_query(target_id, None, None, 100).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, target_id, metric_name, value, tags, created_at + FROM tensorzero.float_metric_feedback + WHERE target_id = $1 ORDER BY id DESC LIMIT $2 + ", + ); + } + + #[test] + fn test_build_float_feedback_query_before_cursor() { + let target_id = Uuid::now_v7(); + let before_id = Uuid::now_v7(); + let qb = build_float_feedback_query(target_id, Some(before_id), None, 50).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, target_id, metric_name, value, tags, created_at + FROM tensorzero.float_metric_feedback + WHERE target_id = $1 AND id < $2 ORDER BY id DESC LIMIT $3 + ", + ); + } + + #[test] + fn test_build_float_feedback_query_after_cursor() { + let target_id = Uuid::now_v7(); + let after_id = Uuid::now_v7(); + let qb = build_float_feedback_query(target_id, None, Some(after_id), 50).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, target_id, metric_name, value, tags, created_at + FROM tensorzero.float_metric_feedback + WHERE target_id = $1 AND id > $2 ORDER BY id ASC LIMIT $3 + ", + ); + } + + // ===== build_comment_feedback_query tests ===== + + #[test] + fn test_build_comment_feedback_query_no_pagination() { + let target_id = Uuid::now_v7(); + let qb = build_comment_feedback_query(target_id, None, None, 100).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, target_id, target_type, value, tags, created_at + FROM tensorzero.comment_feedback + WHERE target_id = $1 ORDER BY id DESC LIMIT $2 + ", + ); + } + + #[test] + fn test_build_comment_feedback_query_before_cursor() { + let target_id = Uuid::now_v7(); + let before_id = Uuid::now_v7(); + let qb = build_comment_feedback_query(target_id, Some(before_id), None, 50).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, target_id, target_type, value, tags, created_at + FROM tensorzero.comment_feedback + WHERE target_id = $1 AND id < $2 ORDER BY id DESC LIMIT $3 + ", + ); + } + + #[test] + fn test_build_comment_feedback_query_after_cursor() { + let target_id = Uuid::now_v7(); + let after_id = Uuid::now_v7(); + let qb = build_comment_feedback_query(target_id, None, Some(after_id), 50).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, target_id, target_type, value, tags, created_at + FROM tensorzero.comment_feedback + WHERE target_id = $1 AND id > $2 ORDER BY id ASC LIMIT $3 + ", + ); + } + + // ===== build_demonstration_feedback_query tests ===== + + #[test] + fn test_build_demonstration_feedback_query_no_pagination() { + let inference_id = Uuid::now_v7(); + let qb = build_demonstration_feedback_query(inference_id, None, None, 100).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, inference_id, value, tags, created_at + FROM tensorzero.demonstration_feedback + WHERE inference_id = $1 ORDER BY id DESC LIMIT $2 + ", + ); + } + + #[test] + fn test_build_demonstration_feedback_query_before_cursor() { + let inference_id = Uuid::now_v7(); + let before_id = Uuid::now_v7(); + let qb = + build_demonstration_feedback_query(inference_id, Some(before_id), None, 50).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, inference_id, value, tags, created_at + FROM tensorzero.demonstration_feedback + WHERE inference_id = $1 AND id < $2 ORDER BY id DESC LIMIT $3 + ", + ); + } + + #[test] + fn test_build_demonstration_feedback_query_after_cursor() { + let inference_id = Uuid::now_v7(); + let after_id = Uuid::now_v7(); + let qb = + build_demonstration_feedback_query(inference_id, None, Some(after_id), 50).unwrap(); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT id, inference_id, value, tags, created_at + FROM tensorzero.demonstration_feedback + WHERE inference_id = $1 AND id > $2 ORDER BY id ASC LIMIT $3 + ", + ); + } + + // ===== build_feedback_by_variant_query tests ===== + + #[test] + fn test_build_feedback_by_variant_query_no_variant_filter() { + let qb = build_feedback_by_variant_query("my_metric", "my_function", &[]); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback AS ( + SELECT target_id, value::INT::DOUBLE PRECISION as value + FROM tensorzero.boolean_metric_feedback + WHERE metric_name = $1 + UNION ALL + SELECT target_id, value + FROM tensorzero.float_metric_feedback + WHERE metric_name = $2 + ), + inferences AS ( + SELECT id, variant_name + FROM tensorzero.chat_inferences + WHERE function_name = $3 + UNION ALL + SELECT id, variant_name + FROM tensorzero.json_inferences + WHERE function_name = $4 + ) + SELECT + i.variant_name, + AVG(f.value)::REAL as mean, + VAR_SAMP(f.value)::REAL as variance, + COUNT(*)::BIGINT as count + FROM feedback f + JOIN inferences i ON f.target_id = i.id + WHERE 1=1 GROUP BY i.variant_name + ", + ); + } + + #[test] + fn test_build_feedback_by_variant_query_with_variant_filter() { + let variant_names = vec!["variant_a".to_string(), "variant_b".to_string()]; + let qb = build_feedback_by_variant_query("my_metric", "my_function", &variant_names); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback AS ( + SELECT target_id, value::INT::DOUBLE PRECISION as value + FROM tensorzero.boolean_metric_feedback + WHERE metric_name = $1 + UNION ALL + SELECT target_id, value + FROM tensorzero.float_metric_feedback + WHERE metric_name = $2 + ), + inferences AS ( + SELECT id, variant_name + FROM tensorzero.chat_inferences + WHERE function_name = $3 + UNION ALL + SELECT id, variant_name + FROM tensorzero.json_inferences + WHERE function_name = $4 + ) + SELECT + i.variant_name, + AVG(f.value)::REAL as mean, + VAR_SAMP(f.value)::REAL as variance, + COUNT(*)::BIGINT as count + FROM feedback f + JOIN inferences i ON f.target_id = i.id + WHERE 1=1 AND i.variant_name = ANY($5) GROUP BY i.variant_name + ", + ); + } + + // ===== build_metrics_with_feedback_query tests ===== + + #[test] + fn test_build_metrics_with_feedback_query_chat_no_variant() { + let qb = + build_metrics_with_feedback_query("my_function", "tensorzero.chat_inferences", None); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH inference_ids AS (SELECT id FROM tensorzero.chat_inferences WHERE function_name = $1) + SELECT metric_name, feedback_count::INT FROM ( + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.boolean_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.float_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT 'demonstration' as metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.demonstration_feedback + WHERE inference_id IN (SELECT id FROM inference_ids) + ) combined + WHERE feedback_count > 0 + ORDER BY metric_name + ", + ); + } + + #[test] + fn test_build_metrics_with_feedback_query_chat_with_variant() { + let qb = build_metrics_with_feedback_query( + "my_function", + "tensorzero.chat_inferences", + Some("my_variant"), + ); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH inference_ids AS (SELECT id FROM tensorzero.chat_inferences + WHERE function_name = $1 AND variant_name = $2) + SELECT metric_name, feedback_count::INT FROM ( + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.boolean_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.float_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT 'demonstration' as metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.demonstration_feedback + WHERE inference_id IN (SELECT id FROM inference_ids) + ) combined + WHERE feedback_count > 0 + ORDER BY metric_name + ", + ); + } + + #[test] + fn test_build_metrics_with_feedback_query_json_table() { + let qb = + build_metrics_with_feedback_query("my_function", "tensorzero.json_inferences", None); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH inference_ids AS (SELECT id FROM tensorzero.json_inferences WHERE function_name = $1) + SELECT metric_name, feedback_count::INT FROM ( + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.boolean_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.float_metric_feedback + WHERE target_id IN (SELECT id FROM inference_ids) + GROUP BY metric_name + UNION ALL + SELECT 'demonstration' as metric_name, COUNT(*)::INT as feedback_count + FROM tensorzero.demonstration_feedback + WHERE inference_id IN (SELECT id FROM inference_ids) + ) combined + WHERE feedback_count > 0 + ORDER BY metric_name + ", + ); + } + + // ===== build_cumulative_feedback_timeseries_query tests ===== + + #[test] + fn test_build_cumulative_feedback_timeseries_query_day_no_variants() { + let qb = + build_cumulative_feedback_timeseries_query("my_metric", "my_function", &[], "day", 30); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback_with_variant AS (SELECT date_trunc('day', f.created_at) + INTERVAL '1 day' AS period_end, + i.variant_name, + f.value + FROM ( + SELECT target_id, value::INT::DOUBLE PRECISION as value, created_at + FROM tensorzero.boolean_metric_feedback WHERE metric_name = $1 + UNION ALL + SELECT target_id, value, created_at + FROM tensorzero.float_metric_feedback WHERE metric_name = $2 + ) f + JOIN ( + SELECT id, variant_name FROM tensorzero.chat_inferences WHERE function_name = $3 + UNION ALL + SELECT id, variant_name FROM tensorzero.json_inferences WHERE function_name = $4) i ON f.target_id = i.id WHERE 1=1 + ), + period_stats AS ( + SELECT + period_end, + variant_name, + COUNT(*) as period_count, + SUM(value) as period_sum, + SUM(value * value) as period_sum_sq + FROM feedback_with_variant + GROUP BY period_end, variant_name + ), + cumulative_stats AS ( + SELECT + period_end, + variant_name, + SUM(period_count) OVER w as count, + SUM(period_sum) OVER w as sum_val, + SUM(period_sum_sq) OVER w as sum_sq + FROM period_stats + WINDOW w AS (PARTITION BY variant_name ORDER BY period_end ROWS UNBOUNDED PRECEDING) + ), + max_period AS ( + SELECT MAX(period_end) as max_end FROM cumulative_stats + ) + SELECT + period_end, + variant_name, + (sum_val / count)::REAL as mean, + CASE WHEN count > 1 THEN ((sum_sq - sum_val * sum_val / count) / (count - 1))::REAL END as variance, + count::BIGINT + FROM cumulative_stats + WHERE period_end >= (SELECT max_end FROM max_period) - ($5 || ' day')::INTERVAL ORDER BY period_end, variant_name + ", + ); + } + + #[test] + fn test_build_cumulative_feedback_timeseries_query_hour_with_variants() { + let variant_names = vec!["variant_a".to_string(), "variant_b".to_string()]; + let qb = build_cumulative_feedback_timeseries_query( + "my_metric", + "my_function", + &variant_names, + "hour", + 24, + ); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback_with_variant AS (SELECT date_trunc('hour', f.created_at) + INTERVAL '1 hour' AS period_end, + i.variant_name, + f.value + FROM ( + SELECT target_id, value::INT::DOUBLE PRECISION as value, created_at + FROM tensorzero.boolean_metric_feedback WHERE metric_name = $1 + UNION ALL + SELECT target_id, value, created_at + FROM tensorzero.float_metric_feedback WHERE metric_name = $2 + ) f + JOIN ( + SELECT id, variant_name FROM tensorzero.chat_inferences WHERE function_name = $3 + UNION ALL + SELECT id, variant_name FROM tensorzero.json_inferences WHERE function_name = $4) i ON f.target_id = i.id WHERE 1=1 AND i.variant_name = ANY($5) + ), + period_stats AS ( + SELECT + period_end, + variant_name, + COUNT(*) as period_count, + SUM(value) as period_sum, + SUM(value * value) as period_sum_sq + FROM feedback_with_variant + GROUP BY period_end, variant_name + ), + cumulative_stats AS ( + SELECT + period_end, + variant_name, + SUM(period_count) OVER w as count, + SUM(period_sum) OVER w as sum_val, + SUM(period_sum_sq) OVER w as sum_sq + FROM period_stats + WINDOW w AS (PARTITION BY variant_name ORDER BY period_end ROWS UNBOUNDED PRECEDING) + ), + max_period AS ( + SELECT MAX(period_end) as max_end FROM cumulative_stats + ) + SELECT + period_end, + variant_name, + (sum_val / count)::REAL as mean, + CASE WHEN count > 1 THEN ((sum_sq - sum_val * sum_val / count) / (count - 1))::REAL END as variance, + count::BIGINT + FROM cumulative_stats + WHERE period_end >= (SELECT max_end FROM max_period) - ($6 || ' hour')::INTERVAL ORDER BY period_end, variant_name + ", + ); + } + + #[test] + fn test_build_cumulative_feedback_timeseries_query_week() { + let qb = + build_cumulative_feedback_timeseries_query("my_metric", "my_function", &[], "week", 12); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + // Just check that week interval is used correctly + assert!( + sql.contains("date_trunc('week'"), + "Query should use week truncation" + ); + assert!( + sql.contains("INTERVAL '1 week'"), + "Query should use week interval" + ); + assert!( + sql.contains("' week')::INTERVAL"), + "Query should use week in period filter" + ); + } + + // ===== build_variant_performances_query tests ===== + + #[test] + fn test_build_variant_performances_query_inference_level_boolean() { + let params = VariantPerformancesQueryParams { + metric_name: "my_metric", + function_name: "my_function", + inference_table: "tensorzero.chat_inferences", + metric_table: "tensorzero.boolean_metric_feedback", + value_cast: "value::INT::DOUBLE PRECISION", + time_bucket_expr: "date_trunc('day', i.created_at)", + metric_level: MetricConfigLevel::Inference, + variant_name: None, + }; + let qb = build_variant_performances_query(¶ms); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback AS ( + SELECT DISTINCT ON (target_id) target_id, value::INT::DOUBLE PRECISION as value, created_at + FROM tensorzero.boolean_metric_feedback WHERE metric_name = $1 ORDER BY target_id, created_at DESC + ) + SELECT date_trunc('day', i.created_at) as period_start, + i.variant_name, + COUNT(*)::INT as count, + AVG(f.value) as avg_metric, + STDDEV_SAMP(f.value) as stdev, + CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(f.value) / SQRT(COUNT(*))) END as ci_error + FROM tensorzero.chat_inferences i + JOIN feedback f ON f.target_id = i.id + WHERE i.function_name = $2 GROUP BY period_start, i.variant_name ORDER BY period_start ASC, i.variant_name ASC + ", + ); + } + + #[test] + fn test_build_variant_performances_query_inference_level_float() { + let params = VariantPerformancesQueryParams { + metric_name: "my_metric", + function_name: "my_function", + inference_table: "tensorzero.json_inferences", + metric_table: "tensorzero.float_metric_feedback", + value_cast: "value::DOUBLE PRECISION", + time_bucket_expr: "date_trunc('hour', i.created_at)", + metric_level: MetricConfigLevel::Inference, + variant_name: None, + }; + let qb = build_variant_performances_query(¶ms); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback AS ( + SELECT DISTINCT ON (target_id) target_id, value::DOUBLE PRECISION as value, created_at + FROM tensorzero.float_metric_feedback WHERE metric_name = $1 ORDER BY target_id, created_at DESC + ) + SELECT date_trunc('hour', i.created_at) as period_start, + i.variant_name, + COUNT(*)::INT as count, + AVG(f.value) as avg_metric, + STDDEV_SAMP(f.value) as stdev, + CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(f.value) / SQRT(COUNT(*))) END as ci_error + FROM tensorzero.json_inferences i + JOIN feedback f ON f.target_id = i.id + WHERE i.function_name = $2 GROUP BY period_start, i.variant_name ORDER BY period_start ASC, i.variant_name ASC + ", + ); + } + + #[test] + fn test_build_variant_performances_query_inference_level_with_variant() { + let params = VariantPerformancesQueryParams { + metric_name: "my_metric", + function_name: "my_function", + inference_table: "tensorzero.chat_inferences", + metric_table: "tensorzero.boolean_metric_feedback", + value_cast: "value::INT::DOUBLE PRECISION", + time_bucket_expr: "date_trunc('day', i.created_at)", + metric_level: MetricConfigLevel::Inference, + variant_name: Some("my_variant"), + }; + let qb = build_variant_performances_query(¶ms); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback AS ( + SELECT DISTINCT ON (target_id) target_id, value::INT::DOUBLE PRECISION as value, created_at + FROM tensorzero.boolean_metric_feedback WHERE metric_name = $1 ORDER BY target_id, created_at DESC + ) + SELECT date_trunc('day', i.created_at) as period_start, + i.variant_name, + COUNT(*)::INT as count, + AVG(f.value) as avg_metric, + STDDEV_SAMP(f.value) as stdev, + CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(f.value) / SQRT(COUNT(*))) END as ci_error + FROM tensorzero.chat_inferences i + JOIN feedback f ON f.target_id = i.id + WHERE i.function_name = $2 AND i.variant_name = $3 GROUP BY period_start, i.variant_name ORDER BY period_start ASC, i.variant_name ASC + ", + ); + } + + #[test] + fn test_build_variant_performances_query_episode_level_boolean() { + let params = VariantPerformancesQueryParams { + metric_name: "my_metric", + function_name: "my_function", + inference_table: "tensorzero.chat_inferences", + metric_table: "tensorzero.boolean_metric_feedback", + value_cast: "value::INT::DOUBLE PRECISION", + time_bucket_expr: "date_trunc('day', i.created_at)", + metric_level: MetricConfigLevel::Episode, + variant_name: None, + }; + let qb = build_variant_performances_query(¶ms); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback AS ( + SELECT DISTINCT ON (target_id) target_id, value::INT::DOUBLE PRECISION as value + FROM tensorzero.boolean_metric_feedback WHERE metric_name = $1 ORDER BY target_id, created_at DESC + ), + per_episode AS ( + SELECT date_trunc('day', i.created_at) AS period_start, + i.variant_name, + i.episode_id, + f.value + FROM tensorzero.chat_inferences i + JOIN feedback f ON f.target_id = i.episode_id + WHERE i.function_name = $2 GROUP BY period_start, i.variant_name, i.episode_id, f.value + ) + SELECT + period_start, + variant_name, + COUNT(*)::INT as count, + AVG(value) as avg_metric, + STDDEV_SAMP(value) as stdev, + CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(value) / SQRT(COUNT(*))) END as ci_error + FROM per_episode + GROUP BY period_start, variant_name + ORDER BY period_start ASC, variant_name ASC + ", + ); + } + + #[test] + fn test_build_variant_performances_query_episode_level_with_variant() { + let params = VariantPerformancesQueryParams { + metric_name: "my_metric", + function_name: "my_function", + inference_table: "tensorzero.chat_inferences", + metric_table: "tensorzero.boolean_metric_feedback", + value_cast: "value::INT::DOUBLE PRECISION", + time_bucket_expr: "date_trunc('day', i.created_at)", + metric_level: MetricConfigLevel::Episode, + variant_name: Some("my_variant"), + }; + let qb = build_variant_performances_query(¶ms); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + WITH feedback AS ( + SELECT DISTINCT ON (target_id) target_id, value::INT::DOUBLE PRECISION as value + FROM tensorzero.boolean_metric_feedback WHERE metric_name = $1 ORDER BY target_id, created_at DESC + ), + per_episode AS ( + SELECT date_trunc('day', i.created_at) AS period_start, + i.variant_name, + i.episode_id, + f.value + FROM tensorzero.chat_inferences i + JOIN feedback f ON f.target_id = i.episode_id + WHERE i.function_name = $2 AND i.variant_name = $3 GROUP BY period_start, i.variant_name, i.episode_id, f.value + ) + SELECT + period_start, + variant_name, + COUNT(*)::INT as count, + AVG(value) as avg_metric, + STDDEV_SAMP(value) as stdev, + CASE WHEN COUNT(*) >= 2 THEN 1.96 * (STDDEV_SAMP(value) / SQRT(COUNT(*))) END as ci_error + FROM per_episode + GROUP BY period_start, variant_name + ORDER BY period_start ASC, variant_name ASC + ", + ); + } + + #[test] + fn test_build_variant_performances_query_cumulative_time_window() { + let params = VariantPerformancesQueryParams { + metric_name: "my_metric", + function_name: "my_function", + inference_table: "tensorzero.chat_inferences", + metric_table: "tensorzero.boolean_metric_feedback", + value_cast: "value::INT::DOUBLE PRECISION", + time_bucket_expr: "'1970-01-01 00:00:00'::TIMESTAMPTZ", + metric_level: MetricConfigLevel::Inference, + variant_name: None, + }; + let qb = build_variant_performances_query(¶ms); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + // Verify the cumulative time bucket is used + assert!( + sql.contains("'1970-01-01 00:00:00'::TIMESTAMPTZ as period_start"), + "Query should use cumulative timestamp" + ); + } +} diff --git a/tensorzero-core/src/db/postgres/inference_queries.rs b/tensorzero-core/src/db/postgres/inference_queries.rs index 114d68e0726..ef252b99141 100644 --- a/tensorzero-core/src/db/postgres/inference_queries.rs +++ b/tensorzero-core/src/db/postgres/inference_queries.rs @@ -145,70 +145,13 @@ impl InferenceQueries for PostgresConnectionInfo { ) -> Result, Error> { let pool = self.get_pool_result()?; - // Use UNION ALL to query both tables in a single query // Determine ORDER BY direction based on pagination let order_direction = match ¶ms.pagination { Some(PaginationParams::After { .. }) => "ASC", Some(PaginationParams::Before { .. }) | None => "DESC", }; - let order_by_clause = format!("ORDER BY id {order_direction}"); - - // Build the entire query using a single QueryBuilder - let mut query_builder: QueryBuilder = QueryBuilder::new( - r" - SELECT * FROM ( - (SELECT - 'chat'::text as function_type, - id, - function_name, - variant_name, - episode_id, - snapshot_hash - FROM tensorzero.chat_inferences - WHERE 1=1", - ); - - // Add chat filters - apply_metadata_filters(&mut query_builder, params); - - query_builder.push(" "); - query_builder.push(&order_by_clause); - query_builder.push(" LIMIT "); - query_builder.push_bind(params.limit as i64); - - // UNION ALL with json subquery - query_builder.push( - r") - UNION ALL - (SELECT - 'json'::text as function_type, - id, - function_name, - variant_name, - episode_id, - snapshot_hash - FROM tensorzero.json_inferences - WHERE 1=1", - ); - - // Add json filters - apply_metadata_filters(&mut query_builder, params); - - query_builder.push(" "); - query_builder.push(&order_by_clause); - query_builder.push(" LIMIT "); - query_builder.push_bind(params.limit as i64); - - // Close subqueries and add outer ORDER BY + LIMIT - query_builder.push( - ") - ) AS combined - ", - ); - query_builder.push(&order_by_clause); - query_builder.push(" LIMIT "); - query_builder.push_bind(params.limit as i64); + let mut query_builder = build_inference_metadata_query(params, order_direction); let mut results: Vec = query_builder.build_query_as().fetch_all(pool).await?; @@ -335,7 +278,7 @@ impl InferenceQueries for PostgresConnectionInfo { let pool = self.get_pool_result()?; let result = sqlx::query!( - r#" + r" SELECT dynamic_tools, dynamic_provider_tools, @@ -345,7 +288,7 @@ impl InferenceQueries for PostgresConnectionInfo { FROM tensorzero.chat_inferences WHERE function_name = $1 AND id = $2 LIMIT 1 - "#, + ", function_name, inference_id ) @@ -384,12 +327,12 @@ impl InferenceQueries for PostgresConnectionInfo { let pool = self.get_pool_result()?; let result = sqlx::query!( - r#" + r" SELECT output_schema FROM tensorzero.json_inferences WHERE function_name = $1 AND id = $2 LIMIT 1 - "#, + ", function_name, inference_id ) @@ -410,7 +353,7 @@ impl InferenceQueries for PostgresConnectionInfo { match function_info.function_type { FunctionType::Chat => { let result = sqlx::query!( - r#" + r" SELECT output FROM tensorzero.chat_inferences WHERE id = $1 @@ -418,7 +361,7 @@ impl InferenceQueries for PostgresConnectionInfo { AND function_name = $3 AND variant_name = $4 LIMIT 1 - "#, + ", inference_id, function_info.episode_id, function_info.function_name, @@ -432,7 +375,7 @@ impl InferenceQueries for PostgresConnectionInfo { } FunctionType::Json => { let result = sqlx::query!( - r#" + r" SELECT output FROM tensorzero.json_inferences WHERE id = $1 @@ -440,7 +383,7 @@ impl InferenceQueries for PostgresConnectionInfo { AND function_name = $3 AND variant_name = $4 LIMIT 1 - "#, + ", inference_id, function_info.episode_id, function_info.function_name, @@ -753,15 +696,84 @@ fn build_order_by_clause( }) } +// ===== Query builder functions for list_inference_metadata ===== + +/// Builds the query for listing inference metadata (UNION ALL of chat and json tables). +/// The `order_direction` parameter should be "ASC" for After pagination, "DESC" otherwise. +fn build_inference_metadata_query( + params: &ListInferenceMetadataParams, + order_direction: &str, +) -> QueryBuilder { + let order_by_clause = format!("ORDER BY id {order_direction}"); + + let mut qb = QueryBuilder::new( + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1", + ); + + // Add chat filters + apply_metadata_filters(&mut qb, params); + + qb.push(" "); + qb.push(&order_by_clause); + qb.push(" LIMIT "); + qb.push_bind(params.limit as i64); + + // UNION ALL with json subquery + qb.push( + r") + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1", + ); + + // Add json filters + apply_metadata_filters(&mut qb, params); + + qb.push(" "); + qb.push(&order_by_clause); + qb.push(" LIMIT "); + qb.push_bind(params.limit as i64); + + // Close subqueries and add outer ORDER BY + LIMIT + qb.push( + ") + ) AS combined + ", + ); + qb.push(&order_by_clause); + qb.push(" LIMIT "); + qb.push_bind(params.limit as i64); + + qb +} + // ===== Helper functions for chat inference queries ===== -async fn query_chat_inferences( - pool: &sqlx::PgPool, +/// Builds the query for listing chat inferences. +/// This is separated from execution to allow unit testing of the generated SQL. +fn build_chat_inferences_query( config: &Config, params: &ListInferencesParams<'_>, limit: u32, offset: u32, -) -> Result, Error> { +) -> Result, Error> { // Build ORDER BY clause first to get any required metric JOINs let order_by_result = build_order_by_clause(params, config)?; @@ -880,19 +892,32 @@ async fn query_chat_inferences( query_builder.push(" OFFSET "); query_builder.push_bind(offset as i64); + Ok(query_builder) +} + +async fn query_chat_inferences( + pool: &sqlx::PgPool, + config: &Config, + params: &ListInferencesParams<'_>, + limit: u32, + offset: u32, +) -> Result, Error> { + let mut query_builder = build_chat_inferences_query(config, params, limit, offset)?; + let results: Vec = query_builder.build_query_as().fetch_all(pool).await?; Ok(results) } -async fn query_json_inferences( - pool: &sqlx::PgPool, +/// Builds the query for listing JSON inferences. +/// This is separated from execution to allow unit testing of the generated SQL. +fn build_json_inferences_query( config: &Config, params: &ListInferencesParams<'_>, limit: u32, offset: u32, -) -> Result, Error> { +) -> Result, Error> { // Build ORDER BY clause first to get any required metric JOINs let order_by_result = build_order_by_clause(params, config)?; @@ -1007,6 +1032,18 @@ async fn query_json_inferences( query_builder.push(" OFFSET "); query_builder.push_bind(offset as i64); + Ok(query_builder) +} + +async fn query_json_inferences( + pool: &sqlx::PgPool, + config: &Config, + params: &ListInferencesParams<'_>, + limit: u32, + offset: u32, +) -> Result, Error> { + let mut query_builder = build_json_inferences_query(config, params, limit, offset)?; + let results: Vec = query_builder.build_query_as().fetch_all(pool).await?; Ok(results) @@ -1728,10 +1765,16 @@ mod tests { use super::*; use crate::config::ConfigFileGlob; - use crate::db::clickhouse::query_builder::{OrderBy, OrderByTerm, OrderDirection}; + use crate::db::clickhouse::query_builder::{ + InferenceFilter, OrderBy, OrderByTerm, OrderDirection, + }; use crate::db::inferences::ListInferencesParams; use crate::db::test_helpers::{assert_query_contains, assert_query_equals}; + fn make_test_config() -> Config { + Config::default() + } + async fn get_test_config() -> Config { Config::load_from_path_optional_verify_credentials( &ConfigFileGlob::new_from_path(Path::new("tests/e2e/config/tensorzero.*.toml")) @@ -1743,134 +1786,329 @@ mod tests { .into_config_without_writing_for_tests() } - #[tokio::test] - async fn test_build_inferences_union_query_allows_timestamp_order_by() { - let config = get_test_config().await; - let order_by = vec![OrderBy { - term: OrderByTerm::Timestamp, - direction: OrderDirection::Desc, - }]; + #[test] + fn test_build_chat_inferences_query_basic() { + let config = make_test_config(); let params = ListInferencesParams { - order_by: Some(&order_by), - ..Default::default() + function_name: Some("test_function"), + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: None, + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, }; - let result = build_inferences_union_query(&config, ¶ms); - assert!( - result.is_ok(), - "ORDER BY timestamp should be allowed: {:?}", - result.err() + let query_builder = build_chat_inferences_query(&config, ¶ms, 10, 0) + .expect("Query building should succeed"); + let sql = query_builder.sql(); + assert_query_equals( + sql.as_str(), + r" + SELECT + i.id, + i.function_name, + i.variant_name, + i.episode_id, + i.created_at as timestamp, + i.input, + i.output, NULL::jsonb as dispreferred_output, + i.dynamic_tools, + i.dynamic_provider_tools, + i.allowed_tools, + i.tool_choice, + i.parallel_tool_calls, + i.tags, + i.extra_body, + i.inference_params, + i.processing_time_ms, + i.ttft_ms + FROM tensorzero.chat_inferences i + WHERE 1=1 AND i.function_name = $1 + ORDER BY i.id DESC + LIMIT $2 OFFSET $3 + ", ); - - let query_builder = result.unwrap(); - let sql_str = query_builder.sql(); - let sql = sql_str.as_str(); - assert_query_contains(sql, "ORDER BY timestamp DESC"); } - #[tokio::test] - async fn test_build_inferences_union_query_allows_no_order_by() { - let config = get_test_config().await; - let params = ListInferencesParams::default(); + #[test] + fn test_build_chat_inferences_query_with_demonstration_output_source() { + let config = make_test_config(); + let params = ListInferencesParams { + function_name: Some("test_function"), + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: None, + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Demonstration, + search_query_experimental: None, + }; - let result = build_inferences_union_query(&config, ¶ms); - assert!( - result.is_ok(), - "No ORDER BY should be allowed: {:?}", - result.err() + let query_builder = build_chat_inferences_query(&config, ¶ms, 10, 0) + .expect("Query building should succeed"); + let sql = query_builder.sql(); + assert_query_equals( + sql.as_str(), + r" + SELECT + i.id, + i.function_name, + i.variant_name, + i.episode_id, + i.created_at as timestamp, + i.input, + demo_f.value AS output, i.output as dispreferred_output, + i.dynamic_tools, + i.dynamic_provider_tools, + i.allowed_tools, + i.tool_choice, + i.parallel_tool_calls, + i.tags, + i.extra_body, + i.inference_params, + i.processing_time_ms, + i.ttft_ms + FROM tensorzero.chat_inferences i + JOIN ( + SELECT DISTINCT ON (inference_id) + inference_id, + value + FROM tensorzero.demonstration_feedback + ORDER BY inference_id, created_at DESC + ) AS demo_f ON i.id = demo_f.inference_id + WHERE 1=1 AND i.function_name = $1 + ORDER BY i.id DESC + LIMIT $2 OFFSET $3 + ", ); } - #[tokio::test] - async fn test_build_inferences_union_query_rejects_metric_order_by() { - let config = get_test_config().await; - let order_by = vec![OrderBy { - term: OrderByTerm::Metric { - name: "test_metric".to_string(), - }, - direction: OrderDirection::Desc, - }]; + #[test] + fn test_build_chat_inferences_query_with_pagination_before() { + let config = make_test_config(); + let cursor_id = Uuid::now_v7(); let params = ListInferencesParams { - order_by: Some(&order_by), - ..Default::default() + function_name: Some("test_function"), + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: None, + limit: 10, + offset: 0, + pagination: Some(PaginationParams::Before { id: cursor_id }), + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, }; - let result = build_inferences_union_query(&config, ¶ms); - let err = match result { - Err(e) => e, - Ok(_) => panic!("ORDER BY metric should be rejected"), - }; - assert!( - err.to_string() - .contains("ORDER BY metric is not supported when querying without function_name"), - "Error message should mention metric not supported: {err}" + let query_builder = build_chat_inferences_query(&config, ¶ms, 10, 0) + .expect("Query building should succeed"); + let sql = query_builder.sql(); + assert_query_equals( + sql.as_str(), + r" + SELECT + i.id, + i.function_name, + i.variant_name, + i.episode_id, + i.created_at as timestamp, + i.input, + i.output, NULL::jsonb as dispreferred_output, + i.dynamic_tools, + i.dynamic_provider_tools, + i.allowed_tools, + i.tool_choice, + i.parallel_tool_calls, + i.tags, + i.extra_body, + i.inference_params, + i.processing_time_ms, + i.ttft_ms + FROM tensorzero.chat_inferences i + WHERE 1=1 AND i.function_name = $1 AND i.id < $2 + ORDER BY i.id DESC + LIMIT $3 OFFSET $4 + ", ); } - #[tokio::test] - async fn test_build_inferences_union_query_rejects_search_relevance_order_by() { - let config = get_test_config().await; - let order_by = vec![OrderBy { - term: OrderByTerm::SearchRelevance, - direction: OrderDirection::Desc, - }]; + #[test] + fn test_build_chat_inferences_query_with_pagination_after() { + let config = make_test_config(); + let cursor_id = Uuid::now_v7(); let params = ListInferencesParams { - order_by: Some(&order_by), - ..Default::default() + function_name: Some("test_function"), + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: None, + limit: 10, + offset: 0, + pagination: Some(PaginationParams::After { id: cursor_id }), + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, }; - let result = build_inferences_union_query(&config, ¶ms); - let err = match result { - Err(e) => e, - Ok(_) => panic!("ORDER BY search_relevance should be rejected"), - }; - assert!( - err.to_string() - .contains("ORDER BY search_relevance is not yet supported"), - "Error message should mention search_relevance not supported: {err}" + let query_builder = build_chat_inferences_query(&config, ¶ms, 10, 0) + .expect("Query building should succeed"); + let sql = query_builder.sql(); + assert_query_equals( + sql.as_str(), + r" + SELECT + i.id, + i.function_name, + i.variant_name, + i.episode_id, + i.created_at as timestamp, + i.input, + i.output, NULL::jsonb as dispreferred_output, + i.dynamic_tools, + i.dynamic_provider_tools, + i.allowed_tools, + i.tool_choice, + i.parallel_tool_calls, + i.tags, + i.extra_body, + i.inference_params, + i.processing_time_ms, + i.ttft_ms + FROM tensorzero.chat_inferences i + WHERE 1=1 AND i.function_name = $1 AND i.id > $2 + ORDER BY i.id ASC + LIMIT $3 OFFSET $4 + ", ); } - #[tokio::test] - async fn test_build_inferences_union_query_rejects_metric_in_multiple_order_by_terms() { - let config = get_test_config().await; - let order_by = vec![ - OrderBy { - term: OrderByTerm::Timestamp, - direction: OrderDirection::Desc, - }, - OrderBy { - term: OrderByTerm::Metric { - name: "test_metric".to_string(), - }, - direction: OrderDirection::Asc, - }, - ]; + #[test] + fn test_build_chat_inferences_query_with_search() { + let config = make_test_config(); let params = ListInferencesParams { - order_by: Some(&order_by), - ..Default::default() + function_name: Some("test_function"), + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: None, + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Inference, + search_query_experimental: Some("test query"), }; - let result = build_inferences_union_query(&config, ¶ms); - assert!( - result.is_err(), - "ORDER BY with metric in multiple terms should be rejected" + let query_builder = build_chat_inferences_query(&config, ¶ms, 10, 0) + .expect("Query building should succeed"); + let sql = query_builder.sql(); + assert_query_equals( + sql.as_str(), + r" + SELECT + i.id, + i.function_name, + i.variant_name, + i.episode_id, + i.created_at as timestamp, + i.input, + i.output, NULL::jsonb as dispreferred_output, + i.dynamic_tools, + i.dynamic_provider_tools, + i.allowed_tools, + i.tool_choice, + i.parallel_tool_calls, + i.tags, + i.extra_body, + i.inference_params, + i.processing_time_ms, + i.ttft_ms + FROM tensorzero.chat_inferences i + WHERE 1=1 AND i.function_name = $1 + AND (i.input::text ILIKE $2 OR i.output::text ILIKE $3) + ORDER BY i.id DESC + LIMIT $4 OFFSET $5 + ", ); } - #[tokio::test] - async fn test_build_inferences_union_query_generates_union_all() { - let config = get_test_config().await; - let params = ListInferencesParams::default(); + #[test] + fn test_build_json_inferences_query_basic() { + let config = make_test_config(); + let params = ListInferencesParams { + function_name: Some("test_function"), + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: None, + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, + }; - let result = build_inferences_union_query(&config, ¶ms); - assert!(result.is_ok(), "Query building should succeed"); + let query_builder = build_json_inferences_query(&config, ¶ms, 10, 0) + .expect("Query building should succeed"); + let sql = query_builder.sql(); + assert_query_equals( + sql.as_str(), + r" + SELECT + i.id, + i.function_name, + i.variant_name, + i.episode_id, + i.created_at as timestamp, + i.input, + i.output, NULL::jsonb as dispreferred_output, + i.output_schema, + i.tags, + i.extra_body, + i.inference_params, + i.processing_time_ms, + i.ttft_ms + FROM tensorzero.json_inferences i + WHERE 1=1 AND i.function_name = $1 + ORDER BY i.id DESC + LIMIT $2 OFFSET $3 + ", + ); + } - let query_builder = result.unwrap(); - let sql_str = query_builder.sql(); - let sql = sql_str.as_str(); + #[test] + fn test_build_inferences_union_query_basic() { + let config = make_test_config(); + let params = ListInferencesParams { + function_name: None, // UNION query is used when function_name is None + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: None, + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, + }; - let expected = r" + let query_builder = + build_inferences_union_query(&config, ¶ms).expect("Query building should succeed"); + let sql = query_builder.sql(); + assert_query_equals( + sql.as_str(), + r" SELECT * FROM ( (SELECT 'chat'::text as inference_type, @@ -1919,8 +2157,541 @@ mod tests { WHERE 1=1 ORDER BY id DESC LIMIT $2) ) AS combined ORDER BY id DESC LIMIT $3 OFFSET $4 - "; + ", + ); + } + + #[test] + fn test_build_inferences_union_query_rejects_metric_order() { + let config = make_test_config(); + let order_by = [OrderBy { + term: OrderByTerm::Metric { + name: "some_metric".to_string(), + }, + direction: OrderDirection::Desc, + }]; + let params = ListInferencesParams { + function_name: None, + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: Some(&order_by), + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, + }; + + let result = build_inferences_union_query(&config, ¶ms); + let err = match result { + Ok(_) => panic!("UNION query should reject metric ordering"), + Err(e) => e, + }; + assert!( + err.to_string().contains("ORDER BY metric is not supported"), + "Error message should indicate metric ordering is not supported: {err}" + ); + } + + #[test] + fn test_build_inferences_union_query_rejects_search_relevance_order() { + let config = make_test_config(); + let order_by = [OrderBy { + term: OrderByTerm::SearchRelevance, + direction: OrderDirection::Desc, + }]; + let params = ListInferencesParams { + function_name: None, + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: Some(&order_by), + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, + }; + + let result = build_inferences_union_query(&config, ¶ms); + let err = match result { + Ok(_) => panic!("UNION query should reject search_relevance ordering"), + Err(e) => e, + }; + assert!( + err.to_string() + .contains("ORDER BY search_relevance is not yet supported"), + "Error message should indicate search_relevance ordering is not supported: {err}" + ); + } + + #[test] + fn test_build_inferences_union_query_with_timestamp_order() { + let config = make_test_config(); + let order_by = [OrderBy { + term: OrderByTerm::Timestamp, + direction: OrderDirection::Asc, + }]; + let params = ListInferencesParams { + function_name: None, + variant_name: None, + episode_id: None, + ids: None, + filters: None, + order_by: Some(&order_by), + limit: 10, + offset: 0, + pagination: None, + output_source: InferenceOutputSource::Inference, + search_query_experimental: None, + }; + + let query_builder = + build_inferences_union_query(&config, ¶ms).expect("Query building should succeed"); + let sql = query_builder.sql(); + + // Should have ORDER BY with timestamp ASC + assert_query_contains(sql.as_str(), "ORDER BY timestamp ASC"); + } + + #[tokio::test] + async fn test_build_inferences_union_query_with_float_metric_filter() { + use crate::db::clickhouse::query_builder::{FloatComparisonOperator, FloatMetricFilter}; + + let config = get_test_config().await; + let filter = InferenceFilter::FloatMetric(FloatMetricFilter { + metric_name: "brevity_score".to_string(), + comparison_operator: FloatComparisonOperator::GreaterThan, + value: 0.5, + }); + let params = ListInferencesParams { + function_name: None, + filters: Some(&filter), + ..Default::default() + }; + + let query_builder = + build_inferences_union_query(&config, ¶ms).expect("Query building should succeed"); + let sql = query_builder.sql(); + + // Both subqueries should have the metric filter via EXISTS subquery + assert_query_contains( + sql.as_str(), + "EXISTS (SELECT 1 FROM tensorzero.float_metric_feedback f WHERE f.target_id = i.id AND f.metric_name =", + ); + } + + #[tokio::test] + async fn test_build_inferences_union_query_with_boolean_metric_filter() { + use crate::db::clickhouse::query_builder::BooleanMetricFilter; + + let config = get_test_config().await; + let filter = InferenceFilter::BooleanMetric(BooleanMetricFilter { + metric_name: "task_success".to_string(), + value: true, + }); + let params = ListInferencesParams { + function_name: None, + filters: Some(&filter), + ..Default::default() + }; + + let query_builder = + build_inferences_union_query(&config, ¶ms).expect("Query building should succeed"); + let sql = query_builder.sql(); + + // Both subqueries should have the metric filter via EXISTS subquery + assert_query_contains( + sql.as_str(), + "EXISTS (SELECT 1 FROM tensorzero.boolean_metric_feedback f WHERE f.target_id = i.id AND f.metric_name =", + ); + } - assert_query_equals(sql, expected); + #[tokio::test] + async fn test_build_inferences_union_query_with_demonstration_filter() { + use crate::db::clickhouse::query_builder::DemonstrationFeedbackFilter; + + let config = get_test_config().await; + let filter = InferenceFilter::DemonstrationFeedback(DemonstrationFeedbackFilter { + has_demonstration: true, + }); + let params = ListInferencesParams { + function_name: None, + filters: Some(&filter), + ..Default::default() + }; + + let query_builder = + build_inferences_union_query(&config, ¶ms).expect("Query building should succeed"); + let sql = query_builder.sql(); + + // Both subqueries should have the demonstration filter via EXISTS subquery + assert_query_contains( + sql.as_str(), + "EXISTS (SELECT 1 FROM tensorzero.demonstration_feedback WHERE inference_id = i.id)", + ); + } + + #[tokio::test] + async fn test_build_inferences_union_query_rejects_invalid_metric_name() { + use crate::db::clickhouse::query_builder::{FloatComparisonOperator, FloatMetricFilter}; + + let config = get_test_config().await; + let filter = InferenceFilter::FloatMetric(FloatMetricFilter { + metric_name: "nonexistent_metric".to_string(), + comparison_operator: FloatComparisonOperator::GreaterThan, + value: 0.5, + }); + let params = ListInferencesParams { + function_name: None, + filters: Some(&filter), + ..Default::default() + }; + + let result = build_inferences_union_query(&config, ¶ms); + let err = match result { + Err(e) => e, + Ok(_) => panic!("Should reject invalid metric name"), + }; + assert!( + err.to_string().contains("nonexistent_metric"), + "Error should mention the invalid metric name: {err}" + ); + } + + #[tokio::test] + async fn test_build_inferences_union_query_with_episode_level_metric_filter() { + use crate::db::clickhouse::query_builder::{FloatComparisonOperator, FloatMetricFilter}; + + let config = get_test_config().await; + // user_rating is an episode-level metric + let filter = InferenceFilter::FloatMetric(FloatMetricFilter { + metric_name: "user_rating".to_string(), + comparison_operator: FloatComparisonOperator::GreaterThanOrEqual, + value: 4.0, + }); + let params = ListInferencesParams { + function_name: None, + filters: Some(&filter), + ..Default::default() + }; + + let query_builder = + build_inferences_union_query(&config, ¶ms).expect("Query building should succeed"); + let sql = query_builder.sql(); + + // Episode-level metrics should join on episode_id + assert_query_contains(sql.as_str(), "f.target_id = i.episode_id"); + } + + // ===== build_inference_metadata_query tests ===== + + #[test] + fn test_build_inference_metadata_query_no_filters() { + let params = ListInferenceMetadataParams { + function_name: None, + variant_name: None, + episode_id: None, + pagination: None, + limit: 100, + }; + + let qb = build_inference_metadata_query(¶ms, "DESC"); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1 ORDER BY id DESC LIMIT $1) + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1 ORDER BY id DESC LIMIT $2) + ) AS combined + ORDER BY id DESC LIMIT $3 + ", + ); + } + + #[test] + fn test_build_inference_metadata_query_with_function_name() { + let params = ListInferenceMetadataParams { + function_name: Some("test_function".to_string()), + variant_name: None, + episode_id: None, + pagination: None, + limit: 50, + }; + + let qb = build_inference_metadata_query(¶ms, "DESC"); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1 AND function_name = $1 ORDER BY id DESC LIMIT $2) + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1 AND function_name = $3 ORDER BY id DESC LIMIT $4) + ) AS combined + ORDER BY id DESC LIMIT $5 + ", + ); + } + + #[test] + fn test_build_inference_metadata_query_with_variant_name() { + let params = ListInferenceMetadataParams { + function_name: None, + variant_name: Some("test_variant".to_string()), + episode_id: None, + pagination: None, + limit: 50, + }; + + let qb = build_inference_metadata_query(¶ms, "DESC"); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1 AND variant_name = $1 ORDER BY id DESC LIMIT $2) + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1 AND variant_name = $3 ORDER BY id DESC LIMIT $4) + ) AS combined + ORDER BY id DESC LIMIT $5 + ", + ); + } + + #[test] + fn test_build_inference_metadata_query_with_episode_id() { + let episode_id = Uuid::now_v7(); + let params = ListInferenceMetadataParams { + function_name: None, + variant_name: None, + episode_id: Some(episode_id), + pagination: None, + limit: 50, + }; + + let qb = build_inference_metadata_query(¶ms, "DESC"); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1 AND episode_id = $1 ORDER BY id DESC LIMIT $2) + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1 AND episode_id = $3 ORDER BY id DESC LIMIT $4) + ) AS combined + ORDER BY id DESC LIMIT $5 + ", + ); + } + + #[test] + fn test_build_inference_metadata_query_pagination_before() { + let before_id = Uuid::now_v7(); + let params = ListInferenceMetadataParams { + function_name: None, + variant_name: None, + episode_id: None, + pagination: Some(PaginationParams::Before { id: before_id }), + limit: 50, + }; + + let qb = build_inference_metadata_query(¶ms, "DESC"); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1 AND id < $1 ORDER BY id DESC LIMIT $2) + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1 AND id < $3 ORDER BY id DESC LIMIT $4) + ) AS combined + ORDER BY id DESC LIMIT $5 + ", + ); + } + + #[test] + fn test_build_inference_metadata_query_pagination_after() { + let after_id = Uuid::now_v7(); + let params = ListInferenceMetadataParams { + function_name: None, + variant_name: None, + episode_id: None, + pagination: Some(PaginationParams::After { id: after_id }), + limit: 50, + }; + + let qb = build_inference_metadata_query(¶ms, "ASC"); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1 AND id > $1 ORDER BY id ASC LIMIT $2) + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1 AND id > $3 ORDER BY id ASC LIMIT $4) + ) AS combined + ORDER BY id ASC LIMIT $5 + ", + ); + } + + #[test] + fn test_build_inference_metadata_query_all_filters() { + let episode_id = Uuid::now_v7(); + let before_id = Uuid::now_v7(); + let params = ListInferenceMetadataParams { + function_name: Some("my_function".to_string()), + variant_name: Some("my_variant".to_string()), + episode_id: Some(episode_id), + pagination: Some(PaginationParams::Before { id: before_id }), + limit: 25, + }; + + let qb = build_inference_metadata_query(¶ms, "DESC"); + let sql_str = qb.sql(); + let sql = sql_str.as_str(); + + assert_query_equals( + sql, + r" + SELECT * FROM ( + (SELECT + 'chat'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.chat_inferences + WHERE 1=1 AND function_name = $1 AND variant_name = $2 AND episode_id = $3 AND id < $4 ORDER BY id DESC LIMIT $5) + UNION ALL + (SELECT + 'json'::text as function_type, + id, + function_name, + variant_name, + episode_id, + snapshot_hash + FROM tensorzero.json_inferences + WHERE 1=1 AND function_name = $6 AND variant_name = $7 AND episode_id = $8 AND id < $9 ORDER BY id DESC LIMIT $10) + ) AS combined + ORDER BY id DESC LIMIT $11 + ", + ); } } From 96cac614d74b6ce44e4cc97ac8e6f54c9915491a Mon Sep 17 00:00:00 2001 From: simeonlee <15081451+simeonlee@users.noreply.github.com> Date: Tue, 3 Feb 2026 21:27:33 -0500 Subject: [PATCH 3/5] Convert autopilot sessions from use() to Await with error handling (#6085) * Convert autopilot sessions from use() to Await with error handling * Use existing TableAsyncErrorState for consistent error handling Replace custom TableErrorState with the reusable TableAsyncErrorState component from ~/components/ui/table, fixing: - colSpan mismatch (was 2, should be 3 columns) - Inconsistent error styling (now uses standard TableErrorNotice) --- ui/app/routes/autopilot/sessions/route.tsx | 88 ++++++++-------------- 1 file changed, 30 insertions(+), 58 deletions(-) diff --git a/ui/app/routes/autopilot/sessions/route.tsx b/ui/app/routes/autopilot/sessions/route.tsx index cc7f8c01993..f075159896b 100644 --- a/ui/app/routes/autopilot/sessions/route.tsx +++ b/ui/app/routes/autopilot/sessions/route.tsx @@ -1,7 +1,7 @@ import { Plus } from "lucide-react"; -import { Suspense, use } from "react"; +import { Suspense } from "react"; import type { Route } from "./+types/route"; -import { data, useLocation, useNavigate } from "react-router"; +import { Await, data, useLocation, useNavigate } from "react-router"; import { useTensorZeroStatusFetcher } from "~/routes/api/tensorzero/status"; import { PageHeader, @@ -18,6 +18,7 @@ import type { Session } from "~/types/tensorzero"; import { Skeleton } from "~/components/ui/skeleton"; import { Table, + TableAsyncErrorState, TableBody, TableCell, TableHead, @@ -95,51 +96,6 @@ function SkeletonRows() { ); } -// Resolves promise and renders table rows -function TableBodyContent({ - data, - gatewayVersion, - uiVersion, -}: { - data: Promise; - gatewayVersion?: string; - uiVersion?: string; -}) { - const { sessions } = use(data); - - return ( - - ); -} - -// Resolves promise and renders pagination -function PaginationContent({ - data, - offset, - onPreviousPage, - onNextPage, -}: { - data: Promise; - offset: number; - onPreviousPage: () => void; - onNextPage: () => void; -}) { - const { hasMore } = use(data); - - return ( - - ); -} - export default function AutopilotSessionsPage({ loaderData, }: Route.ComponentProps) { @@ -189,21 +145,37 @@ export default function AutopilotSessionsPage({ }> - + + } + > + {({ sessions }) => ( + + )} + }> - + }> + {({ hasMore }) => ( + + )} + From 60f7149ffaedc4bbda6c2d550fe04e6319c3b119 Mon Sep 17 00:00:00 2001 From: simeonlee <15081451+simeonlee@users.noreply.github.com> Date: Tue, 3 Feb 2026 21:41:31 -0500 Subject: [PATCH 4/5] Add async error handling to ModelUsage and ModelLatency components (#6084) --- ui/app/components/model/ModelLatency.tsx | 26 ++++++++++++++++++++++-- ui/app/components/model/ModelUsage.tsx | 26 ++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/ui/app/components/model/ModelLatency.tsx b/ui/app/components/model/ModelLatency.tsx index 43e78cf06c3..d917b77df49 100644 --- a/ui/app/components/model/ModelLatency.tsx +++ b/ui/app/components/model/ModelLatency.tsx @@ -8,7 +8,9 @@ import { Tooltip, } from "recharts"; import React, { useState, useMemo } from "react"; -import { Await } from "react-router"; +import { Await, useAsyncError, isRouteErrorResponse } from "react-router"; +import { SectionErrorNotice } from "~/components/ui/error/ErrorContentPrimitives"; +import { AlertCircle } from "lucide-react"; import { CHART_COLORS } from "~/utils/chart"; import { Select, @@ -89,6 +91,23 @@ function CustomTooltipContent({ active, payload, label }: TooltipProps) { const MARGIN = { top: 12, right: 16, bottom: 28, left: 56 }; +function ModelLatencyError() { + const error = useAsyncError(); + let message = "Failed to load latency data"; + if (isRouteErrorResponse(error)) { + message = typeof error.data === "string" ? error.data : message; + } else if (error instanceof Error) { + message = error.message; + } + return ( + + ); +} + function LatencyTimeWindowSelector({ value, onValueChange, @@ -280,7 +299,10 @@ export function ModelLatency({ Loading latency data...}> - + } + > {(latencyData) => ( + ); +} + function MetricSelector({ selectedMetric, onMetricChange, @@ -122,7 +141,10 @@ export function ModelUsage({ Loading model usage data...}> - + } + > {(modelUsageData) => { const { data, modelNames } = transformModelUsageData( modelUsageData, From 7e630406b047809edea2732a45e03d5fc7eb58c4 Mon Sep 17 00:00:00 2001 From: simeonlee <15081451+simeonlee@users.noreply.github.com> Date: Tue, 3 Feb 2026 22:12:04 -0500 Subject: [PATCH 5/5] Add getErrorMessage utility for consistent error handling (#6093) * Add getErrorMessage utility for consistent error handling * Use object params for getErrorMessage --- .../ui/error/ErrorContentPrimitives.tsx | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/ui/app/components/ui/error/ErrorContentPrimitives.tsx b/ui/app/components/ui/error/ErrorContentPrimitives.tsx index 20ce2bce3f8..25757e69f54 100644 --- a/ui/app/components/ui/error/ErrorContentPrimitives.tsx +++ b/ui/app/components/ui/error/ErrorContentPrimitives.tsx @@ -4,6 +4,30 @@ import { AlertCircle, type LucideIcon } from "lucide-react"; import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; import { cn } from "~/utils/common"; +/** + * Extracts a user-friendly message from an error of any type. + * Handles React Router's route error responses, standard Errors, and unknown types. + */ +export function getErrorMessage({ + error, + fallback, +}: { + /** The error to extract a message from (typically from useAsyncError()) */ + error: unknown; + /** Default message if error type is unrecognized */ + fallback: string; +}): string { + if (isRouteErrorResponse(error)) { + return typeof error.data === "string" + ? error.data + : `${error.status} ${error.statusText}`; + } + if (error instanceof Error) { + return error.message; + } + return fallback; +} + interface ErrorContentCardProps { children: React.ReactNode; className?: string; @@ -222,24 +246,11 @@ export function SectionAsyncErrorState({ ); } - let message: string; - if (isRouteErrorResponse(error)) { - if (typeof error.data === "string") { - message = error.data; - } else { - message = `${error.status} ${error.statusText}`; - } - } else if (error instanceof Error) { - message = error.message; - } else { - message = defaultMessage; - } - return ( ); }