diff --git a/CONTEXT.md b/CONTEXT.md index 07e52de..78858dd 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -292,6 +292,11 @@ Repository checkouts live under ignored `benchmarks/.cache/repos/`; generated ar ignored `benchmarks/results/`. Benchmark Dense and Sparse vectors and channel rankings live under the ignored `benchmarks/.cache/retrieval/v1/` cache; production indexes remain separate. See `benchmarks/README.md` and `benchmarks/BASELINE.md`. +Schema 25 makes promotion an excluded-evidence decision: grouped intent folds and repository holdouts +must all be present and pass aggregate, query-form, and repository guardrails. Fit-all candidates are +diagnostic and inherit that decision. The last grouped fold is the explicit untouched final test and +must pass independently. Artifacts retain exact blocker values, deterministic grouped bootstrap +intervals, and fold-level selection/local-perturbation stability diagnostics. ### Scorer diff --git a/benchmarks/README.md b/benchmarks/README.md index 25c3fa7..c863ac9 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -211,10 +211,12 @@ Weight selection uses two grouped strategies: - Leave-one-repository-out: calibrate on two repositories and validate on the third. This is emitted only when multiple repositories are selected in the same run. -Candidate selection is development-only. Grouped and repository holdouts are reported separately; the -artifact records nested cross-validation as the final promotion plan rather than presenting fit-all -quality as an untouched final test. The Markdown report also emits unweighted query-form and repository -holdout rows with candidate-versus-production guardrail metrics. +Candidate selection is development-only inside each outer fold. Grouped intent folds and repository +holdouts are the outer evaluation. Schema 25 reserves the last grouped fold as the explicit untouched +final test; its candidate is selected without those samples, and a missing or failing final fold blocks +promotion. Fit-all quality remains diagnostic: its promotion status is copied from the aggregated +excluded-fold evidence and can never make itself eligible. The Markdown report emits aggregate, +unweighted query-form, and repository holdout rows with candidate-versus-production guardrails. The grid optimizes development `Recall@20`, then `Recall@10`, 4k context recall, and MRR. Each fold's weights are evaluated unchanged on its excluded samples. Only after cross-validation does the report @@ -238,9 +240,17 @@ Identifier-shape and query-length slopes range from -1 to 1 so one query signal while damping another. This bounded coarse-to-fine search avoids the combinatorial explosion of a full 20-parameter product; it is deterministic but does not claim a global optimum. Exact metric ties prefer the lower-coefficient candidate. Each objective must remain within the configured 1% development -guardrail against the current Production router before it can be recommended. -If no candidate satisfies those guardrails, the artifact records `promotionStatus: no-eligible-candidate`; -the best non-eligible candidate remains diagnostic only and must not be promoted. +guardrail against the current Production router before it can be evaluated. Final promotion additionally +requires every configured outer strategy and every aggregate, query-form, and repository partition to +pass. Every failure records its strategy, fold, partition, metric, candidate value, baseline, tolerance, +and delta. If no candidate satisfies those guardrails, the artifact records +`promotionStatus: no-eligible-candidate`; the best non-eligible candidate remains diagnostic only. + +Schema 25 adds deterministic grouped bootstrap intervals for paired candidate-minus-baseline deltas. +It also records exact-selection frequency across outer folds, the number and width of observed +single-coordinate perturbations, epsilon-neighbor fraction, and median/worst holdout drop. The search is +currently deterministic with one seed and restart; those counts are explicit rather than implying +unmeasured restart stability. Each router result also records a deterministic `random-scout` baseline using the same parameter grid and global-scout budget. `Random R@20` and `Random Ctx@4k` show whether the structured search beats that @@ -350,8 +360,9 @@ output size without introducing an LLM or provider-specific tokenizer. Each run writes ignored JSON and Markdown artifacts under `benchmarks/results`. JSON rows retain the repository, revision, language, size, category, difficulty, query form, grouped fold, model, variant, -individual gold ranks, timing, and every metric. Schema 24 adds selectable router strategies while -retaining shared candidate-queue lifecycle and +individual gold ranks, timing, and every metric. Schema 25 derives promotion from excluded folds and adds +exact blockers, grouped bootstrap uncertainty, and selection-stability evidence. It retains schema 24's +selectable router strategies, shared candidate-queue lifecycle, and per-router candidate-pool initialization timings. Each artifact stores each authored query and its exact file-qualified ground truth once, records productive Sparse timings, and adds the fixed equal-weight RRF baseline. The Markdown report includes quality by query form, diff --git a/benchmarks/retrieval/evaluation/promotion-evidence.ts b/benchmarks/retrieval/evaluation/promotion-evidence.ts new file mode 100644 index 0000000..7a892bf --- /dev/null +++ b/benchmarks/retrieval/evaluation/promotion-evidence.ts @@ -0,0 +1,262 @@ +import type { EvidenceRouterConfig } from "../../../src/domain/retrieval.js" +import type { + CandidateStability, + EvidenceRouterSearchResult, + GuardrailBlocker, + HoldoutUncertainty, + PromotionEvidence, + QualityMetric, + QualitySummary, + RouterObjective, + ValidationStrategy, +} from "./types.js" + +const BOOTSTRAP_SAMPLES = 1_000 +const STABILITY_EPSILON = 0.005 +const QUALITY_METRICS: readonly QualityMetric[] = [ + "recallAt5", + "recallAt10", + "recallAt20", + "recallAt50", + "contextRecallAt4096", + "meanReciprocalRank", +] + +/** Excluded-fold fields needed to derive promotion evidence. */ +export type PromotionHoldoutRow = Pick< + EvidenceRouterSearchResult, + | "model" + | "fusion" + | "objective" + | "strategy" + | "fold" + | "validation" + | "productionValidation" + | "config" + | "holdoutBreakdown" +> + +const precise = (value: number): number => Number(value.toFixed(12)) + +/** Describe every metric that falls below its baseline after applying the configured tolerance. */ +export const buildGuardrailBlockers = ( + partition: GuardrailBlocker["partition"], + name: string, + candidate: QualitySummary, + baseline: QualitySummary, + metrics: readonly QualityMetric[], + tolerance: number, +): readonly GuardrailBlocker[] => + metrics.flatMap((metric) => { + const delta = precise(candidate[metric] - baseline[metric]) + return delta < -tolerance + ? [ + { + partition, + name, + metric, + candidateValue: candidate[metric], + baselineValue: baseline[metric], + tolerance, + delta, + }, + ] + : [] + }) + +const percentile = (values: readonly number[], fraction: number): number => { + if (values.length === 0) return 0 + const sorted = [...values].sort((left, right) => left - right) + return sorted[Math.min(sorted.length - 1, Math.floor(fraction * sorted.length))] ?? 0 +} + +const bootstrapInterval = ( + deltas: readonly number[], +): Pick => { + if (deltas.length === 0) + return { meanDelta: 0, lowerBound: 0, upperBound: 0, bootstrapSamples: 0 } + let state = 1 + const means = Array.from({ length: BOOTSTRAP_SAMPLES }, () => { + let total = 0 + for (let index = 0; index < deltas.length; index++) { + state = (Math.imul(state, 1_664_525) + 1_013_904_223) >>> 0 + total += deltas[state % deltas.length] ?? 0 + } + return total / deltas.length + }) + return { + meanDelta: precise(deltas.reduce((sum, delta) => sum + delta, 0) / deltas.length), + lowerBound: precise(percentile(means, 0.025)), + upperBound: precise(percentile(means, 0.975)), + bootstrapSamples: BOOTSTRAP_SAMPLES, + } +} + +const buildUncertainty = (rows: readonly PromotionHoldoutRow[]): readonly HoldoutUncertainty[] => { + const observations = new Map< + string, + { + strategy: ValidationStrategy + partition: GuardrailBlocker["partition"] + name: string + metric: QualityMetric + deltas: number[] + } + >() + for (const row of rows) + for (const holdout of row.holdoutBreakdown) + for (const metric of QUALITY_METRICS) { + const key = `${row.strategy}\0${holdout.dimension}\0${holdout.name}\0${metric}` + const observation = observations.get(key) ?? { + strategy: row.strategy, + partition: holdout.dimension, + name: holdout.name, + metric, + deltas: [], + } + observation.deltas.push(holdout.candidate[metric] - holdout.baseline[metric]) + observations.set(key, observation) + } + return [...observations.values()].map(({ deltas, ...observation }) => ({ + ...observation, + ...bootstrapInterval(deltas), + })) +} + +const objectiveMetric = (objective: RouterObjective): QualityMetric => { + if (objective === "direct") return "recallAt5" + return objective === "reranker-top20" ? "recallAt20" : "recallAt50" +} + +const configValues = (config: EvidenceRouterConfig): readonly number[] => [ + ...Object.values(config.baseWeights), + ...Object.values(config.scoreInfluence), + ...Object.values(config.geometryInfluence), + ...Object.values(config.termCoverageInfluence), + ...Object.values(config.pairwiseAgreementInfluence), + ...Object.values(config.denseConfidenceInfluence), + ...Object.values(config.identifierInfluence), + ...Object.values(config.queryLengthInfluence), +] + +const differingCoordinates = ( + left: EvidenceRouterConfig, + right: EvidenceRouterConfig, +): { readonly count: number; readonly width: number } => { + const leftValues = configValues(left) + const rightValues = configValues(right) + let count = 0 + let width = 0 + for (let index = 0; index < leftValues.length; index++) { + const difference = Math.abs((leftValues[index] ?? 0) - (rightValues[index] ?? 0)) + if (difference > 0) { + count++ + width = Math.max(width, difference) + } + } + return { count, width } +} + +const selectionFrequency = ( + rows: readonly PromotionHoldoutRow[], +): Pick => { + const frequencies = new Map() + for (const row of rows) { + const key = JSON.stringify(row.config) + frequencies.set(key, (frequencies.get(key) ?? 0) + 1) + } + return { + distinctSelections: frequencies.size, + selectionFrequency: rows.length === 0 ? 0 : Math.max(0, ...frequencies.values()) / rows.length, + } +} + +const localPerturbations = ( + rows: readonly PromotionHoldoutRow[], +): Pick => { + const neighbors: Array<{ readonly qualityDifference: number; readonly width: number }> = [] + for (let leftIndex = 0; leftIndex < rows.length; leftIndex++) + for (let rightIndex = leftIndex + 1; rightIndex < rows.length; rightIndex++) { + const left = rows[leftIndex] + const right = rows[rightIndex] + if (left === undefined || right === undefined) continue + const distance = differingCoordinates(left.config, right.config) + if (distance.count !== 1) continue + const metric = objectiveMetric(left.objective) + neighbors.push({ + qualityDifference: Math.abs(left.validation[metric] - right.validation[metric]), + width: distance.width, + }) + } + const epsilonNeighbors = neighbors.filter( + ({ qualityDifference }) => qualityDifference <= STABILITY_EPSILON, + ) + return { + localPerturbations: neighbors.length, + plateauWidth: precise(Math.max(0, ...epsilonNeighbors.map(({ width }) => width))), + epsilonNeighborFraction: + neighbors.length === 0 ? 0 : epsilonNeighbors.length / neighbors.length, + } +} + +const buildStability = (rows: readonly PromotionHoldoutRow[]): CandidateStability => { + const metric = rows[0] === undefined ? "recallAt20" : objectiveMetric(rows[0].objective) + const holdoutDrops = rows.map((row) => row.productionValidation[metric] - row.validation[metric]) + return { + folds: rows.length, + ...selectionFrequency(rows), + ...localPerturbations(rows), + medianHoldoutDrop: precise(percentile(holdoutDrops, 0.5)), + worstCaseHoldoutDrop: precise(Math.max(0, ...holdoutDrops)), + seeds: 1, + restarts: 1, + } +} + +const promotionKey = (row: PromotionHoldoutRow): string => + `${row.model}\0${row.fusion}\0${row.objective}` + +/** Derive promotion decisions exclusively from excluded-fold results, never fit-all quality. */ +export const derivePromotionEvidence = ( + rows: readonly PromotionHoldoutRow[], + expectedStrategies: readonly ValidationStrategy[], + finalTest: { readonly strategy: ValidationStrategy; readonly fold: string }, +): readonly PromotionEvidence[] => { + const groups = new Map() + for (const row of rows) + groups.set(promotionKey(row), [...(groups.get(promotionKey(row)) ?? []), row]) + return [...groups.values()].map((group) => { + const first = group[0] + if (first === undefined) throw new Error("Promotion evidence group cannot be empty") + const strategies = new Set(group.map((row) => row.strategy)) + const missingStrategies = expectedStrategies.filter((strategy) => !strategies.has(strategy)) + const finalTestRow = group.find( + (row) => row.strategy === finalTest.strategy && row.fold === finalTest.fold, + ) + const finalTestGuardrailsMet = + finalTestRow?.holdoutBreakdown.every((holdout) => holdout.guardrailsMet) ?? false + const blockers = group.flatMap((row) => + row.holdoutBreakdown.flatMap((holdout) => + holdout.blockers.map((blocker) => ({ ...blocker, strategy: row.strategy, fold: row.fold })), + ), + ) + return { + model: first.model, + fusion: first.fusion, + objective: first.objective, + promotionStatus: + missingStrategies.length === 0 && finalTestGuardrailsMet && blockers.length === 0 + ? "eligible" + : "no-eligible-candidate", + missingStrategies, + finalTest: { + ...finalTest, + present: finalTestRow !== undefined, + guardrailsMet: finalTestGuardrailsMet, + }, + blockers, + uncertainty: buildUncertainty(group), + stability: buildStability(group), + } + }) +} diff --git a/benchmarks/retrieval/evaluation/report.ts b/benchmarks/retrieval/evaluation/report.ts index 6da5715..3285f96 100644 --- a/benchmarks/retrieval/evaluation/report.ts +++ b/benchmarks/retrieval/evaluation/report.ts @@ -62,6 +62,51 @@ const formatRouterWeightColumns = (result: { influences: formatInfluences(result.config), }) +const renderPromotionEvidence = (artifact: BenchmarkArtifact): readonly string[] => { + const summaries = artifact.promotionEvidence.map( + (evidence) => + `| ${evidence.model} | ${evidence.fusion} | ${evidence.objective} | ${promotionLabel(evidence.promotionStatus)} | ${evidence.missingStrategies.join(", ") || "none"} | ${evidence.finalTest.strategy}:${evidence.finalTest.fold} (${evidence.finalTest.guardrailsMet ? "pass" : "fail"}) | ${evidence.stability.folds} | ${evidence.stability.distinctSelections} | ${percent(evidence.stability.selectionFrequency)} | ${evidence.stability.localPerturbations} | ${evidence.stability.plateauWidth.toFixed(3)} | ${percent(evidence.stability.epsilonNeighborFraction)} | ${percent(evidence.stability.medianHoldoutDrop)} | ${percent(evidence.stability.worstCaseHoldoutDrop)} |`, + ) + const blockers = artifact.promotionEvidence.flatMap((evidence) => + evidence.blockers.map( + (blocker) => + `| ${evidence.model} | ${evidence.fusion} | ${evidence.objective} | ${blocker.strategy} | ${blocker.fold} | ${blocker.partition}:${blocker.name} | ${blocker.metric} | ${percent(blocker.candidateValue)} | ${percent(blocker.baselineValue)} | ${percent(blocker.tolerance)} | ${percent(blocker.delta)} |`, + ), + ) + const uncertainty = artifact.promotionEvidence.flatMap((evidence) => + evidence.uncertainty.map( + (interval) => + `| ${evidence.model} | ${evidence.fusion} | ${evidence.objective} | ${interval.strategy} | ${interval.partition}:${interval.name} | ${interval.metric} | ${percent(interval.meanDelta)} | ${percent(interval.lowerBound)} | ${percent(interval.upperBound)} | ${interval.bootstrapSamples} |`, + ), + ) + return [ + "", + "## Promotion Evidence", + "", + "Fit-all quality is diagnostic. Promotion status below is derived only from excluded grouped and repository holdouts; missing required strategies block promotion.", + "", + "| Model | Fusion | Objective | Promotion | Missing strategies | Final test | Folds | Distinct selections | Selection frequency | Local perturbations | Plateau width | Epsilon neighbors | Median drop | Worst drop |", + "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ...summaries, + "", + "### Guardrail Blockers", + "", + "| Model | Fusion | Objective | Strategy | Fold | Partition | Metric | Candidate | Baseline | Tolerance | Delta |", + "| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: |", + ...(blockers.length === 0 + ? ["| - | - | - | - | - | - | - | - | - | - | no blockers |"] + : blockers), + "", + "### Holdout Uncertainty", + "", + "Deterministic grouped bootstrap intervals resample excluded folds and report paired candidate-minus-baseline deltas.", + "", + "| Model | Fusion | Objective | Strategy | Partition | Metric | Mean delta | 95% lower | 95% upper | Bootstrap samples |", + "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: |", + ...uncertainty, + ] +} + /** Render quality and marginal channel contribution grouped by query representation. */ export const renderMarkdownReport = (artifact: BenchmarkArtifact): string => { const groups = new Map() @@ -92,7 +137,7 @@ export const renderMarkdownReport = (artifact: BenchmarkArtifact): string => { .map(([kind, weight]) => `${kind}=${weight}`) .join(", ")}.`, "", - `Validation protocol: candidates use ${artifact.validationProtocol.selection}; holdouts are ${artifact.validationProtocol.holdouts.join(" and ")}; final promotion requires the recorded ${artifact.validationProtocol.finalTest}.`, + `Validation protocol: candidates use ${artifact.validationProtocol.selection}; holdouts are ${artifact.validationProtocol.holdouts.join(" and ")}; final promotion requires untouched ${artifact.validationProtocol.finalTest.strategy} fold ${artifact.validationProtocol.finalTest.fold}.`, "", `Search strategy: \`${artifact.searchStrategy.algorithm}\` (${artifact.searchStrategy.globalScouts} global scouts, beam ${artifact.searchStrategy.beamWidth}, ${artifact.searchStrategy.coordinatePasses} coordinate passes, ${artifact.searchStrategy.proxySampleFraction * 100}% proxy with minimum ${artifact.searchStrategy.proxyMinimumSamples}, ${strategyFactorLabel} factor ${strategyFactor}x).`, "", @@ -323,6 +368,8 @@ export const renderMarkdownReport = (artifact: BenchmarkArtifact): string => { ) } + lines.push(...renderPromotionEvidence(artifact)) + lines.push( "", "## Recommended Evidence Router", diff --git a/benchmarks/retrieval/evaluation/search.ts b/benchmarks/retrieval/evaluation/search.ts index 3610d8c..910b497 100644 --- a/benchmarks/retrieval/evaluation/search.ts +++ b/benchmarks/retrieval/evaluation/search.ts @@ -8,10 +8,12 @@ import { type CandidateEvaluationQueue, } from "../execution/candidate-evaluation-pool.js" import type { OptimizationProfile } from "./optimization-profiles.js" +import { derivePromotionEvidence } from "./promotion-evidence.js" import type { BenchmarkArtifact, EvidenceRouterSearchResult, RecommendedEvidenceRouter, + PromotionStatus, ValidationStrategy, } from "./types.js" import { @@ -49,6 +51,7 @@ export interface BenchmarkSearchResults { readonly recommendedFusionWeights: readonly BenchmarkArtifact["recommendedFusionWeights"][number][] readonly evidenceRouterSearch: readonly BenchmarkArtifact["evidenceRouterSearch"][number][] readonly recommendedEvidenceRouters: readonly BenchmarkArtifact["recommendedEvidenceRouters"][number][] + readonly promotionEvidence: readonly BenchmarkArtifact["promotionEvidence"][number][] readonly weightSearchDurationMs: number readonly fusionSearchDurationMs: number readonly evidenceRouterSearchDurationMs: number @@ -398,10 +401,33 @@ const runFusionSearchStage = ( searchOptions, )), ) + const hasRepositoryHoldouts = [...samplesByModel.values()].some( + (modelSamples) => new Set(modelSamples.map((sample) => sample.repository)).size > 1, + ) + const expectedStrategies: readonly ValidationStrategy[] = + config.repositoryHoldouts && hasRepositoryHoldouts + ? [groupedStrategy, "leave-one-repository-out"] + : [groupedStrategy] + const validatedRecommendations = recommendedFusionWeights.map((recommendation) => { + const holdouts = fusionSearch.filter( + (row) => row.model === recommendation.model && row.fusion === recommendation.fusion, + ) + const strategies = new Set(holdouts.map((row) => row.strategy)) + const eligible = + expectedStrategies.every((strategy) => strategies.has(strategy)) && + holdouts.length > 0 && + holdouts.every((row) => row.holdoutBreakdown.every((partition) => partition.guardrailsMet)) + const promotionStatus: PromotionStatus = eligible ? "eligible" : "no-eligible-candidate" + return { + ...recommendation, + guardrailsMet: eligible, + promotionStatus, + } + }) return { productionRouterSearch, fusionSearch, - recommendedFusionWeights, + recommendedFusionWeights: validatedRecommendations, fusionSearchDurationMs: performance.now() - startedAt, } }) @@ -451,6 +477,7 @@ const runEvidenceRouterSearchStage = ( BenchmarkSearchResults, | "evidenceRouterSearch" | "recommendedEvidenceRouters" + | "promotionEvidence" | "evidenceRouterSearchDurationMs" | "candidateQueueStartupDurationMs" | "candidateQueueShutdownDurationMs" @@ -509,9 +536,35 @@ const runEvidenceRouterSearchStage = ( if (result.kind === "holdout") evidenceRouterSearch.push(...result.results) else recommendedEvidenceRouters.push(...result.results) } + const expectedStrategies: readonly ValidationStrategy[] = + config.repositoryHoldouts && + [...samplesByModel.values()].some( + (samples) => new Set(samples.map((sample) => sample.repository)).size > 1, + ) + ? [groupedStrategy, "leave-one-repository-out"] + : [groupedStrategy] + const promotionEvidence = derivePromotionEvidence(evidenceRouterSearch, expectedStrategies, { + strategy: groupedStrategy, + fold: String(config.groupedFolds), + }) + const validatedRecommendations = recommendedEvidenceRouters.map((recommendation) => { + const evidence = promotionEvidence.find( + (row) => + row.model === recommendation.model && + row.fusion === recommendation.fusion && + row.objective === recommendation.objective, + ) + const promotionStatus = evidence?.promotionStatus ?? "no-eligible-candidate" + return { + ...recommendation, + guardrailsMet: promotionStatus === "eligible", + promotionStatus, + } + }) return { evidenceRouterSearch, - recommendedEvidenceRouters, + recommendedEvidenceRouters: validatedRecommendations, + promotionEvidence, evidenceRouterSearchDurationMs: performance.now() - startedAt, candidateQueueStartupDurationMs, candidateQueueShutdownDurationMs, diff --git a/benchmarks/retrieval/evaluation/types.ts b/benchmarks/retrieval/evaluation/types.ts index bdc4a3b..513671e 100644 --- a/benchmarks/retrieval/evaluation/types.ts +++ b/benchmarks/retrieval/evaluation/types.ts @@ -172,17 +172,78 @@ export interface QualitySummary { readonly meanReciprocalRank: number } +/** Quality field that can participate in candidate selection or deployment guardrails. */ +export type QualityMetric = keyof QualitySummary + /** Whether a benchmark candidate is eligible for production promotion. */ export type PromotionStatus = "eligible" | "no-eligible-candidate" +/** Exact reason why one candidate failed a deployment guardrail. */ +export interface GuardrailBlocker { + readonly partition: "aggregate" | "query-form" | "repository" + readonly name: string + readonly metric: QualityMetric + readonly candidateValue: number + readonly baselineValue: number + readonly tolerance: number + readonly delta: number +} + /** Quality and guardrail outcome for one query-form or repository holdout partition. */ export interface HoldoutQuality { - readonly dimension: "query-form" | "repository" + readonly dimension: GuardrailBlocker["partition"] readonly name: string readonly queries: number readonly candidate: QualitySummary readonly baseline: QualitySummary readonly guardrailsMet: boolean + readonly blockers: readonly GuardrailBlocker[] +} + +/** Bootstrap interval for a paired candidate-minus-baseline holdout delta. */ +export interface HoldoutUncertainty { + readonly strategy: ValidationStrategy + readonly partition: GuardrailBlocker["partition"] + readonly name: string + readonly metric: QualityMetric + readonly meanDelta: number + readonly lowerBound: number + readonly upperBound: number + readonly bootstrapSamples: number +} + +/** Empirical robustness of independently selected candidates across excluded folds. */ +export interface CandidateStability { + readonly folds: number + readonly distinctSelections: number + readonly selectionFrequency: number + readonly localPerturbations: number + readonly plateauWidth: number + readonly epsilonNeighborFraction: number + readonly medianHoldoutDrop: number + readonly worstCaseHoldoutDrop: number + readonly seeds: number + readonly restarts: number +} + +/** Holdout-only decision attached to a diagnostic fit-all router candidate. */ +export interface PromotionEvidence { + readonly model: string + readonly fusion: FusionMethod + readonly objective: RouterObjective + readonly promotionStatus: PromotionStatus + readonly missingStrategies: readonly ValidationStrategy[] + readonly finalTest: { + readonly strategy: ValidationStrategy + readonly fold: string + readonly present: boolean + readonly guardrailsMet: boolean + } + readonly blockers: ReadonlyArray< + GuardrailBlocker & { readonly strategy: ValidationStrategy; readonly fold: string } + > + readonly uncertainty: readonly HoldoutUncertainty[] + readonly stability: CandidateStability } /** One cross-validation fold with weights selected without its validation samples. */ @@ -295,9 +356,11 @@ export interface SearchBaselineComparison { export interface ValidationProtocol { readonly selection: "development-only" readonly holdouts: readonly ValidationStrategy[] - readonly finalTest: "nested-cross-validation-plan" - readonly nestedOuterFolds: number - readonly nestedInnerFolds: number + readonly finalTest: { + readonly kind: "untouched-grouped-fold" + readonly strategy: ValidationStrategy + readonly fold: string + } } /** One holdout evaluation of a router selected from query and channel evidence. */ @@ -361,7 +424,7 @@ export interface BenchmarkTimings { /** Reproducible machine-readable output of one complete benchmark run. */ export interface BenchmarkArtifact { - readonly schemaVersion: 24 + readonly schemaVersion: 25 /** Profile controlling benchmark coverage without changing retrieval behavior. */ readonly benchmarkProfile: BenchmarkProfile /** Versioned objective profile used for candidate selection and aggregate metrics. */ @@ -415,4 +478,5 @@ export interface BenchmarkArtifact { readonly recommendedFusionWeights: readonly RecommendedFusionWeights[] readonly evidenceRouterSearch: readonly EvidenceRouterSearchResult[] readonly recommendedEvidenceRouters: readonly RecommendedEvidenceRouter[] + readonly promotionEvidence: readonly PromotionEvidence[] } diff --git a/benchmarks/retrieval/evaluation/weight-search.ts b/benchmarks/retrieval/evaluation/weight-search.ts index 6e693d5..d6e0125 100644 --- a/benchmarks/retrieval/evaluation/weight-search.ts +++ b/benchmarks/retrieval/evaluation/weight-search.ts @@ -31,6 +31,7 @@ import { import { contextRecallAtBudget, recallAt, reciprocalRank } from "./metrics.js" import { SEARCH_PRIORITY_PROFILE, type OptimizationProfile } from "./optimization-profiles.js" import { prepareFusion, type PreparedFusionEvaluator } from "./prepared-fusion.js" +import { buildGuardrailBlockers } from "./promotion-evidence.js" import { ROUTER_OBJECTIVES, ROUTER_SEARCH_STRATEGY, @@ -41,6 +42,7 @@ import { type HoldoutQuality, type ProductionRouterSearchResult, type PromotionStatus, + type QualityMetric, type QualitySummary, type QueryKind, type RecommendedEvidenceRouter, @@ -362,14 +364,6 @@ const buildProxySamples = ( .map(({ sample }) => sample) } -type QualityMetric = - | "recallAt5" - | "recallAt10" - | "recallAt20" - | "recallAt50" - | "contextRecallAt4096" - | "meanReciprocalRank" - const OBJECTIVE_PRIORITIES: Readonly> = { direct: [ "recallAt5", @@ -412,8 +406,10 @@ const unweightedProfile = (profile: OptimizationProfile): OptimizationProfile => queryFormWeights: { identifier: 1, agentTask: 1, naturalQuestion: 1, searchPhrase: 1 }, }) +type GuardrailDimension = Exclude + interface GuardrailPartition { - readonly dimension: HoldoutQuality["dimension"] + readonly dimension: GuardrailDimension readonly name: string readonly samples: readonly WeightSearchSample[] } @@ -421,27 +417,24 @@ interface GuardrailPartition { const guardrailPartitions = ( samples: readonly WeightSearchSample[], ): readonly GuardrailPartition[] => { - const partitions = new Map() + const partitions = new Map>() for (const sample of samples) { - const groups = [ - { dimension: "query-form" as const, name: sample.queryKind }, - { dimension: "repository" as const, name: sample.repository }, + const groups: readonly { readonly dimension: GuardrailDimension; readonly name: string }[] = [ + { dimension: "query-form", name: sample.queryKind }, + { dimension: "repository", name: sample.repository }, ] for (const group of groups) { - const key = `${group.dimension}:${group.name}` - const partition = partitions.get(key) ?? [] + const namedPartitions = + partitions.get(group.dimension) ?? new Map() + const partition = namedPartitions.get(group.name) ?? [] partition.push(sample) - partitions.set(key, partition) + namedPartitions.set(group.name, partition) + partitions.set(group.dimension, namedPartitions) } } - return [...partitions].map(([key, partition]) => { - const separator = key.indexOf(":") - return { - dimension: key.slice(0, separator) as GuardrailPartition["dimension"], - name: key.slice(separator + 1), - samples: partition, - } - }) + return [...partitions].flatMap(([dimension, namedPartitions]) => + [...namedPartitions].map(([name, partition]) => ({ dimension, name, samples: partition })), + ) } const buildGuardrailBaselines = ( @@ -460,19 +453,37 @@ const buildGuardrailBaselines = ( const buildHoldoutBreakdown = ( samples: readonly WeightSearchSample[], - candidate: (samples: readonly WeightSearchSample[]) => QualitySummary, + candidate: ( + samples: readonly WeightSearchSample[], + evaluationProfile: OptimizationProfile, + ) => QualitySummary, profile: OptimizationProfile, ): readonly HoldoutQuality[] => { const holdoutProfile = unweightedProfile(profile) - return buildGuardrailBaselines(samples, profile).partitions.map((partition) => { - const candidateQuality = candidate(partition.samples) + const baselines = buildGuardrailBaselines(samples, profile) + const partitions: readonly GuardrailBaselinePartition[] = [ + { dimension: "aggregate", name: "all", samples, baseline: baselines.overall }, + ...baselines.partitions, + ] + return partitions.map((partition) => { + const evaluationProfile = partition.dimension === "aggregate" ? profile : holdoutProfile + const candidateQuality = candidate(partition.samples, evaluationProfile) + const blockers = buildGuardrailBlockers( + partition.dimension, + partition.name, + candidateQuality, + partition.baseline, + evaluationProfile.metricObjective.guardrailMetrics, + evaluationProfile.metricObjective.guardrailTolerance, + ) return { dimension: partition.dimension, name: partition.name, queries: partition.samples.length, candidate: candidateQuality, baseline: partition.baseline, - guardrailsMet: isWithinGuardrails(candidateQuality, partition.baseline, holdoutProfile), + guardrailsMet: blockers.length === 0, + blockers, } }) } @@ -1645,16 +1656,14 @@ const optimizeFusionWeightsWithPool = async ( pool: CandidateEvaluationPool, ): Promise => { const selected = await selectBestWeights(pool, "reranker-top20", undefined, profile) - const guardrailBaselines = buildGuardrailBaselines(development, profile) - const holdoutProfile = unweightedProfile(profile) const validationQuality = summarize(validationSamples, selected.weights, fusion, profile) - const guardrailsMet = fusionGuardrailsMet( - development, - selected.weights, - fusion, - guardrailBaselines, + const holdoutBreakdown = buildHoldoutBreakdown( + validationSamples, + (partition, evaluationProfile) => + summarize(partition, selected.weights, fusion, evaluationProfile), profile, ) + const guardrailsMet = holdoutBreakdown.every((holdout) => holdout.guardrailsMet) return { model, fusion, @@ -1667,11 +1676,7 @@ const optimizeFusionWeightsWithPool = async ( validation: validationQuality, guardrailsMet, promotionStatus: guardrailsMet ? "eligible" : "no-eligible-candidate", - holdoutBreakdown: buildHoldoutBreakdown( - validationSamples, - (partition) => summarize(partition, selected.weights, fusion, holdoutProfile), - profile, - ), + holdoutBreakdown, } } @@ -1866,7 +1871,6 @@ export const optimizeEvidenceRouter = async ( withEvidencePools(development, fusion, profile, options, async (dynamicSelection, fullPool) => { const productionValidation = summarizeProductionRouter(validation, profile) const validationEvidence = prepareEvidenceSamples(validation) - const holdoutProfile = unweightedProfile(profile) const randomBaseline: SearchBaselineComparison = dynamicSelection.randomCandidates === 0 ? { @@ -1903,40 +1907,49 @@ export const optimizeEvidenceRouter = async ( options, ) const results: EvidenceRouterSearchResult[] = staticSelections.map( - ({ selection, staticSelection }) => ({ - model, - fusion, - objective: selection.objective, - strategy, - fold, - developmentQueries: development.length, - validationQueries: validation.length, - staticWeights: staticSelection.weights, - config: benchmarkRouterConfig(fusion, selection.config), - staticDevelopment: staticSelection.quality, - staticValidation: summarize(validation, staticSelection.weights, fusion, profile), - development: selection.quality, - validation: summarizeEvidenceRouter(validationEvidence, selection.config, fusion, profile), - productionDevelopment: dynamicSelection.productionQuality, - productionValidation, - guardrailsMet: selection.guardrailsMet, - promotionStatus: selection.promotionStatus, - proxyEvaluations: dynamicSelection.proxyEvaluations, - fullEvaluations: dynamicSelection.fullEvaluations, - searchDiagnostics: dynamicSelection.searchDiagnostics, - searchBaseline: randomBaseline, - holdoutBreakdown: buildHoldoutBreakdown( + ({ selection, staticSelection }) => { + const holdoutBreakdown = buildHoldoutBreakdown( validation, - (partition) => + (partition, evaluationProfile) => summarizeEvidenceRouter( prepareEvidenceSamples(partition), selection.config, fusion, - holdoutProfile, + evaluationProfile, ), profile, - ), - }), + ) + const guardrailsMet = holdoutBreakdown.every((holdout) => holdout.guardrailsMet) + return { + model, + fusion, + objective: selection.objective, + strategy, + fold, + developmentQueries: development.length, + validationQueries: validation.length, + staticWeights: staticSelection.weights, + config: benchmarkRouterConfig(fusion, selection.config), + staticDevelopment: staticSelection.quality, + staticValidation: summarize(validation, staticSelection.weights, fusion, profile), + development: selection.quality, + validation: summarizeEvidenceRouter( + validationEvidence, + selection.config, + fusion, + profile, + ), + productionDevelopment: dynamicSelection.productionQuality, + productionValidation, + guardrailsMet, + promotionStatus: guardrailsMet ? "eligible" : "no-eligible-candidate", + proxyEvaluations: dynamicSelection.proxyEvaluations, + fullEvaluations: dynamicSelection.fullEvaluations, + searchDiagnostics: dynamicSelection.searchDiagnostics, + searchBaseline: randomBaseline, + holdoutBreakdown, + } + }, ) return results }) diff --git a/benchmarks/retrieval/runner.ts b/benchmarks/retrieval/runner.ts index 5036884..2b68326 100644 --- a/benchmarks/retrieval/runner.ts +++ b/benchmarks/retrieval/runner.ts @@ -21,7 +21,6 @@ import { type BenchmarkArtifact, type BenchmarkProfile, type CorpusManifest, - type RouterSearchStrategy, type RouterSearchStrategyName, type ValidationStrategy, } from "./evaluation/types.js" @@ -109,29 +108,27 @@ const selectModels = (): Effect.Effect => { const selectOptimizationProfile = (): Effect.Effect => { const requested = process.env.PIX_BENCH_OPTIMIZATION_PROFILE if (requested === undefined) return Effect.succeed(OPTIMIZATION_PROFILES["search-priority"]) - const selected = (OPTIMIZATION_PROFILES as Record)[ - requested - ] + const selected = Object.values(OPTIMIZATION_PROFILES).find( + (profile) => profile.name === requested, + ) return selected === undefined ? Effect.fail(new Error(`Unknown PIX_BENCH_OPTIMIZATION_PROFILE value: ${requested}`)) : Effect.succeed(selected) } +const isRouterSearchStrategyName = (requested: string): requested is RouterSearchStrategyName => + Object.keys(ROUTER_SEARCH_STRATEGIES).some((strategy) => strategy === requested) + export const resolveRouterSearchStrategy = ( requested: string | undefined, ): RouterSearchStrategyName => { if (requested === undefined) return DEFAULT_ROUTER_SEARCH_STRATEGY - if (!Object.hasOwn(ROUTER_SEARCH_STRATEGIES, requested)) { + if (!isRouterSearchStrategyName(requested)) { throw new Error( `Unknown PIX_BENCH_ROUTER_STRATEGY value: ${requested}; expected one of ${Object.keys(ROUTER_SEARCH_STRATEGIES).join(", ")}`, ) } - const selected = (ROUTER_SEARCH_STRATEGIES as Record)[ - requested - ] - if (selected === undefined) - throw new Error(`Router search strategy is not configured: ${requested}`) - return requested as RouterSearchStrategyName + return requested } const selectRouterSearchStrategy = (): Effect.Effect => @@ -213,7 +210,7 @@ export const runRetrievalBenchmark = ( ) const artifact: BenchmarkArtifact = { - schemaVersion: 24, + schemaVersion: 25, benchmarkProfile: profile, optimizationProfile, validationProtocol: { @@ -222,9 +219,11 @@ export const runRetrievalBenchmark = ( config.repositoryHoldouts && repositories.length > 1 ? [groupedStrategy, "leave-one-repository-out"] : [groupedStrategy], - finalTest: "nested-cross-validation-plan", - nestedOuterFolds: config.groupedFolds, - nestedInnerFolds: Math.max(3, config.groupedFolds - 2), + finalTest: { + kind: "untouched-grouped-fold", + strategy: groupedStrategy, + fold: String(config.groupedFolds), + }, }, generatedAt: new Date().toISOString(), searchStrategy: ROUTER_SEARCH_STRATEGIES[routerSearchStrategy], @@ -265,6 +264,7 @@ export const runRetrievalBenchmark = ( recommendedFusionWeights: search.recommendedFusionWeights, evidenceRouterSearch: search.evidenceRouterSearch, recommendedEvidenceRouters: search.recommendedEvidenceRouters, + promotionEvidence: search.promotionEvidence, } const outputPath = yield* writeArtifact(artifact) return { artifact, outputPath } diff --git a/benchmarks/tests/channels.test.ts b/benchmarks/tests/channels.test.ts index d4e2891..9173c44 100644 --- a/benchmarks/tests/channels.test.ts +++ b/benchmarks/tests/channels.test.ts @@ -568,7 +568,12 @@ describe("retrieval benchmark fixture", () => { expect(result.searchBaseline.validation.recallAt20).toBeGreaterThanOrEqual(0) expect( result.holdoutBreakdown.map(({ dimension, name }) => `${dimension}:${name}`).sort(), - ).toEqual(["query-form:naturalQuestion", "query-form:searchPhrase", "repository:fixture"]) + ).toEqual([ + "aggregate:all", + "query-form:naturalQuestion", + "query-form:searchPhrase", + "repository:fixture", + ]) expect( result.holdoutBreakdown.every( ({ candidate, baseline }) => candidate.recallAt20 >= 0 && baseline.recallAt20 >= 0, diff --git a/benchmarks/tests/promotion-evidence.test.ts b/benchmarks/tests/promotion-evidence.test.ts new file mode 100644 index 0000000..d496739 --- /dev/null +++ b/benchmarks/tests/promotion-evidence.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from "@effect/vitest" + +import { PRODUCTION_COMPATIBILITY_CONFIG } from "../../src/domain/retrieval.js" +import { + buildGuardrailBlockers, + derivePromotionEvidence, +} from "../retrieval/evaluation/promotion-evidence.js" +import type { PromotionHoldoutRow } from "../retrieval/evaluation/promotion-evidence.js" +import type { HoldoutQuality, QualitySummary } from "../retrieval/evaluation/types.js" + +const quality = (recallAt20: number): QualitySummary => ({ + recallAt5: recallAt20, + recallAt10: recallAt20, + recallAt20, + recallAt50: recallAt20, + contextRecallAt4096: recallAt20, + meanReciprocalRank: recallAt20, +}) + +const holdout = (candidate: number, baseline: number): HoldoutQuality => ({ + dimension: "repository", + name: "fixture", + queries: 10, + candidate: quality(candidate), + baseline: quality(baseline), + guardrailsMet: candidate >= baseline - 0.01, + blockers: buildGuardrailBlockers( + "repository", + "fixture", + quality(candidate), + quality(baseline), + ["recallAt20"], + 0.01, + ), +}) + +describe("promotion evidence", () => { + it("reports exact blockers for every failed guardrail", () => { + expect( + buildGuardrailBlockers( + "query-form", + "identifier", + quality(0.7), + quality(0.8), + ["recallAt20", "contextRecallAt4096"], + 0.01, + ), + ).toEqual([ + { + partition: "query-form", + name: "identifier", + metric: "recallAt20", + candidateValue: 0.7, + baselineValue: 0.8, + tolerance: 0.01, + delta: -0.1, + }, + { + partition: "query-form", + name: "identifier", + metric: "contextRecallAt4096", + candidateValue: 0.7, + baselineValue: 0.8, + tolerance: 0.01, + delta: -0.1, + }, + ]) + }) + + it("requires every configured excluded strategy and preserves no-eligible-candidate", () => { + const rows: readonly PromotionHoldoutRow[] = [ + { + model: "fixture", + fusion: "dbsf", + objective: "direct", + strategy: "grouped-5-fold", + fold: "1", + validation: quality(0.8), + productionValidation: quality(0.8), + holdoutBreakdown: [holdout(0.8, 0.8)], + config: PRODUCTION_COMPATIBILITY_CONFIG, + }, + ] + + const evidence = derivePromotionEvidence(rows, ["grouped-5-fold", "leave-one-repository-out"], { + strategy: "grouped-5-fold", + fold: "1", + }) + + expect(evidence).toHaveLength(1) + expect(evidence[0]?.promotionStatus).toBe("no-eligible-candidate") + expect(evidence[0]?.missingStrategies).toEqual(["leave-one-repository-out"]) + }) + + it("promotes only blocker-free outer evidence and reports deterministic uncertainty", () => { + const base: Omit = { + model: "fixture", + fusion: "dbsf", + objective: "direct", + validation: quality(0.81), + productionValidation: quality(0.8), + holdoutBreakdown: [holdout(0.81, 0.8)], + config: PRODUCTION_COMPATIBILITY_CONFIG, + } + const rows: readonly PromotionHoldoutRow[] = [ + { ...base, strategy: "grouped-5-fold", fold: "1" }, + { ...base, strategy: "leave-one-repository-out", fold: "fixture" }, + ] + + const evidence = derivePromotionEvidence(rows, ["grouped-5-fold", "leave-one-repository-out"], { + strategy: "grouped-5-fold", + fold: "1", + })[0] + + expect(evidence?.promotionStatus).toBe("eligible") + expect(evidence?.blockers).toEqual([]) + expect(evidence?.finalTest).toEqual({ + strategy: "grouped-5-fold", + fold: "1", + present: true, + guardrailsMet: true, + }) + expect(evidence?.uncertainty.every((interval) => interval.bootstrapSamples === 1_000)).toBe( + true, + ) + expect(evidence?.stability).toMatchObject({ + folds: 2, + distinctSelections: 1, + selectionFrequency: 1, + seeds: 1, + restarts: 1, + }) + }) +}) diff --git a/benchmarks/tests/retrieval.test.ts b/benchmarks/tests/retrieval.test.ts index c7843a4..44e8433 100644 --- a/benchmarks/tests/retrieval.test.ts +++ b/benchmarks/tests/retrieval.test.ts @@ -93,13 +93,17 @@ const runProfile = (profile: BenchmarkProfile, groupedFolds: number, fusionMetho searchPhrase: 4, }) expect(artifact.validationProtocol.selection).toBe("development-only") - expect(artifact.validationProtocol.finalTest).toBe("nested-cross-validation-plan") + expect(artifact.validationProtocol.finalTest).toEqual({ + kind: "untouched-grouped-fold", + strategy: groupedFolds === 3 ? "grouped-3-fold" : "grouped-5-fold", + fold: String(groupedFolds), + }) expect(artifact.repositories.length).toBeGreaterThan(0) expect(artifact.evaluationCases.length).toBeGreaterThan(0) expect(artifact.evaluationCases.every(({ groundTruth }) => groundTruth.length > 0)).toBe(true) expect(artifact.models.length).toBeGreaterThan(0) expect(artifact.measurements.length).toBeGreaterThan(0) - expect(artifact.schemaVersion).toBe(24) + expect(artifact.schemaVersion).toBe(25) expect(artifact.searchStrategy).toEqual( ROUTER_SEARCH_STRATEGIES[resolveRouterSearchStrategy(process.env.PIX_BENCH_ROUTER_STRATEGY)], ) @@ -129,6 +133,9 @@ const runProfile = (profile: BenchmarkProfile, groupedFolds: number, fusionMetho expect(artifact.recommendedEvidenceRouters.length).toBe( artifact.models.length * routerFusionMethods * ROUTER_OBJECTIVES.length, ) + expect(artifact.promotionEvidence.length).toBe( + artifact.models.length * routerFusionMethods * ROUTER_OBJECTIVES.length, + ) expect(artifact.evidenceRouterSearch.every((row) => row.proxyEvaluations >= 0)).toBe(true) expect(artifact.evidenceRouterSearch.every((row) => row.fullEvaluations > 0)).toBe(true) expect(artifact.recommendedEvidenceRouters.every((row) => row.proxyEvaluations >= 0)).toBe(true)