1- import { afterEach , describe , expect , it , mock } from 'bun:test' ;
1+ import { afterEach , describe , expect , it , mock , spyOn } from 'bun:test' ;
22import {
33 existsSync ,
44 mkdirSync ,
@@ -905,6 +905,56 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`,
905905 }
906906 } ) ;
907907
908+ it ( 'disables provider batching when repeat attempts are configured' , async ( ) => {
909+ class BatchCapableProvider implements Provider {
910+ readonly id = 'batch:repeat' ;
911+ readonly kind = 'mock' as const ;
912+ readonly targetName = 'repeat' ;
913+ readonly supportsBatch = true ;
914+ invokeCalls = 0 ;
915+ batchCalls = 0 ;
916+
917+ async invoke ( ) : Promise < ProviderResponse > {
918+ this . invokeCalls += 1 ;
919+ return {
920+ output : [ { role : 'assistant' , content : 'OK' } ] ,
921+ } ;
922+ }
923+
924+ async invokeBatch ( ) : Promise < readonly ProviderResponse [ ] > {
925+ this . batchCalls += 1 ;
926+ throw new Error ( 'batch should not be used for repeat attempts' ) ;
927+ }
928+ }
929+
930+ const provider = new BatchCapableProvider ( ) ;
931+ const warnSpy = spyOn ( console , 'warn' ) . mockImplementation ( ( ) => { } ) ;
932+ try {
933+ const results = await runEvaluation ( {
934+ testFilePath : 'in-memory.yaml' ,
935+ repoRoot : 'in-memory' ,
936+ target : {
937+ ...baseTarget ,
938+ providerBatching : true ,
939+ workers : 1 ,
940+ } ,
941+ providerFactory : ( ) => provider ,
942+ evaluators : evaluatorRegistry ,
943+ evalCases : [ baseTestCase ] ,
944+ trials : { count : 2 , strategy : 'pass_any' } ,
945+ } ) ;
946+
947+ expect ( results ) . toHaveLength ( 1 ) ;
948+ expect ( provider . batchCalls ) . toBe ( 0 ) ;
949+ expect ( provider . invokeCalls ) . toBe ( 2 ) ;
950+ expect ( warnSpy ) . toHaveBeenCalledWith (
951+ 'Warning: Batch mode is disabled when evaluate_options.repeat.count > 1. Using per-case dispatch for attempts.' ,
952+ ) ;
953+ } finally {
954+ warnSpy . mockRestore ( ) ;
955+ }
956+ } ) ;
957+
908958 it ( 'uses a custom evaluator prompt when provided' , async ( ) => {
909959 const directory = mkdtempSync ( path . join ( tmpdir ( ) , 'agentv-custom-grader-' ) ) ;
910960 const promptPath = path . join ( directory , 'grader-prompt.md' ) ;
@@ -1936,7 +1986,7 @@ describe('runEvaluation with trials', () => {
19361986 expect ( result . costLimited ) . toBeUndefined ( ) ;
19371987 } ) ;
19381988
1939- it ( 'disables cache when trials > 1 ' , async ( ) => {
1989+ it ( 'disables cache when repeat attempts are configured ' , async ( ) => {
19401990 const provider = new MultiCallProvider ( ) ;
19411991 const evalRegistry = createScoringEvaluator ( [ 0.5 , 0.9 ] ) ;
19421992 const trials : TrialsConfig = { count : 2 , strategy : 'pass_any' } ;
@@ -1948,21 +1998,29 @@ describe('runEvaluation with trials', () => {
19481998 async set ( ) { } ,
19491999 } ;
19502000
1951- const results = await runEvaluation ( {
1952- testFilePath : 'in-memory.yaml' ,
1953- repoRoot : 'in-memory' ,
1954- target : baseTarget ,
1955- providerFactory : ( ) => provider ,
1956- evaluators : evalRegistry ,
1957- evalCases : [ baseTestCase ] ,
1958- trials,
1959- cache,
1960- useCache : true , // Should be overridden to false
1961- } ) ;
2001+ const warnSpy = spyOn ( console , 'warn' ) . mockImplementation ( ( ) => { } ) ;
2002+ try {
2003+ const results = await runEvaluation ( {
2004+ testFilePath : 'in-memory.yaml' ,
2005+ repoRoot : 'in-memory' ,
2006+ target : baseTarget ,
2007+ providerFactory : ( ) => provider ,
2008+ evaluators : evalRegistry ,
2009+ evalCases : [ baseTestCase ] ,
2010+ trials,
2011+ cache,
2012+ useCache : true , // Should be overridden to false
2013+ } ) ;
19622014
1963- // Provider should have been called for each trial (cache disabled)
1964- expect ( provider . callCount ) . toBe ( 2 ) ;
1965- expect ( results [ 0 ] . trials ) . toHaveLength ( 2 ) ;
2015+ // Provider should have been called for each attempt (cache disabled)
2016+ expect ( provider . callCount ) . toBe ( 2 ) ;
2017+ expect ( results [ 0 ] . trials ) . toHaveLength ( 2 ) ;
2018+ expect ( warnSpy ) . toHaveBeenCalledWith (
2019+ 'Warning: Caching is disabled when evaluate_options.repeat.count > 1 (cached responses would make attempts deterministic).' ,
2020+ ) ;
2021+ } finally {
2022+ warnSpy . mockRestore ( ) ;
2023+ }
19662024 } ) ;
19672025} ) ;
19682026
0 commit comments