diff --git a/CHANGELOG.md b/CHANGELOG.md index 4345c71..f6a22ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ changes, new features. **Patch:** fixes. Images are published to a release is validated. Release process: [runbook](docs/operate/runbook.md#release-a-new-version). Gateway/CLI-only releases (`make release-gateway`) reuse the previous serving images. +## Unreleased + +- `bench-intents`: more than 26 candidate intents run as a 2-stage bracket (groups of ≤ 20, top 5 to a final round) + instead of being rejected by the server (`--dataset banking77` sends 30). + ## v0.1.2 (2026-09-28) Gateway and monitoring; serving images unchanged (v0.1.0). diff --git a/benchmarks/runs/20260928-v010-verification/README.md b/benchmarks/runs/20260928-v010-verification/README.md index f5054da..24cc657 100644 --- a/benchmarks/runs/20260928-v010-verification/README.md +++ b/benchmarks/runs/20260928-v010-verification/README.md @@ -12,7 +12,7 @@ committed receipts in `benchmarks/` (earlier images and hardware) and with the i | `bench-calibration` | 50 | 46 (92%) | 43–46 (image parity) | Pass | | `bench` (triage, code review, security) | 30 | 22 (73.3%) | 23–24 on L4 receipts | Pass (±2 items is noise on 30) | | `bench-intents` clinc150 (small set) | 30 | 29 (96.7%) after tie-break; domain 28/30 | Committed receipt 96.7% | Pass | -| `bench-intents` banking77 (small set) | 30 | 0: every request rejected, `at most 26 alternatives` | Committed receipt (Sep 20, older server) | **Harness issue**, see below | +| `bench-intents` banking77 (small set) | 30 | 0 at first (every request rejected, `at most 26 alternatives`); after the bracket fix: **23 (76.7%)** | Committed receipt 26 (Sep 20, older server allowed 30 options in one question) | Fixed (harness) | | `bench-bbox` | 12 | mIoU 0.406 argmax / 0.422 expectation | 0.290 / 0.377 (Cloud Run L4) | Pass | | `bench-rerank` | 30 queries | nDCG@10 0.824, 0.856, 0.856 (3 runs); 0% ties, 100% poison quarantine | 0.9265 Cloud Run L4, 0.8502 Vertex L4 | Pass (in the range of earlier receipts) | | `bench-permutation` | 16 | 100% accuracy, 6.25% flip rate | EXP-13 | Pass | @@ -21,9 +21,11 @@ committed receipts in `benchmarks/` (earlier images and hardware) and with the i | `bench-ecotone` | — | Not run (needs the C++ ecotone sidecar) | — | Skipped | **Two findings, both pre-existing, not regressions:** -- `bench-intents --dataset banking77` sends all 30 candidate intents in one `choice` question. The server has +- `bench-intents --dataset banking77` sent all 30 candidate intents in one `choice` question. The server has rejected more than 26 options since before v0.1.0 (`ab208dd` had the same check); the committed receipt predates - it. The harness should split into brackets (as `dgem systemone serve` does) or use the 26-option slice. + it. **Fixed:** with more than 26 options the harness now runs a 2-stage bracket (groups of ≤ 20, top 5 of each + group to a final round), like `dgem systemone serve`. Result 23/30 (76.7%); keeping the top 3 per group gave 22, + the top 8 gave 23. The misses are confusable intent pairs (e.g. `card_arrival` vs `card_delivery_estimate`). - `intent_banking77`, `intent_clinc150`, `intent_tiebreak`, `jevbench_generic` and `tn_disambiguation` are harness templates: they take option lists from `bench-intents`, `bench-jev` or `bench-ecotone`, so the gateway's sample variables leave them with fewer than two options (400). The harnesses themselves pass (above). diff --git a/benchmarks/runs/20260928-v010-verification/intents_banking77.json b/benchmarks/runs/20260928-v010-verification/intents_banking77.json new file mode 100644 index 0000000..f9d2790 --- /dev/null +++ b/benchmarks/runs/20260928-v010-verification/intents_banking77.json @@ -0,0 +1,929 @@ +{ + "avg_wall_time_ms": 437.46666666666664, + "candidate_intents": 30, + "corpus": "benchmarks/intents/banking77_eval.jsonl", + "correct_mean_confidence": 0.9737174147042104, + "correct_mean_entropy_nats": 0, + "dataset": "PolyAI/banking77", + "domain_accuracy_pct": 100, + "domain_correct": 30, + "entropy_threshold": 0.08, + "in_domain_correct": 23, + "in_domain_total": 30, + "incorrect_mean_confidence": 0.7484150704072452, + "incorrect_mean_entropy_nats": 0, + "intent_accuracy_pct": 76.66666666666667, + "intent_correct": 23, + "mean_confidence": 0.9211468677015853, + "mean_entropy_nats": 0, + "oos_correct": 0, + "oos_total": 0, + "pass1_accuracy_pct": 76.66666666666667, + "pass1_correct": 23, + "results": [ + { + "id": "b77-01", + "dataset": "PolyAI/banking77", + "text": "How do I locate my card?", + "domain_accurate": true, + "expected_intent": "card_arrival", + "pass1_intent": "card_arrival", + "pass1_accurate": true, + "actual_intent": "card_arrival", + "intent_accurate": true, + "confidence": 0.9767422382163579, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.9767422382163579, + "card_delivery_estimate": 0.010481843580796934, + "card_linking": 0.00021875734804291473, + "card_not_working": 0.00021875734804291473, + "cash_withdrawal_charge": 0.0005089028769875678, + "cash_withdrawal_not_recognised": 0.00006965376605473822, + "declined_card_payment": 0.0009125662042346195, + "declined_cash_withdrawal": 0.00021875734804291473, + "lost_or_stolen_card": 0.010481843580796934, + "pending_top_up": 0.00014667973064252796 + }, + "tie_break_triggered": false, + "wall_time_ms": 485 + }, + { + "id": "b77-02", + "dataset": "PolyAI/banking77", + "text": "Why won't my card show up on the app?", + "domain_accurate": true, + "expected_intent": "card_linking", + "pass1_intent": "card_delivery_estimate", + "pass1_accurate": false, + "actual_intent": "card_delivery_estimate", + "intent_accurate": false, + "confidence": 0.9775238305224455, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_delivery_estimate": 0.9775238305224455, + "card_linking": 0.005861364689241046, + "card_not_working": 0.0017244660956011089, + "cash_withdrawal_not_recognised": 0.000022400813925504044, + "declined_card_payment": 0.0001693208941665372, + "declined_cash_withdrawal": 0.00023556262893857684, + "lost_or_stolen_card": 0.013148989035012599, + "pending_top_up": 0.0001432082649549071, + "pending_transfer": 0.00005539196947050054, + "top_up_by_card_charge": 0.0011154650862435736 + }, + "tie_break_triggered": false, + "wall_time_ms": 487 + }, + { + "id": "b77-03", + "dataset": "PolyAI/banking77", + "text": "I need to know your exchange rates.", + "domain_accurate": true, + "expected_intent": "exchange_rate", + "pass1_intent": "exchange_rate", + "pass1_accurate": true, + "actual_intent": "exchange_rate", + "intent_accurate": true, + "confidence": 0.9991602624671079, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.00006597195685310895, + "card_payment_wrong_exchange_rate": 0.00007775160780851758, + "cash_withdrawal_charge": 0.00004344482728304526, + "compromised_card": 0.00020147163335224058, + "exchange_charge": 0.00011642349519952179, + "exchange_rate": 0.9991602624671079, + "lost_or_stolen_card": 0.0000011309609235602006, + "top_up_by_card_charge": 0.0002945473743536395, + "transfer_timing": 0.00002832470895233014, + "wrong_exchange_rate_for_cash_withdrawal": 0.00001067096816610375 + }, + "tie_break_triggered": false, + "wall_time_ms": 487 + }, + { + "id": "b77-04", + "dataset": "PolyAI/banking77", + "text": "I purchased something in a foreign currency but the rate applied is wrong", + "domain_accurate": true, + "expected_intent": "card_payment_wrong_exchange_rate", + "pass1_intent": "card_payment_wrong_exchange_rate", + "pass1_accurate": true, + "actual_intent": "card_payment_wrong_exchange_rate", + "intent_accurate": true, + "confidence": 0.9273659925584156, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.0001693117042568148, + "card_delivery_estimate": 0.000761762112438536, + "card_payment_wrong_exchange_rate": 0.9273659925584156, + "cash_withdrawal_charge": 0.0001171902414885438, + "cash_withdrawal_not_recognised": 0.000042983711375630526, + "exchange_charge": 0.00004654238481123019, + "exchange_rate": 0.06533187624916952, + "lost_or_stolen_card": 0.0039857413242871, + "top_up_by_bank_transfer_charge": 0.0006694977449986907, + "wrong_exchange_rate_for_cash_withdrawal": 0.0015091019687584284 + }, + "tie_break_triggered": false, + "wall_time_ms": 493 + }, + { + "id": "b77-05", + "dataset": "PolyAI/banking77", + "text": "I need my card now!", + "domain_accurate": true, + "expected_intent": "card_delivery_estimate", + "pass1_intent": "card_arrival", + "pass1_accurate": false, + "actual_intent": "card_arrival", + "intent_accurate": false, + "confidence": 0.6624392515012145, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.6624392515012145, + "card_delivery_estimate": 0.3219419470485434, + "card_linking": 0.0006176557923809675, + "cash_withdrawal_not_recognised": 0.00002765833402391254, + "declined_card_payment": 0.000901214018600799, + "declined_cash_withdrawal": 0.00001587710526874647, + "failed_transfer": 0.003703850211005109, + "lost_or_stolen_card": 0.009673655790886644, + "pending_top_up": 0.00025975478140000786, + "top_up_by_card_charge": 0.0004191354166759651 + }, + "tie_break_triggered": true, + "finalist_options": [ + "card_arrival", + "lost_or_stolen_card", + "card_delivery_estimate" + ], + "wall_time_ms": 511 + }, + { + "id": "b77-06", + "dataset": "PolyAI/banking77", + "text": "I've tried my card a bunch of times and it never worked.", + "domain_accurate": true, + "expected_intent": "card_not_working", + "pass1_intent": "card_not_working", + "pass1_accurate": true, + "actual_intent": "card_not_working", + "intent_accurate": true, + "confidence": 0.9995840750384313, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.00002876990307841974, + "card_not_working": 0.9995840750384313, + "card_payment_fee_charged": 0.0000621336869491999, + "cash_withdrawal_charge": 0.000003646182668689712, + "cash_withdrawal_not_recognised": 0.0000013002042599757285, + "declined_card_payment": 0.00001412236940175228, + "declined_cash_withdrawal": 0.00002209777082072721, + "lost_or_stolen_card": 0.00022542778249999936, + "top_up_by_card_charge": 0.000044304692488060365, + "transfer_timing": 0.00001412236940175228 + }, + "tie_break_triggered": false, + "wall_time_ms": 384 + }, + { + "id": "b77-07", + "dataset": "PolyAI/banking77", + "text": "My entire gym bag, including my wallet, was stolen out of my locker today. Everything in my wallet is gone - how do I block the card to make it can't be used?", + "domain_accurate": true, + "expected_intent": "lost_or_stolen_card", + "pass1_intent": "lost_or_stolen_card", + "pass1_accurate": true, + "actual_intent": "lost_or_stolen_card", + "intent_accurate": true, + "confidence": 0.9973850519521791, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_delivery_estimate": 0.000022991244980069753, + "cash_withdrawal_not_recognised": 0.000005995685905298701, + "compromised_card": 0.002456725413230456, + "declined_card_payment": 0.00004375200868460336, + "declined_cash_withdrawal": 0.0000066162895676933215, + "exchange_charge": 0.0000025567396566468285, + "failed_transfer": 0.00002768572407017153, + "lost_or_stolen_card": 0.9973850519521791, + "top_up_by_bank_transfer_charge": 0.00002768572407017153, + "top_up_by_card_charge": 0.00002093921765575756 + }, + "tie_break_triggered": false, + "wall_time_ms": 382 + }, + { + "id": "b77-08", + "dataset": "PolyAI/banking77", + "text": "how much do you charge to accept transfers", + "domain_accurate": true, + "expected_intent": "top_up_by_bank_transfer_charge", + "pass1_intent": "transfer_fee_charged", + "pass1_accurate": false, + "actual_intent": "transfer_into_account", + "intent_accurate": false, + "confidence": 0.6887420066853087, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_payment_fee_charged": 0.011163756058391445, + "exchange_charge": 0.2747947455552438, + "pending_top_up": 0.000282144319267128, + "pending_transfer": 0.000023607865368768437, + "top_up_by_bank_transfer_charge": 0.002252822336619945, + "top_up_by_card_charge": 0.0019619992298428586, + "top_up_failed": 0.00008155070215258066, + "top_up_limits": 0.000828627928134938, + "transfer_fee_charged": 0.6887420066853087, + "transfer_into_account": 0.019868739319669884 + }, + "tie_break_triggered": true, + "finalist_options": [ + "transfer_fee_charged", + "transfer_into_account", + "card_payment_fee_charged", + "exchange_charge" + ], + "wall_time_ms": 506 + }, + { + "id": "b77-09", + "dataset": "PolyAI/banking77", + "text": "new customer and filling gas today, it's not working, the thing shows pending half an hour!", + "domain_accurate": true, + "expected_intent": "pending_top_up", + "pass1_intent": "pending_top_up", + "pass1_accurate": true, + "actual_intent": "pending_top_up", + "intent_accurate": true, + "confidence": 0.8829457594436773, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_not_working": 0.016814092614878925, + "card_payment_fee_charged": 0.0019419782553519158, + "card_payment_not_recognised": 0.022293675714960357, + "failed_transfer": 0.010512301380903345, + "lost_or_stolen_card": 0.0027409432917415532, + "pending_top_up": 0.8829457594436773, + "pending_transfer": 0.02629563972918325, + "top_up_failed": 0.034389977870664314, + "transfer_fee_charged": 0.0018105543902716809, + "transfer_timing": 0.0002550773083673564 + }, + "tie_break_triggered": true, + "finalist_options": [ + "pending_top_up", + "card_payment_not_recognised", + "top_up_failed", + "pending_transfer", + "card_not_working", + "failed_transfer" + ], + "wall_time_ms": 535 + }, + { + "id": "b77-10", + "dataset": "PolyAI/banking77", + "text": "can you limit my top up?", + "domain_accurate": true, + "expected_intent": "top_up_limits", + "pass1_intent": "top_up_limits", + "pass1_accurate": true, + "actual_intent": "top_up_limits", + "intent_accurate": true, + "confidence": 0.9992176982641682, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_delivery_estimate": 0.00001184270389563741, + "declined_cash_withdrawal": 0.000004770341256238059, + "exchange_charge": 0.0000025865004882794724, + "pending_top_up": 0.000006895571909986137, + "pending_transfer": 0.00001684938197277894, + "top_up_by_bank_transfer_charge": 0.000009904853378904158, + "top_up_by_card_charge": 0.0006142001475403205, + "top_up_failed": 0.00011301168713129296, + "top_up_limits": 0.9992176982641682, + "transfer_fee_charged": 0.000002240548258468251 + }, + "tie_break_triggered": false, + "wall_time_ms": 403 + }, + { + "id": "b77-11", + "dataset": "PolyAI/banking77", + "text": "I payed with a card and was charged an extra fee", + "domain_accurate": true, + "expected_intent": "card_payment_fee_charged", + "pass1_intent": "card_payment_fee_charged", + "pass1_accurate": true, + "actual_intent": "card_payment_fee_charged", + "intent_accurate": true, + "confidence": 0.9342978177987303, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_not_working": 0.0000893356763041434, + "card_payment_fee_charged": 0.9342978177987303, + "cash_withdrawal_charge": 0.0005719089187168078, + "cash_withdrawal_not_recognised": 0.000013307822218950634, + "declined_card_payment": 0.004799113767393752, + "exchange_charge": 0.0006607739592789343, + "lost_or_stolen_card": 0.0004942195428762776, + "top_up_by_bank_transfer_charge": 0.0006148573742882397, + "top_up_by_card_charge": 0.029229332570096297, + "transfer_fee_charged": 0.029229332570096297 + }, + "tie_break_triggered": false, + "wall_time_ms": 412 + }, + { + "id": "b77-12", + "dataset": "PolyAI/banking77", + "text": "I completed an in country transfer a few days ago and was sure to check the account information several times since the transaction was completed, but the funds are still not available. What is the problem?", + "domain_accurate": true, + "expected_intent": "transfer_not_received_by_recipient", + "pass1_intent": "transfer_not_received_by_recipient", + "pass1_accurate": true, + "actual_intent": "transfer_not_received_by_recipient", + "intent_accurate": true, + "confidence": 0.9935491527729811, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "compromised_card": 0.000317383753357623, + "failed_transfer": 0.0002557117878904335, + "pending_transfer": 0.004085212769304482, + "top_up_by_card_charge": 0.0001213036298127259, + "top_up_failed": 0.0002954451049614153, + "top_up_limits": 0.000018763941617830233, + "top_up_reverted": 0.0006802690661429304, + "transfer_fee_charged": 0.00042104538604098086, + "transfer_not_received_by_recipient": 0.9935491527729811, + "transfer_timing": 0.0002557117878904335 + }, + "tie_break_triggered": false, + "wall_time_ms": 407 + }, + { + "id": "b77-13", + "dataset": "PolyAI/banking77", + "text": "My top-up was cancelled; will I receive a refund?", + "domain_accurate": true, + "expected_intent": "top_up_reverted", + "pass1_intent": "top_up_reverted", + "pass1_accurate": true, + "actual_intent": "top_up_reverted", + "intent_accurate": true, + "confidence": 0.990834482055758, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "failed_transfer": 0.000084888555380219, + "lost_or_stolen_card": 0.0005278911837674811, + "pending_top_up": 0.00014248244072583217, + "pending_transfer": 0.00018990439975542016, + "top_up_by_card_charge": 0.00026958958047434, + "top_up_failed": 0.007648321255079772, + "top_up_limits": 0.00014248244072583217, + "top_up_reverted": 0.990834482055758, + "transfer_fee_charged": 0.000006775690717383752, + "transfer_timing": 0.00015318239761573818 + }, + "tie_break_triggered": false, + "wall_time_ms": 395 + }, + { + "id": "b77-14", + "dataset": "PolyAI/banking77", + "text": "I have a strange payment in my statement", + "domain_accurate": true, + "expected_intent": "card_payment_not_recognised", + "pass1_intent": "card_payment_not_recognised", + "pass1_accurate": true, + "actual_intent": "card_payment_not_recognised", + "intent_accurate": true, + "confidence": 0.9997447686894366, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_delivery_estimate": 0.000011947864163334931, + "card_payment_fee_charged": 0.00001086876116094279, + "card_payment_not_recognised": 0.9997447686894366, + "cash_withdrawal_charge": 0.000007064887629474495, + "cash_withdrawal_not_recognised": 0.0000011682845888601324, + "declined_cash_withdrawal": 0.0000014353326691163504, + "lost_or_stolen_card": 0.000036085654218591767, + "pending_top_up": 0.0000012301510488785176, + "top_up_by_card_charge": 0.00018473642923994598, + "wrong_exchange_rate_for_cash_withdrawal": 6.939458442636502e-7 + }, + "tie_break_triggered": false, + "wall_time_ms": 399 + }, + { + "id": "b77-15", + "dataset": "PolyAI/banking77", + "text": "I think someone is using my card to make transactions I don't remember.", + "domain_accurate": true, + "expected_intent": "compromised_card", + "pass1_intent": "compromised_card", + "pass1_accurate": true, + "actual_intent": "compromised_card", + "intent_accurate": true, + "confidence": 0.9873212351755358, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_not_working": 0.0032798582200941053, + "card_payment_not_recognised": 0.001417876008068322, + "cash_withdrawal_charge": 0.000004604861776796166, + "cash_withdrawal_not_recognised": 0.00012494097818499677, + "compromised_card": 0.9873212351755358, + "declined_cash_withdrawal": 0.0009128411405927213, + "failed_transfer": 0.00003068814595965197, + "lost_or_stolen_card": 0.00674133025666207, + "top_up_by_card_charge": 0.0001359370671658038, + "wrong_exchange_rate_for_cash_withdrawal": 0.00003068814595965197 + }, + "tie_break_triggered": false, + "wall_time_ms": 395 + }, + { + "id": "b77-16", + "dataset": "PolyAI/banking77", + "text": "Can you tell me why you won't let me get my own money out of the money machine? I know I used the right PIN.", + "domain_accurate": true, + "expected_intent": "declined_cash_withdrawal", + "pass1_intent": "declined_cash_withdrawal", + "pass1_accurate": true, + "actual_intent": "declined_cash_withdrawal", + "intent_accurate": true, + "confidence": 0.845575994263481, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_not_working": 0.00010848137787777561, + "cash_withdrawal_charge": 0.00007405467142346262, + "cash_withdrawal_not_recognised": 0.000007259278954757665, + "declined_card_payment": 0.15392240137128255, + "declined_cash_withdrawal": 0.845575994263481, + "lost_or_stolen_card": 0.00018208215965150623, + "top_up_by_card_charge": 0.000015848043271896538, + "top_up_failed": 0.00006852883754456372, + "top_up_limits": 0.0000026757706868191495, + "wrong_exchange_rate_for_cash_withdrawal": 0.000042674225825725095 + }, + "tie_break_triggered": true, + "finalist_options": [ + "declined_cash_withdrawal", + "declined_card_payment", + "wrong_exchange_rate_for_cash_withdrawal", + "cash_withdrawal_charge", + "cash_withdrawal_not_recognised" + ], + "wall_time_ms": 519 + }, + { + "id": "b77-17", + "dataset": "PolyAI/banking77", + "text": "My credit card was declined.", + "domain_accurate": true, + "expected_intent": "declined_card_payment", + "pass1_intent": "declined_card_payment", + "pass1_accurate": true, + "actual_intent": "declined_card_payment", + "intent_accurate": true, + "confidence": 0.999486307329621, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_not_working": 0.000150920894259059, + "card_payment_fee_charged": 0.00006798750067398693, + "card_payment_not_recognised": 0.000022709545233256895, + "cash_withdrawal_charge": 0.00001338171200658215, + "cash_withdrawal_not_recognised": 0.0000576871474100067, + "declined_card_payment": 0.999486307329621, + "declined_cash_withdrawal": 0.00011023133118833367, + "lost_or_stolen_card": 0.00007999876333950517, + "top_up_by_card_charge": 0.000007090509241178252, + "wrong_exchange_rate_for_cash_withdrawal": 0.000003685267027186105 + }, + "tie_break_triggered": false, + "wall_time_ms": 384 + }, + { + "id": "b77-18", + "dataset": "PolyAI/banking77", + "text": "I did transfer, why is it pending", + "domain_accurate": true, + "expected_intent": "pending_transfer", + "pass1_intent": "pending_transfer", + "pass1_accurate": true, + "actual_intent": "pending_transfer", + "intent_accurate": true, + "confidence": 0.9581833331018353, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "failed_transfer": 0.0009851006595283317, + "pending_top_up": 0.040534039337472724, + "pending_transfer": 0.9581833331018353, + "top_up_by_bank_transfer_charge": 0.000004286873104961599, + "top_up_by_card_charge": 0.0000010760329158186946, + "top_up_failed": 0.000011579603331234165, + "top_up_limits": 1.7326851857059587e-7, + "top_up_reverted": 0.00004890991223818247, + "transfer_fee_charged": 0.00007309020806233027, + "transfer_timing": 0.00015841100299241714 + }, + "tie_break_triggered": false, + "wall_time_ms": 383 + }, + { + "id": "b77-19", + "dataset": "PolyAI/banking77", + "text": "What do I do if an ATM \"stole\" my card?", + "domain_accurate": true, + "expected_intent": "card_swallowed", + "pass1_intent": "card_swallowed", + "pass1_accurate": true, + "actual_intent": "card_swallowed", + "intent_accurate": true, + "confidence": 0.9972398049761276, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.00037976515007191194, + "card_not_working": 0.0005709568602946767, + "card_swallowed": 0.9972398049761276, + "cash_withdrawal_charge": 0.000019014623180685086, + "cash_withdrawal_not_recognised": 0.000022499771153388697, + "declined_cash_withdrawal": 0.00010276777313859534, + "lost_or_stolen_card": 0.0015850855708055842, + "pending_top_up": 0.00002446037931518864, + "top_up_by_card_charge": 0.00005517406761076227, + "wrong_exchange_rate_for_cash_withdrawal": 4.7082830165918075e-7 + }, + "tie_break_triggered": false, + "wall_time_ms": 378 + }, + { + "id": "b77-20", + "dataset": "PolyAI/banking77", + "text": "What is the number of days I have to wait for my Europe transfer?", + "domain_accurate": true, + "expected_intent": "transfer_timing", + "pass1_intent": "transfer_timing", + "pass1_accurate": true, + "actual_intent": "transfer_timing", + "intent_accurate": true, + "confidence": 0.9994217544916443, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.00004871319599802749, + "card_delivery_estimate": 0.00004486257803560085, + "cash_withdrawal_charge": 0.00003214370469317879, + "failed_transfer": 0.000011377097551421993, + "lost_or_stolen_card": 0.0000023689134642433293, + "pending_transfer": 0.0002694893513339985, + "top_up_by_card_charge": 0.00004486257803560085, + "top_up_limits": 0.00006221404462179313, + "transfer_fee_charged": 0.00006221404462179313, + "transfer_timing": 0.9994217544916443 + }, + "tie_break_triggered": false, + "wall_time_ms": 393 + }, + { + "id": "b77-21", + "dataset": "PolyAI/banking77", + "text": "I've tried numerous times to submit a transfer of funds. Why isn't it going through?", + "domain_accurate": true, + "expected_intent": "beneficiary_not_allowed", + "pass1_intent": "failed_transfer", + "pass1_accurate": false, + "actual_intent": "failed_transfer", + "intent_accurate": false, + "confidence": 0.9508456128002716, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_arrival": 0.00016477089984276335, + "cash_withdrawal_charge": 0.000046294839421691933, + "failed_transfer": 0.9508456128002716, + "lost_or_stolen_card": 0.00010359237933950632, + "pending_transfer": 0.0034298934579293032, + "top_up_by_card_charge": 0.00485745879842862, + "top_up_failed": 0.03793898498834442, + "top_up_limits": 0.00017777421235948575, + "transfer_fee_charged": 0.0021138095628150497, + "transfer_timing": 0.0003218080612475554 + }, + "tie_break_triggered": false, + "wall_time_ms": 392 + }, + { + "id": "b77-22", + "dataset": "PolyAI/banking77", + "text": "When I made a transfer, there was a fee. Why?", + "domain_accurate": true, + "expected_intent": "transfer_fee_charged", + "pass1_intent": "transfer_fee_charged", + "pass1_accurate": true, + "actual_intent": "transfer_fee_charged", + "intent_accurate": true, + "confidence": 0.9984423977623008, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_payment_fee_charged": 0.001444547016456457, + "cash_withdrawal_charge": 0.000053447375037693845, + "lost_or_stolen_card": 0.000016922829610273424, + "pending_top_up": 0.000002158446826806838, + "pending_transfer": 4.1253886267661523e-7, + "top_up_by_bank_transfer_charge": 0.00000846761034671584, + "top_up_by_card_charge": 0.0000031347853652912525, + "top_up_failed": 0.00001841955361181793, + "transfer_fee_charged": 0.9984423977623008, + "transfer_timing": 0.000010092081581430712 + }, + "tie_break_triggered": false, + "wall_time_ms": 402 + }, + { + "id": "b77-23", + "dataset": "PolyAI/banking77", + "text": "dont understand why transfer failed", + "domain_accurate": true, + "expected_intent": "failed_transfer", + "pass1_intent": "failed_transfer", + "pass1_accurate": true, + "actual_intent": "failed_transfer", + "intent_accurate": true, + "confidence": 0.9997095045744705, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "cash_withdrawal_charge": 0.00001677274780348312, + "compromised_card": 0.0000074922856747424275, + "failed_transfer": 0.9997095045744705, + "lost_or_stolen_card": 0.000004722388070734861, + "pending_transfer": 0.000055094728743651054, + "top_up_by_card_charge": 0.000042983504879109465, + "top_up_failed": 0.000039538185842849864, + "top_up_limits": 6.433190004318136e-7, + "transfer_fee_charged": 0.00001830745655775037, + "transfer_timing": 0.00010494080895670842 + }, + "tie_break_triggered": false, + "wall_time_ms": 402 + }, + { + "id": "b77-24", + "dataset": "PolyAI/banking77", + "text": "Can I do a bank transfer to put additional money in my account because I am out of money?", + "domain_accurate": true, + "expected_intent": "transfer_into_account", + "pass1_intent": "transfer_into_account", + "pass1_accurate": true, + "actual_intent": "transfer_into_account", + "intent_accurate": true, + "confidence": 0.9992657310497352, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "pending_top_up": 0.000026223197051600764, + "pending_transfer": 0.00004292750772485258, + "top_up_by_bank_transfer_charge": 0.000013290384795056705, + "top_up_by_card_charge": 0.00040713398131264476, + "top_up_failed": 0.00001719979035450284, + "top_up_limits": 0.000020384974578845576, + "top_up_reverted": 0.000024121291210102563, + "transfer_fee_charged": 0.00017275517534165574, + "transfer_into_account": 0.9992657310497352, + "transfer_timing": 0.000010232647895618843 + }, + "tie_break_triggered": false, + "wall_time_ms": 395 + }, + { + "id": "b77-25", + "dataset": "PolyAI/banking77", + "text": "I did not get the right amount of money when I used a different country's currency.", + "domain_accurate": true, + "expected_intent": "wrong_exchange_rate_for_cash_withdrawal", + "pass1_intent": "card_payment_wrong_exchange_rate", + "pass1_accurate": false, + "actual_intent": "exchange_rate", + "intent_accurate": false, + "confidence": 0.49334257661583125, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_delivery_estimate": 0.0002415958567258595, + "card_payment_not_recognised": 0.0015643330542223417, + "card_payment_wrong_exchange_rate": 0.49334257661583125, + "cash_withdrawal_charge": 0.00016361823242650808, + "exchange_charge": 0.002474182125599288, + "exchange_rate": 0.43766458867437547, + "lost_or_stolen_card": 0.009831396448202691, + "top_up_by_bank_transfer_charge": 0.00035320816171131023, + "transfer_timing": 0.00006164587585394572, + "wrong_exchange_rate_for_cash_withdrawal": 0.054302854955051244 + }, + "tie_break_triggered": true, + "finalist_options": [ + "card_payment_wrong_exchange_rate", + "lost_or_stolen_card", + "wrong_exchange_rate_for_cash_withdrawal", + "exchange_rate", + "card_payment_fee_charged", + "card_payment_not_recognised" + ], + "wall_time_ms": 511 + }, + { + "id": "b77-26", + "dataset": "PolyAI/banking77", + "text": "The app denied my topped up.", + "domain_accurate": true, + "expected_intent": "top_up_failed", + "pass1_intent": "top_up_failed", + "pass1_accurate": true, + "actual_intent": "top_up_failed", + "intent_accurate": true, + "confidence": 0.915716187781156, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "declined_card_payment": 0.029955477344775515, + "declined_cash_withdrawal": 0.00015583390093808406, + "failed_transfer": 0.03695509939599446, + "lost_or_stolen_card": 0.0005546383226365609, + "pending_top_up": 0.007568744595214377, + "pending_transfer": 0.0005984090093893569, + "top_up_by_bank_transfer_charge": 0.0013444981910906843, + "top_up_by_card_charge": 0.006281427331929314, + "top_up_failed": 0.915716187781156, + "top_up_limits": 0.0008696841268755266 + }, + "tie_break_triggered": true, + "finalist_options": [ + "top_up_failed", + "failed_transfer", + "declined_card_payment", + "top_up_by_card_charge", + "pending_top_up", + "top_up_by_bank_transfer_charge" + ], + "wall_time_ms": 513 + }, + { + "id": "b77-27", + "dataset": "PolyAI/banking77", + "text": "Think someone has took money out with my card. What shall I do?", + "domain_accurate": true, + "expected_intent": "cash_withdrawal_not_recognised", + "pass1_intent": "compromised_card", + "pass1_accurate": false, + "actual_intent": "compromised_card", + "intent_accurate": false, + "confidence": 0.7269180070742536, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_not_working": 0.024570991588985135, + "card_payment_not_recognised": 0.011174230796893557, + "cash_withdrawal_charge": 0.012216091507628724, + "cash_withdrawal_not_recognised": 0.006490875458127828, + "compromised_card": 0.7269180070742536, + "declined_cash_withdrawal": 0.0016476710283158719, + "lost_or_stolen_card": 0.21507048355801905, + "pending_top_up": 0.00009338025990980076, + "top_up_by_card_charge": 0.001729777095303078, + "wrong_exchange_rate_for_cash_withdrawal": 0.00008849163256349414 + }, + "tie_break_triggered": true, + "finalist_options": [ + "compromised_card", + "card_payment_not_recognised", + "cash_withdrawal_not_recognised", + "cash_withdrawal_charge", + "lost_or_stolen_card", + "card_not_working" + ], + "wall_time_ms": 507 + }, + { + "id": "b77-28", + "dataset": "PolyAI/banking77", + "text": "Is there an exchange fee?", + "domain_accurate": true, + "expected_intent": "exchange_charge", + "pass1_intent": "exchange_charge", + "pass1_accurate": true, + "actual_intent": "exchange_charge", + "intent_accurate": true, + "confidence": 0.9985774074488359, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_payment_fee_charged": 0.000053057799775821196, + "card_payment_not_recognised": 0.00003487036714344918, + "card_payment_wrong_exchange_rate": 0.00037250346549588136, + "cash_withdrawal_charge": 0.00003202400305825465, + "exchange_charge": 0.9985774074488359, + "exchange_rate": 0.0004006346194264117, + "lost_or_stolen_card": 0.0000031922204820124976, + "pending_top_up": 0.000037954444490335966, + "top_up_by_card_charge": 0.00005763468589682584, + "transfer_fee_charged": 0.0004307209453949232 + }, + "tie_break_triggered": false, + "wall_time_ms": 382 + }, + { + "id": "b77-29", + "dataset": "PolyAI/banking77", + "text": "Is there a fee for using a European bank card to top up?", + "domain_accurate": true, + "expected_intent": "top_up_by_card_charge", + "pass1_intent": "transfer_fee_charged", + "pass1_accurate": false, + "actual_intent": "card_payment_fee_charged", + "intent_accurate": false, + "confidence": 0.7390942076513918, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_delivery_estimate": 0.000014163901840768045, + "card_payment_not_recognised": 0.00010570346830984492, + "cash_withdrawal_charge": 0.00002606934640579481, + "exchange_charge": 0.20635106992978933, + "pending_top_up": 0.0028056673603691668, + "top_up_by_card_charge": 0.00012368363051490776, + "top_up_failed": 0.0008256398953871882, + "top_up_limits": 0.049770265117278875, + "transfer_fee_charged": 0.7390942076513918, + "transfer_into_account": 0.0008835296987123268 + }, + "tie_break_triggered": true, + "finalist_options": [ + "transfer_fee_charged", + "top_up_limits", + "exchange_charge", + "card_payment_fee_charged" + ], + "wall_time_ms": 506 + }, + { + "id": "b77-30", + "dataset": "PolyAI/banking77", + "text": "I see some fees for cash withdraw.", + "domain_accurate": true, + "expected_intent": "cash_withdrawal_charge", + "pass1_intent": "cash_withdrawal_charge", + "pass1_accurate": true, + "actual_intent": "cash_withdrawal_charge", + "intent_accurate": true, + "confidence": 0.9957335809848498, + "logprob": 0, + "entropy": 0, + "top_probabilities": { + "card_payment_fee_charged": 0.0038772729046228733, + "card_payment_not_recognised": 0.000018319612069993293, + "cash_withdrawal_charge": 0.9957335809848498, + "cash_withdrawal_not_recognised": 0.00019272543440579013, + "declined_cash_withdrawal": 0.000008242072456434526, + "exchange_charge": 0.000050529999809883075, + "lost_or_stolen_card": 0.000019979885231163993, + "top_up_by_bank_transfer_charge": 0.000005215480400914895, + "top_up_by_card_charge": 0.00007581401408322798, + "wrong_exchange_rate_for_cash_withdrawal": 0.000018319612069993293 + }, + "tie_break_triggered": false, + "wall_time_ms": 376 + } + ], + "samples_policy": 1, + "target_model": "diffgemma-26b-a4b-it-q4", + "target_url": "https://.-.prediction.vertexai.goog/v1/projects//locations/us-central1/endpoints//invoke/v1/chat/completions", + "tie_break_enabled": true, + "tie_breaks_triggered": 8, + "timestamp": "2026-09-28T23:25:40Z", + "total_cases": 30, + "workers": 4 +} \ No newline at end of file diff --git a/cmd/bench_intents.go b/cmd/bench_intents.go index c407591..3d21649 100644 --- a/cmd/bench_intents.go +++ b/cmd/bench_intents.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "github.com/ghchinoy/dgem/pkg/client" "github.com/ghchinoy/dgem/pkg/template" "github.com/spf13/cobra" ) @@ -96,6 +97,111 @@ type IntentResult struct { // buildFinalists selects the 2..5 most plausible competing options for Pass-2 tie-breaking // using Pass-1's top_logprobs subwords and semantic token overlap. +// maxChoiceOptions is the server's limit for one choice question (single-letter labels A-Z). +const maxChoiceOptions = 26 + +// intentBracketSize and intentBracketKeep: with more than 26 candidate intents, round 1 asks one question per +// group of at most intentBracketSize options and keeps each group's top intentBracketKeep, and a final round +// picks among the kept ones (the same 2-stage bracket as `dgem systemone serve`). +const ( + intentBracketSize = 20 + intentBracketKeep = 5 +) + +type intentDecideFunc func(vars map[string]interface{}) (*client.StructuredDecisionResponse, *client.RequestStats, error) + +// splitBalanced splits opts into ceil(len/size) groups of near-equal size, preserving order. +func splitBalanced(opts []string, size int) [][]string { + n := (len(opts) + size - 1) / size + var out [][]string + for i := 0; i < n; i++ { + lo, hi := i*len(opts)/n, (i+1)*len(opts)/n + out = append(out, opts[lo:hi]) + } + return out +} + +// topOptions returns up to k option names from an answer's probabilities, highest first (the chosen option first). +func topOptions(ans client.QuestionAnswer, group []string, k int) []string { + type kv struct { + name string + p float64 + } + var ranked []kv + for _, o := range group { + ranked = append(ranked, kv{o, ans.Probabilities[o]}) + } + chosen := strings.TrimSpace(ans.DisplayValue()) + for i := range ranked { + for j := i + 1; j < len(ranked); j++ { + if ranked[j].p > ranked[i].p || (ranked[j].name == chosen && ranked[i].name != chosen) { + ranked[i], ranked[j] = ranked[j], ranked[i] + } + } + } + var out []string + for _, r := range ranked { + if len(out) == k { + break + } + out = append(out, r.name) + } + return out +} + +// decideIntentBracketed runs one decision when the options fit in one question, otherwise a 2-stage bracket. +// The returned response is the final round's; stats.WallTime sums all rounds. +func decideIntentBracketed(ctx context.Context, tmplPath string, vars map[string]interface{}, options []string, decide intentDecideFunc) (*client.StructuredDecisionResponse, *client.RequestStats, error) { + withOptions := func(opts []string) map[string]interface{} { + v := make(map[string]interface{}, len(vars)) + for k, x := range vars { + v[k] = x + } + v["options"] = opts + return v + } + if len(options) <= maxChoiceOptions { + return decide(withOptions(options)) + } + var total time.Duration + var finalists []string + for _, group := range splitBalanced(options, intentBracketSize) { + resp, stats, err := decide(withOptions(group)) + if err != nil { + return nil, nil, fmt.Errorf("bracket round 1: %w", err) + } + if stats != nil { + total += stats.WallTime + } + finalists = append(finalists, topOptions(intentAnswer(resp), group, intentBracketKeep)...) + } + resp, stats, err := decide(withOptions(finalists)) + if err != nil { + return nil, nil, fmt.Errorf("bracket final: %w", err) + } + if stats != nil { + stats.WallTime += total + } + return resp, stats, nil +} + +// intentAnswer returns the intent answer from a response (key "intent", else "answer", else the only answer). +func intentAnswer(resp *client.StructuredDecisionResponse) client.QuestionAnswer { + if resp == nil { + return client.QuestionAnswer{} + } + if a, ok := resp.Answers["intent"]; ok { + return a + } + if a, ok := resp.Answers["answer"]; ok { + return a + } + for _, v := range resp.Answers { + return v + } + return client.QuestionAnswer{} +} + func buildFinalists(pass1Guess string, topProbs map[string]float64, allOptions []string) []string { seen := make(map[string]bool) var finalists []string @@ -418,16 +524,17 @@ func runBenchIntents(cmd *cobra.Command, args []string) error { "samples": samplesParam, } - rendered, err := engine.RenderFile(tmplPath, vars) - if err != nil { - continue - } - schemaContent, stateContent, err := template.ParseStructuredPayload(rendered, vars) - if err != nil { - continue - } - - resp, stats, err := c.Decide(ctx, schemaContent, stateContent) + resp, stats, err := decideIntentBracketed(ctx, tmplPath, vars, tc.Options, func(v map[string]interface{}) (*client.StructuredDecisionResponse, *client.RequestStats, error) { + rendered, err := engine.RenderFile(tmplPath, v) + if err != nil { + return nil, nil, err + } + schemaContent, stateContent, err := template.ParseStructuredPayload(rendered, v) + if err != nil { + return nil, nil, err + } + return c.Decide(ctx, schemaContent, stateContent) + }) if err != nil { fmt.Fprintf(os.Stderr, "Error on case %s: %v\n", tc.ID, err) continue diff --git a/cmd/bench_intents_bracket_test.go b/cmd/bench_intents_bracket_test.go new file mode 100644 index 0000000..5de49a0 --- /dev/null +++ b/cmd/bench_intents_bracket_test.go @@ -0,0 +1,75 @@ +package cmd + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/ghchinoy/dgem/pkg/client" +) + +func TestSplitBalanced(t *testing.T) { + opts := make([]string, 30) + for i := range opts { + opts[i] = fmt.Sprintf("o%02d", i) + } + g := splitBalanced(opts, 20) + if len(g) != 2 || len(g[0]) != 15 || len(g[1]) != 15 { + t.Fatalf("30/20 -> %d groups %v", len(g), []int{len(g[0]), len(g[1])}) + } + if g := splitBalanced(opts[:77%30], 20); len(g) != 1 { + t.Fatalf("17 options -> %d groups", len(g)) + } +} + +// fake decide: picks the option named "o21" when present, otherwise the first option; rejects > 26 options. +func fakeDecide(calls *[]int) intentDecideFunc { + return func(v map[string]interface{}) (*client.StructuredDecisionResponse, *client.RequestStats, error) { + opts := v["options"].([]string) + *calls = append(*calls, len(opts)) + if len(opts) > maxChoiceOptions { + return nil, nil, fmt.Errorf("at most 26 alternatives") + } + pick := opts[0] + for _, o := range opts { + if o == "o21" { + pick = o + } + } + probs := map[string]float64{} + for _, o := range opts { + probs[o] = 0.01 + } + probs[pick] = 0.9 + return &client.StructuredDecisionResponse{Answers: map[string]client.QuestionAnswer{ + "intent": {Type: "choice", Label: pick, Choice: pick, Probabilities: probs}}}, + &client.RequestStats{WallTime: 10 * time.Millisecond}, nil + } +} + +func TestDecideIntentBracketed(t *testing.T) { + opts := make([]string, 30) + for i := range opts { + opts[i] = fmt.Sprintf("o%02d", i) + } + var calls []int + resp, stats, err := decideIntentBracketed(context.Background(), "", map[string]interface{}{"text": "x"}, opts, fakeDecide(&calls)) + if err != nil { + t.Fatal(err) + } + if got := intentAnswer(resp).DisplayValue(); got != "o21" { + t.Fatalf("final answer %q, want o21", got) + } + if fmt.Sprint(calls) != "[15 15 10]" { + t.Fatalf("calls %v, want two groups of 15 then 10 finalists", calls) + } + if stats.WallTime != 30*time.Millisecond { + t.Fatalf("wall time %v, want summed 30ms", stats.WallTime) + } + + calls = nil + if _, _, err := decideIntentBracketed(context.Background(), "", nil, opts[:26], fakeDecide(&calls)); err != nil || fmt.Sprint(calls) != "[26]" { + t.Fatalf("26 options should be one call: %v %v", calls, err) + } +}