From 6f22bc60abaf0d531ba84fa454ea484684fef591 Mon Sep 17 00:00:00 2001 From: Marc Reichel Date: Mon, 12 Oct 2026 22:08:44 +0200 Subject: [PATCH] Expose laya-serve's full usage report on Result --- README.md | 14 +++++++++++- src/CollapsedOptions.php | 42 ++++++++++++++++++++++++++++++++++ src/Laravel/TryCommand.php | 10 +++++++- src/Result.php | 16 +++++++++++++ tests/LayaTest.php | 47 ++++++++++++++++++++++++++++++++++++++ tests/TryCommandTest.php | 24 ++++++++++++++++--- 6 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 src/CollapsedOptions.php diff --git a/README.md b/README.md index ea45878..03d5564 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ if ($result->choice('department')->answerConfidence < 0.7) { } ``` -Servers that only follow TypeSafe's [OpenAPI spec](https://api.typesafe.ai/openapi.json), such as [sys1](https://github.com/alvarobartt/sys1), don't send an `answer_confidence`. Then `answerConfidence` falls back to `confidence`, and a yes/no answer without a `confidence` gets laya's own `max(P(yes), P(no))`. Thresholds and `minConfidence` keep working, though `confidence` is stricter than the calibrated value for choice and score answers. The same applies to laya-serve 0.3.24 or later started with `LAYA_JEV_STRICT=1`, which strips `answer_confidence`, `routing` and the truncation report from its responses, so `routedModel` is `null` and `truncated` is always `false` there. +Servers that only follow TypeSafe's [OpenAPI spec](https://api.typesafe.ai/openapi.json), such as [sys1](https://github.com/alvarobartt/sys1), don't send an `answer_confidence`. Then `answerConfidence` falls back to `confidence`, and a yes/no answer without a `confidence` gets laya's own `max(P(yes), P(no))`. Thresholds and `minConfidence` keep working, though `confidence` is stricter than the calibrated value for choice and score answers. The same applies to laya-serve 0.3.24 or later started with `LAYA_JEV_STRICT=1`, which strips `answer_confidence`, `routing` and the truncation report from its responses, so `routedModel` is `null`, `truncated` is always `false` and the rest of the usage report is empty there. ### Abstention on the server @@ -219,6 +219,18 @@ if ($result->truncated) { } ``` +laya-serve also reports how much it cut. All of this is optional, so a server that doesn't send it leaves the integers `null` and the lists empty: + +```php +$result->stateTokens; // 900, the tokens the whole state needs +$result->stateTokensDropped; // 388, the most tokens any one question didn't get +$result->truncatedQuestions; // ['department'], the questions whose own window was cut + +foreach ($result->collapsedOptions as $id => $options) { + // laya merged $options->total options into $options->distinct, $options->tokensPerOption tokens each +} +``` + ### Batches (experimental) `predictMany()` and `decideMany()` ask the same questions about many states, which laya-serve 0.3.22 and later answers in shared forward passes. Results keep the keys you pass in: diff --git a/src/CollapsedOptions.php b/src/CollapsedOptions.php new file mode 100644 index 0000000..645d582 --- /dev/null +++ b/src/CollapsedOptions.php @@ -0,0 +1,42 @@ + $this->total, 'distinct' => $this->distinct, 'tokens_per_option' => $this->tokensPerOption]; + } +} diff --git a/src/Laravel/TryCommand.php b/src/Laravel/TryCommand.php index 9a1c84c..c88eae8 100644 --- a/src/Laravel/TryCommand.php +++ b/src/Laravel/TryCommand.php @@ -167,7 +167,11 @@ private function render(array $questions, Result $result, array $values): void $this->line(sprintf('Routed to %s, %d input tokens', $result->routedModel ?? 'an unknown checkpoint', $result->inputTokens)); if ($result->truncated) { - $this->warn('laya cut the text off at its token budget; raise --max-len to read all of it.'); + $this->warn(sprintf( + 'laya cut the text off at its token budget%s%s; raise --max-len to read all of it.', + $result->truncatedQuestions === [] ? '' : ' for '.OutputFormatter::escape(implode(', ', $result->truncatedQuestions)), + $result->stateTokensDropped === null ? '' : sprintf(' (%d tokens dropped)', $result->stateTokensDropped), + )); } } @@ -280,6 +284,10 @@ private static function json(string $class, array $questions, Result $result, ar 'routed_model' => $result->routedModel, 'input_tokens' => $result->inputTokens, 'truncated' => $result->truncated, + 'state_tokens' => $result->stateTokens, + 'state_tokens_dropped' => $result->stateTokensDropped, + 'truncated_questions' => $result->truncatedQuestions, + 'collapsed_options' => (object) $result->collapsedOptions, 'parameters' => $parameters, ], JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION); } diff --git a/src/Result.php b/src/Result.php index 0bb9bfe..7265f5f 100644 --- a/src/Result.php +++ b/src/Result.php @@ -27,6 +27,10 @@ * @param array $routing laya's full routing decision (model, reason, detected language, ...) * @param array $raw the whole response body * @param bool $truncated laya cut the state off at its token budget; raise maxLen to read it all (laya-serve >= 0.3.22) + * @param int|null $stateTokens tokens the whole state needs, null when the server doesn't report it + * @param int|null $stateTokensDropped tokens at least one question didn't get, null when the server doesn't report it + * @param list $truncatedQuestions ids of the questions whose own window was cut + * @param array $collapsedOptions questions whose options laya collapsed, by id */ public function __construct( public array $answers, @@ -35,6 +39,10 @@ public function __construct( public int $inputTokens, public array $raw, public bool $truncated = false, + public ?int $stateTokens = null, + public ?int $stateTokensDropped = null, + public array $truncatedQuestions = [], + public array $collapsedOptions = [], ) {} /** @param array $raw */ @@ -52,6 +60,10 @@ public static function fromArray(array $raw): self $routing = (array) ($raw['routing'] ?? []); $usage = is_array($raw['usage'] ?? null) ? $raw['usage'] : []; $tokens = $usage['input_tokens'] ?? 0; + $stateTokens = $usage['state_tokens'] ?? null; + $dropped = $usage['state_tokens_dropped'] ?? null; + $truncatedQuestions = is_array($usage['truncated_questions'] ?? null) ? $usage['truncated_questions'] : []; + $options = is_array($usage['options'] ?? null) ? $usage['options'] : []; return new self( $answers, @@ -60,6 +72,10 @@ public static function fromArray(array $raw): self is_int($tokens) ? $tokens : 0, $raw, ($usage['truncated'] ?? false) === true, + is_int($stateTokens) ? $stateTokens : null, + is_int($dropped) ? $dropped : null, + array_values(array_filter($truncatedQuestions, is_string(...))), + array_filter(array_map(CollapsedOptions::fromArray(...), $options)), ); } diff --git a/tests/LayaTest.php b/tests/LayaTest.php index dc64c2e..17f1614 100644 --- a/tests/LayaTest.php +++ b/tests/LayaTest.php @@ -9,6 +9,7 @@ use Illuminate\Cache\Repository; use MarcReichel\Laya\Answers\ChoiceAnswer; use MarcReichel\Laya\Answers\YesNoAnswer; +use MarcReichel\Laya\CollapsedOptions; use MarcReichel\Laya\Exceptions\AuthenticationException; use MarcReichel\Laya\Exceptions\InvalidQuestionException; use MarcReichel\Laya\Exceptions\ServerBusyException; @@ -267,6 +268,52 @@ public function sendRequest(RequestInterface $request): ResponseInterface ->and(Result::fromArray(['answers' => [], 'usage' => ['truncated' => 1]])->truncated)->toBeFalse(); }); +it('reads the full usage report', function () { + $result = Result::fromArray(['answers' => [], 'usage' => [ + 'input_tokens' => 512, + 'state_tokens' => 900, + 'state_tokens_dropped' => 388, + 'truncated' => true, + 'truncated_questions' => ['department', 'urgency'], + 'options' => [ + 'department' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3], + 0 => ['total' => 5, 'distinct' => 5, 'tokens_per_option' => 8], + ], + ]]); + + expect($result->stateTokens)->toBe(900) + ->and($result->stateTokensDropped)->toBe(388) + ->and($result->truncatedQuestions)->toBe(['department', 'urgency']) + ->and($result->collapsedOptions)->toEqual(['department' => new CollapsedOptions(40, 12, 3), '0' => new CollapsedOptions(5, 5, 8)]); +}); + +it('leaves an unreported or malformed usage report empty', function () { + $bare = Result::fromArray(['answers' => [], 'usage' => ['input_tokens' => 1, 'output_tokens' => 0]]); + $garbage = Result::fromArray(['answers' => [], 'usage' => [ + 'state_tokens' => '900', + 'state_tokens_dropped' => 1.5, + 'truncated_questions' => 'department', + 'options' => 'garbage', + ]]); + $mixed = Result::fromArray(['answers' => [], 'usage' => [ + 'truncated_questions' => [1 => 'department', 2 => 3, 3 => 'urgency'], + 'options' => [ + 'a' => 'garbage', + 'b' => ['total' => '40', 'distinct' => 12, 'tokens_per_option' => 3], + 'c' => ['total' => 40, 'distinct' => '12', 'tokens_per_option' => 3], + 'd' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => '3'], + 'e' => [], + ], + ]]); + + expect([$bare->stateTokens, $bare->stateTokensDropped, $bare->truncatedQuestions, $bare->collapsedOptions])->toBe([null, null, [], []]) + ->and([$garbage->stateTokens, $garbage->stateTokensDropped, $garbage->truncatedQuestions, $garbage->collapsedOptions])->toBe([null, null, [], []]) + ->and($mixed->truncatedQuestions)->toBe(['department', 'urgency']) + ->and($mixed->collapsedOptions)->toBe([]) + ->and(CollapsedOptions::fromArray(new ArrayObject(['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3])))->toBeNull() + ->and(new Result([], null, [], 0, []))->toMatchObject(['stateTokens' => null, 'stateTokensDropped' => null, 'truncatedQuestions' => [], 'collapsedOptions' => []]); +}); + it('reports a state laya cut off', function () { expect(Result::fromArray(['answers' => [], 'usage' => ['truncated' => true]])->truncated)->toBeTrue() ->and((new Result([], null, [], 0, []))->truncated)->toBeFalse(); diff --git a/tests/TryCommandTest.php b/tests/TryCommandTest.php index c2ff64c..5b846d5 100644 --- a/tests/TryCommandTest.php +++ b/tests/TryCommandTest.php @@ -137,6 +137,13 @@ function tryWithStdin(string $stdin, array $parameters = []): array ->and(Artisan::output())->toEndWith("Routed to english, 512 input tokens\nlaya cut the text off at its token budget; raise --max-len to read all of it.\n"); }); +it('names the questions laya cut off and the tokens it dropped', function () { + app()->instance(Laya::class, layaRespondingWith(200, ['usage' => ['input_tokens' => 512, 'state_tokens_dropped' => 388, 'truncated' => true, 'truncated_questions' => ['department', '']]] + LAYA_RESPONSE)); + + expect(Artisan::call('laya:try', ['class' => 'Decisions\Triage', 'state' => 'x']))->toBe(0) + ->and(Artisan::output())->toEndWith("laya cut the text off at its token budget for department, (388 tokens dropped); raise --max-len to read all of it.\n"); +}); + it('forwards the model and token budgets', function () { $sent = []; app()->instance(Laya::class, layaRespondingWith(200, LAYA_RESPONSE, $sent)); @@ -147,7 +154,14 @@ function tryWithStdin(string $stdin, array $parameters = []): array }); it('prints JSON', function () { - app()->instance(Laya::class, layaRespondingWith(200, ['usage' => ['input_tokens' => 512, 'truncated' => true]] + LAYA_RESPONSE)); + app()->instance(Laya::class, layaRespondingWith(200, ['usage' => [ + 'input_tokens' => 512, + 'state_tokens' => 900, + 'state_tokens_dropped' => 388, + 'truncated' => true, + 'truncated_questions' => ['department'], + 'options' => ['department' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3]], + ]] + LAYA_RESPONSE)); expect(Artisan::call('laya:try', ['class' => 'Decisions\Triage', 'state' => 'x', '--json' => true]))->toBe(0); @@ -157,6 +171,10 @@ function tryWithStdin(string $stdin, array $parameters = []): array 'routed_model' => 'english', 'input_tokens' => 512, 'truncated' => true, + 'state_tokens' => 900, + 'state_tokens_dropped' => 388, + 'truncated_questions' => ['department'], + 'collapsed_options' => ['department' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3]], 'parameters' => [ 'department' => [ 'question' => 'Which department should handle this?', @@ -207,12 +225,12 @@ function tryWithStdin(string $stdin, array $parameters = []): array ->toContain("budget: Is there budget?\n → 0.8 (answer confidence 0.80)\n"); }); -it('keeps numeric labels an object, and whole numbers floats, in JSON', function () { +it('keeps numeric labels and no collapsed options an object, and whole numbers floats, in JSON', function () { Laya::fake(['code' => Code::Zero]); Artisan::call('laya:try', ['class' => Coded::class, 'state' => 'x', '--json' => true]); - expect(Artisan::output())->toContain(<<<'JSON' + expect(Artisan::output())->toContain('"collapsed_options": {},')->toContain(<<<'JSON' "value": 0, "answer_confidence": 1.0, "probabilities": {