Expose laya-serve's full usage report on Result

README.md

@@ -153,7 +153,7 @@ if ($result->choice('department')->answerConfidence < 0.7) {
 }
 ```
 
-Servers that only follow TypeSafe's [OpenAPI spec](https://api.typesafe.ai/openapi.json), such as [sys1](https://github.com/alvarobartt/sys1), don't send an `answer_confidence`. Then `answerConfidence` falls back to `confidence`, and a yes/no answer without a `confidence` gets laya's own `max(P(yes), P(no))`. Thresholds and `minConfidence` keep working, though `confidence` is stricter than the calibrated value for choice and score answers. The same applies to laya-serve 0.3.24 or later started with `LAYA_JEV_STRICT=1`, which strips `answer_confidence`, `routing` and the truncation report from its responses, so `routedModel` is `null` and `truncated` is always `false` there.
+Servers that only follow TypeSafe's [OpenAPI spec](https://api.typesafe.ai/openapi.json), such as [sys1](https://github.com/alvarobartt/sys1), don't send an `answer_confidence`. Then `answerConfidence` falls back to `confidence`, and a yes/no answer without a `confidence` gets laya's own `max(P(yes), P(no))`. Thresholds and `minConfidence` keep working, though `confidence` is stricter than the calibrated value for choice and score answers. The same applies to laya-serve 0.3.24 or later started with `LAYA_JEV_STRICT=1`, which strips `answer_confidence`, `routing` and the truncation report from its responses, so `routedModel` is `null`, `truncated` is always `false` and the rest of the usage report is empty there.
 
 ### Abstention on the server
 
@@ -219,6 +219,18 @@ if ($result->truncated) {
 }
 ```
 
+laya-serve also reports how much it cut. All of this is optional, so a server that doesn't send it leaves the integers `null` and the lists empty:
+
+```php
+$result->stateTokens;         // 900, the tokens the whole state needs
+$result->stateTokensDropped;  // 388, the most tokens any one question didn't get
+$result->truncatedQuestions;  // ['department'], the questions whose own window was cut
+
+foreach ($result->collapsedOptions as $id => $options) {
+    // laya merged $options->total options into $options->distinct, $options->tokensPerOption tokens each
+}
+```
+
 ### Batches (experimental)
 
 `predictMany()` and `decideMany()` ask the same questions about many states, which laya-serve 0.3.22 and later answers in shared forward passes. Results keep the keys you pass in:

src/CollapsedOptions.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+namespace MarcReichel\Laya;
+
+/**
+ * How laya collapsed one question's options to fit its token budget.
+ */
+final readonly class CollapsedOptions implements \JsonSerializable
+{
+    /**
+     * @param  int  $total  options the question has
+     * @param  int  $distinct  options laya told apart after collapsing
+     * @param  int  $tokensPerOption  tokens each option got
+     */
+    public function __construct(
+        public int $total,
+        public int $distinct,
+        public int $tokensPerOption,
+    ) {}
+
+    /** null unless $raw is a complete report */
+    public static function fromArray(mixed $raw): ?self
+    {
+        if (! is_array($raw)) {
+            return null;
+        }
+
+        $total = $raw['total'] ?? null;
+        $distinct = $raw['distinct'] ?? null;
+        $tokens = $raw['tokens_per_option'] ?? null;
+
+        return is_int($total) && is_int($distinct) && is_int($tokens) ? new self($total, $distinct, $tokens) : null;
+    }
+
+    /** @return array{total: int, distinct: int, tokens_per_option: int} */
+    public function jsonSerialize(): array
+    {
+        return ['total' => $this->total, 'distinct' => $this->distinct, 'tokens_per_option' => $this->tokensPerOption];
+    }
+}

src/Laravel/TryCommand.php

@@ -167,7 +167,11 @@ private function render(array $questions, Result $result, array $values): void
         $this->line(sprintf('Routed to %s, %d input tokens', $result->routedModel ?? 'an unknown checkpoint', $result->inputTokens));
 
         if ($result->truncated) {
-            $this->warn('laya cut the text off at its token budget; raise --max-len to read all of it.');
+            $this->warn(sprintf(
+                'laya cut the text off at its token budget%s%s; raise --max-len to read all of it.',
+                $result->truncatedQuestions === [] ? '' : ' for '.OutputFormatter::escape(implode(', ', $result->truncatedQuestions)),
+                $result->stateTokensDropped === null ? '' : sprintf(' (%d tokens dropped)', $result->stateTokensDropped),
+            ));
         }
     }
 
@@ -280,6 +284,10 @@ private static function json(string $class, array $questions, Result $result, ar
             'routed_model' => $result->routedModel,
             'input_tokens' => $result->inputTokens,
             'truncated' => $result->truncated,
+            'state_tokens' => $result->stateTokens,
+            'state_tokens_dropped' => $result->stateTokensDropped,
+            'truncated_questions' => $result->truncatedQuestions,
+            'collapsed_options' => (object) $result->collapsedOptions,
             'parameters' => $parameters,
         ], JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION);
     }

src/Result.php

@@ -27,6 +27,10 @@
      * @param  array<mixed>  $routing  laya's full routing decision (model, reason, detected language, ...)
      * @param  array<mixed>  $raw  the whole response body
      * @param  bool  $truncated  laya cut the state off at its token budget; raise maxLen to read it all (laya-serve >= 0.3.22)
+     * @param  int|null  $stateTokens  tokens the whole state needs, null when the server doesn't report it
+     * @param  int|null  $stateTokensDropped  tokens at least one question didn't get, null when the server doesn't report it
+     * @param  list<string>  $truncatedQuestions  ids of the questions whose own window was cut
+     * @param  array<array-key, CollapsedOptions>  $collapsedOptions  questions whose options laya collapsed, by id
      */
     public function __construct(
         public array $answers,
@@ -35,6 +39,10 @@ public function __construct(
         public int $inputTokens,
         public array $raw,
         public bool $truncated = false,
+        public ?int $stateTokens = null,
+        public ?int $stateTokensDropped = null,
+        public array $truncatedQuestions = [],
+        public array $collapsedOptions = [],
     ) {}
 
     /** @param array<mixed> $raw */
@@ -52,6 +60,10 @@ public static function fromArray(array $raw): self
         $routing = (array) ($raw['routing'] ?? []);
         $usage = is_array($raw['usage'] ?? null) ? $raw['usage'] : [];
         $tokens = $usage['input_tokens'] ?? 0;
+        $stateTokens = $usage['state_tokens'] ?? null;
+        $dropped = $usage['state_tokens_dropped'] ?? null;
+        $truncatedQuestions = is_array($usage['truncated_questions'] ?? null) ? $usage['truncated_questions'] : [];
+        $options = is_array($usage['options'] ?? null) ? $usage['options'] : [];
 
         return new self(
             $answers,
@@ -60,6 +72,10 @@ public static function fromArray(array $raw): self
             is_int($tokens) ? $tokens : 0,
             $raw,
             ($usage['truncated'] ?? false) === true,
+            is_int($stateTokens) ? $stateTokens : null,
+            is_int($dropped) ? $dropped : null,
+            array_values(array_filter($truncatedQuestions, is_string(...))),
+            array_filter(array_map(CollapsedOptions::fromArray(...), $options)),
         );
     }
 

tests/LayaTest.php

@@ -9,6 +9,7 @@
 use Illuminate\Cache\Repository;
 use MarcReichel\Laya\Answers\ChoiceAnswer;
 use MarcReichel\Laya\Answers\YesNoAnswer;
+use MarcReichel\Laya\CollapsedOptions;
 use MarcReichel\Laya\Exceptions\AuthenticationException;
 use MarcReichel\Laya\Exceptions\InvalidQuestionException;
 use MarcReichel\Laya\Exceptions\ServerBusyException;
@@ -267,6 +268,52 @@ public function sendRequest(RequestInterface $request): ResponseInterface
         ->and(Result::fromArray(['answers' => [], 'usage' => ['truncated' => 1]])->truncated)->toBeFalse();
 });
 
+it('reads the full usage report', function () {
+    $result = Result::fromArray(['answers' => [], 'usage' => [
+        'input_tokens' => 512,
+        'state_tokens' => 900,
+        'state_tokens_dropped' => 388,
+        'truncated' => true,
+        'truncated_questions' => ['department', 'urgency'],
+        'options' => [
+            'department' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3],
+            0 => ['total' => 5, 'distinct' => 5, 'tokens_per_option' => 8],
+        ],
+    ]]);
+
+    expect($result->stateTokens)->toBe(900)
+        ->and($result->stateTokensDropped)->toBe(388)
+        ->and($result->truncatedQuestions)->toBe(['department', 'urgency'])
+        ->and($result->collapsedOptions)->toEqual(['department' => new CollapsedOptions(40, 12, 3), '0' => new CollapsedOptions(5, 5, 8)]);
+});
+
+it('leaves an unreported or malformed usage report empty', function () {
+    $bare = Result::fromArray(['answers' => [], 'usage' => ['input_tokens' => 1, 'output_tokens' => 0]]);
+    $garbage = Result::fromArray(['answers' => [], 'usage' => [
+        'state_tokens' => '900',
+        'state_tokens_dropped' => 1.5,
+        'truncated_questions' => 'department',
+        'options' => 'garbage',
+    ]]);
+    $mixed = Result::fromArray(['answers' => [], 'usage' => [
+        'truncated_questions' => [1 => 'department', 2 => 3, 3 => 'urgency'],
+        'options' => [
+            'a' => 'garbage',
+            'b' => ['total' => '40', 'distinct' => 12, 'tokens_per_option' => 3],
+            'c' => ['total' => 40, 'distinct' => '12', 'tokens_per_option' => 3],
+            'd' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => '3'],
+            'e' => [],
+        ],
+    ]]);
+
+    expect([$bare->stateTokens, $bare->stateTokensDropped, $bare->truncatedQuestions, $bare->collapsedOptions])->toBe([null, null, [], []])
+        ->and([$garbage->stateTokens, $garbage->stateTokensDropped, $garbage->truncatedQuestions, $garbage->collapsedOptions])->toBe([null, null, [], []])
+        ->and($mixed->truncatedQuestions)->toBe(['department', 'urgency'])
+        ->and($mixed->collapsedOptions)->toBe([])
+        ->and(CollapsedOptions::fromArray(new ArrayObject(['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3])))->toBeNull()
+        ->and(new Result([], null, [], 0, []))->toMatchObject(['stateTokens' => null, 'stateTokensDropped' => null, 'truncatedQuestions' => [], 'collapsedOptions' => []]);
+});
+
 it('reports a state laya cut off', function () {
     expect(Result::fromArray(['answers' => [], 'usage' => ['truncated' => true]])->truncated)->toBeTrue()
         ->and((new Result([], null, [], 0, []))->truncated)->toBeFalse();

tests/TryCommandTest.php

@@ -137,6 +137,13 @@ function tryWithStdin(string $stdin, array $parameters = []): array
         ->and(Artisan::output())->toEndWith("Routed to english, 512 input tokens\nlaya cut the text off at its token budget; raise --max-len to read all of it.\n");
 });
 
+it('names the questions laya cut off and the tokens it dropped', function () {
+    app()->instance(Laya::class, layaRespondingWith(200, ['usage' => ['input_tokens' => 512, 'state_tokens_dropped' => 388, 'truncated' => true, 'truncated_questions' => ['department', '<urgency>']]] + LAYA_RESPONSE));
+
+    expect(Artisan::call('laya:try', ['class' => 'Decisions\Triage', 'state' => 'x']))->toBe(0)
+        ->and(Artisan::output())->toEndWith("laya cut the text off at its token budget for department, <urgency> (388 tokens dropped); raise --max-len to read all of it.\n");
+});
+
 it('forwards the model and token budgets', function () {
     $sent = [];
     app()->instance(Laya::class, layaRespondingWith(200, LAYA_RESPONSE, $sent));
@@ -147,7 +154,14 @@ function tryWithStdin(string $stdin, array $parameters = []): array
 });
 
 it('prints JSON', function () {
-    app()->instance(Laya::class, layaRespondingWith(200, ['usage' => ['input_tokens' => 512, 'truncated' => true]] + LAYA_RESPONSE));
+    app()->instance(Laya::class, layaRespondingWith(200, ['usage' => [
+        'input_tokens' => 512,
+        'state_tokens' => 900,
+        'state_tokens_dropped' => 388,
+        'truncated' => true,
+        'truncated_questions' => ['department'],
+        'options' => ['department' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3]],
+    ]] + LAYA_RESPONSE));
 
     expect(Artisan::call('laya:try', ['class' => 'Decisions\Triage', 'state' => 'x', '--json' => true]))->toBe(0);
 
@@ -157,6 +171,10 @@ function tryWithStdin(string $stdin, array $parameters = []): array
         'routed_model' => 'english',
         'input_tokens' => 512,
         'truncated' => true,
+        'state_tokens' => 900,
+        'state_tokens_dropped' => 388,
+        'truncated_questions' => ['department'],
+        'collapsed_options' => ['department' => ['total' => 40, 'distinct' => 12, 'tokens_per_option' => 3]],
         'parameters' => [
             'department' => [
                 'question' => 'Which department should handle this?',
@@ -207,12 +225,12 @@ function tryWithStdin(string $stdin, array $parameters = []): array
         ->toContain("budget: Is there budget?\n  → 0.8 (answer confidence 0.80)\n");
 });
 
-it('keeps numeric labels an object, and whole numbers floats, in JSON', function () {
+it('keeps numeric labels and no collapsed options an object, and whole numbers floats, in JSON', function () {
     Laya::fake(['code' => Code::Zero]);
 
     Artisan::call('laya:try', ['class' => Coded::class, 'state' => 'x', '--json' => true]);
 
-    expect(Artisan::output())->toContain(<<<'JSON'
+    expect(Artisan::output())->toContain('"collapsed_options": {},')->toContain(<<<'JSON'
                     "value": 0,
                     "answer_confidence": 1.0,
                     "probabilities": {