Skip to content

Surface chat system_fingerprint and logprobs as response metadata #1104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
PratikDhanave (PratikDhanave) wants to merge 1 commit into
base: main
Choose a base branch
from
+90 −14
Open

Surface chat system_fingerprint and logprobs as response metadata #1104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 36 additions & 14 deletions provider/openaiprovider/chat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ func (a *chatClient) run(ctx context.Context, messages []*message.Message, optio
// the streaming path below.
var contents []message.Content
var finishReason string
var additionalProperties map[string]any
if len(resp.Choices) > 0 {
choice := resp.Choices[0]
contents = make([]message.Content, 0, 1+len(choice.Message.ToolCalls))
Expand All @@ -150,19 +151,29 @@ func (a *chatClient) run(ctx context.Context, messages []*message.Message, optio
contents = append(contents, &message.ErrorContent{Message: choice.Message.Refusal, ErrorCode: "Refusal"})
}
finishReason = choice.FinishReason
if len(choice.Logprobs.Content) > 0 || len(choice.Logprobs.Refusal) > 0 {
additionalProperties = map[string]any{"Logprobs": choice.Logprobs}
}
}
if resp.SystemFingerprint != "" {
if additionalProperties == nil {
additionalProperties = make(map[string]any)
}
additionalProperties["SystemFingerprint"] = resp.SystemFingerprint
}
if resp.JSON.Usage.Valid() {
contents = addUsage(contents, resp.Usage)
}
return func(yield func(*agent.ResponseUpdate, error) bool) {
update := &agent.ResponseUpdate{
Contents: contents,
Role: message.RoleAssistant,
ResponseID: resp.ID,
MessageID: resp.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(resp.Created, 0),
RawRepresentation: resp,
Contents: contents,
Role: message.RoleAssistant,
ResponseID: resp.ID,
MessageID: resp.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(resp.Created, 0),
AdditionalProperties: additionalProperties,
RawRepresentation: resp,
}
if !yield(update, nil) {
return
Expand Down Expand Up @@ -208,17 +219,28 @@ func (a *chatClient) run(ctx context.Context, messages []*message.Message, optio
contents = addUsage(contents, chunk.Usage)
}
var finishReason string
var additionalProperties map[string]any
if len(chunk.Choices) > 0 {
finishReason = chunk.Choices[0].FinishReason
if logprobs := chunk.Choices[0].Logprobs; len(logprobs.Content) > 0 || len(logprobs.Refusal) > 0 {
additionalProperties = map[string]any{"Logprobs": logprobs}
Comment on lines +225 to +226
}
}
if chunk.SystemFingerprint != "" {
if additionalProperties == nil {
additionalProperties = make(map[string]any)
}
additionalProperties["SystemFingerprint"] = chunk.SystemFingerprint
}
resp := &agent.ResponseUpdate{
Contents: contents,
Role: role,
ResponseID: chunk.ID,
MessageID: chunk.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(chunk.Created, 0),
RawRepresentation: chunk,
Contents: contents,
Role: role,
ResponseID: chunk.ID,
MessageID: chunk.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(chunk.Created, 0),
AdditionalProperties: additionalProperties,
RawRepresentation: chunk,
}
if !yield(resp, nil) {
return
Expand Down
54 changes: 54 additions & 0 deletions provider/openaiprovider/chat_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,60 @@ func TestChatBasicRequestResponse_NonStreaming(t *testing.T) {
}
}

// system_fingerprint (response) and per-choice logprobs must be surfaced on the
// update's AdditionalProperties, matching the Python client which carries both
// as chat response metadata.
func TestChatResponseMetadataSurfaced_NonStreaming(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"id":"chatcmpl-md","object":"chat.completion","created":1727888631,"model":"gpt-4o-mini","system_fingerprint":"fp_test","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"logprobs":{"content":[{"token":"ok","logprob":-0.1,"bytes":[111,107],"top_logprobs":[]}]},"finish_reason":"stop"}]}`)
}))
defer server.Close()

resp, err := newTestClient(server).RunText(t.Context(), "hi").Collect()
if err != nil {
t.Fatalf("error = %v", err)
}
props := lastMessageAdditionalProperties(t, resp)
if props["SystemFingerprint"] != "fp_test" {
t.Errorf("SystemFingerprint = %v, want %q", props["SystemFingerprint"], "fp_test")
}
if _, ok := props["Logprobs"]; !ok {
t.Errorf("Logprobs missing from AdditionalProperties: %#v", props)
}
}

func TestChatResponseMetadataSurfaced_Streaming(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, "data: {\"id\":\"chatcmpl-md\",\"object\":\"chat.completion.chunk\",\"created\":1727888631,\"model\":\"gpt-4o-mini\",\"system_fingerprint\":\"fp_test\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\n")
_, _ = io.WriteString(w, "data: [DONE]\n\n")
}))
defer server.Close()

resp, err := newTestClient(server).RunText(t.Context(), "hi", agent.Stream(true)).Collect()
if err != nil {
t.Fatalf("error = %v", err)
}
props := lastMessageAdditionalProperties(t, resp)
if props["SystemFingerprint"] != "fp_test" {
t.Errorf("SystemFingerprint = %v, want %q", props["SystemFingerprint"], "fp_test")
}
}

// lastMessageAdditionalProperties returns the AdditionalProperties of the last
// message carrying any, so metadata assertions do not depend on message count.
func lastMessageAdditionalProperties(t *testing.T, resp *agent.Response) map[string]any {
t.Helper()
for i := len(resp.Messages) - 1; i >= 0; i-- {
if len(resp.Messages[i].AdditionalProperties) > 0 {
return resp.Messages[i].AdditionalProperties
}
}
t.Fatalf("no message carried AdditionalProperties: %#v", resp.Messages)
return nil
}

func TestChatURLCitationAnnotations_NonStreaming(t *testing.T) {
const input = `
{
Expand Down
Loading