diff --git a/docs/api_support.md b/docs/api_support.md index 1a0fa25e..f7a40e37 100644 --- a/docs/api_support.md +++ b/docs/api_support.md @@ -22,6 +22,7 @@ These are the OpenAI-style endpoints that clients use for inference: | Endpoint | Method | Description | |---|---|---| | `/v1/chat/completions` | POST | Chat completions (streaming and non-streaming) | +| `/v1/completions` | POST | Text completions (NearDirect) | | `/v1/responses` | POST | Responses API (streaming and non-streaming) | | `/v1/embeddings` | POST | Text embeddings | | `/v1/audio/transcriptions` | POST | Audio transcription (multipart) | @@ -98,6 +99,7 @@ This matrix applies to OpenAI-compatible inference endpoints. | Image generation | Yes | Yes | — | — | — | — | — | — | | Reranking | Yes | Yes | — | — | — | — | — | — | | Score | Yes | Yes | — | — | — | — | — | — | +| Completions | Yes | — | — | — | — | — | — | — | **—** = Not implemented. Provider returns HTTP 400. @@ -113,6 +115,7 @@ This matrix applies to OpenAI-compatible inference endpoints. | Image generation | Encrypted | Encrypted | — | — | — | — | — | — | | Reranking | Encrypted | Encrypted | — | — | — | — | — | — | | Score | Request encrypted; response plaintext (pinned) | Request encrypted; response plaintext (pinned) | — | — | — | — | — | — | +| Completions | Encrypted | — | — | — | — | — | — | — | **Encrypted** = E2EE is applied to request and response fields. **Fail closed** = Proxy rejects the request with an error because E2EE is not supported for this endpoint in the end-to-end path, either because the proxy does not implement E2EE field dispatch for that request type or because the upstream TEE cannot decrypt it. @@ -158,6 +161,7 @@ NearDirect and NearCloud both use the NearAI field-encryption protocol: Ed25519/ | Image generation | `prompt` | | Reranking | `query`; `documents[]` when each document is a JSON string; `documents[].text` when documents are JSON objects. Other document object metadata remains plaintext | | Score | `text_1`, `text_2` | +| Completions (NearDirect) | `prompt` as a string or an array of strings. Every other field that teep accepts passes in plaintext: the sampling and limit fields (`max_tokens`, `temperature`, `top_p`, `n`, `stream`, `seed` and similar), `stop` (the stop sequences, which can hold text) and `user` (an end-user identifier). Teep accepts `logprobs`, `prompt_logprobs` and `suffix` only as `null`, and `echo` and `return_token_ids` only as `null` or `false`. Any other field, a duplicate member, a prompt of token ids and an empty prompt array are rejected in E2EE mode, because their content would travel or return unencrypted | **E2EE response fields encrypted (NearDirect and NearCloud):** @@ -194,6 +198,7 @@ NearDirect and NearCloud both use the NearAI field-encryption protocol: Ed25519/ | `data[].embedding` | Yes | Embeddings | | `results[].document.text` | Yes | Reranking; other object metadata remains plaintext | | `data[].score` | No | Score response is currently plaintext from upstream (known upstream NearAI limitation) | +| `choices[].text` | Yes | Completions (NearDirect), JSON and streaming. Each choice must carry `text`; besides it only `index`, `finish_reason`, `stop_reason` and `matched_stop`, and `logprobs`, `prompt_logprobs`, `token_ids` and `prompt_token_ids` only as `null`. A NEAR completions stream must end with `[DONE]` | #### Tool calls under E2EE @@ -256,6 +261,7 @@ another endpoint; see [models the probe does not cover](providers/near/near_atte | Image generation | `/v1/images/generations` | Yes | Sensitive fields encrypted; `prompt`, `b64_json`, and `revised_prompt` encrypted | | Reranking | `/v1/rerank` | Yes | Sensitive fields encrypted; `query` and `documents[]` encrypted | | Score | `/v1/score` | Request only | Request fields (`text_1`, `text_2`) encrypted; response `data[].score` currently plaintext (known upstream NearAI limitation) | +| Completions | `/v1/completions` | Yes | `prompt` encrypted; `choices[].text` decrypted. NEAR's gateway refuses E2EE on this endpoint ("End-to-end encryption is not supported on /v1/completions", 2026-11-14), so NearCloud has no completions route | --- diff --git a/internal/e2ee/completions_test.go b/internal/e2ee/completions_test.go new file mode 100644 index 00000000..782d04a3 --- /dev/null +++ b/internal/e2ee/completions_test.go @@ -0,0 +1,179 @@ +package e2ee + +import ( + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" +) + +// The prompt of a completions request is encrypted to the model key, as a +// string or each string of an array. +func TestEncryptCompletionsNearCloud(t *testing.T) { + pubHex, seed := ed25519KeyPairHex(t) + x25519Priv, err := ed25519SeedToX25519(seed) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, body string + want []string + }{ + {"string", `{"model":"m","prompt":"hello","max_tokens":5}`, []string{"hello"}}, + {"array", `{"model":"m","prompt":["hello","world"]}`, []string{"hello", "world"}}, + } { + t.Run(tc.name, func(t *testing.T) { + encBody, session, err := EncryptCompletionsNearCloud([]byte(tc.body), testNearModelKey(t, pubHex)) + if err != nil { + t.Fatal(err) + } + defer session.Zero() + var out map[string]json.RawMessage + if err := json.Unmarshal(encBody, &out); err != nil { + t.Fatal(err) + } + var got []string + if len(tc.want) == 1 { + var s string + if err := json.Unmarshal(out["prompt"], &s); err != nil { + t.Fatal(err) + } + got = []string{s} + } else if err := json.Unmarshal(out["prompt"], &got); err != nil { + t.Fatal(err) + } + for i, want := range tc.want { + pt, err := DecryptXChaCha20(got[i], x25519Priv) + if err != nil || string(pt) != want { + t.Errorf("prompt[%d] = %q, %v; want %q", i, pt, err, want) + } + } + if strings.Contains(string(encBody), `"hello"`) { + t.Error("the plaintext prompt is in the body") + } + }) + } +} + +// A request shape that NEAR's E2EE cannot carry fails with +// ErrCompletionsShape before it is sent: a field outside the allowlist, a +// content field with a value that is not "off", or a duplicate member at any +// depth, which a map would collapse to its last value. An integer 0 still turns +// logprobs on, so only null is off for logprobs. +func TestEncryptCompletionsNearCloudRefuses(t *testing.T) { + pubHex, _ := ed25519KeyPairHex(t) + for _, body := range []string{ + `{"model":"m","prompt":[1,2,3]}`, + `{"model":"m","prompt":["hello",7]}`, + `{"model":"m","prompt":12}`, + `{"model":"m"}`, + `{"model":"m","prompt":null}`, + `{"model":"m","prompt":"hello","suffix":"end"}`, + `{"model":"m","prompt":"hello","logprobs":5}`, + `{"model":"m","prompt":"hello","echo":true}`, + `{"model":"m","prompt":"hello","prompt_logprobs":1}`, + `{"model":"m","prompt":"hello","prompt_embeds":"AAAA"}`, + `{"model":"m","prompt":"hello","logprobs":0}`, + `{"model":"m","prompt":"hello","logprobs":false}`, + `{"model":"m","prompt":"hello","prompt_logprobs":0}`, + `{"model":"m","prompt":"hello","return_token_ids":true}`, + `{"model":"m","prompt":"hello","echo":0}`, + `{"model":"m","prompt":"hello","some_new_field":1}`, + `{"model":"m","prompt":[]}`, + `{"model":"m","prompt":"hello","logprobs":5,"logprobs":null}`, + `{"model":"m","prompt":"hello","prompt":"other"}`, + `{"model":"m","prompt":"hello","stream_options":{"include_usage":true,"include_usage":false}}`, + `{"model":"m","prompt":"hello","echo":true,"\u0065cho":false}`, + } { + _, session, err := EncryptCompletionsNearCloud([]byte(body), testNearModelKey(t, pubHex)) + if !errors.Is(err, ErrCompletionsShape) || session != nil { + t.Errorf("%s: err %v, session %v; want ErrCompletionsShape and no session", body, err, session) + } + } + for _, body := range []string{ + `{"model":"m","prompt":"hello","logprobs":null,"echo":false,"suffix":null,"return_token_ids":false,"prompt_logprobs":null}`, + `{"model":"m","prompt":"hello","max_tokens":5,"temperature":0.2,"stop":["x"],"n":1,"stream":true,"stream_options":{"include_usage":true},"seed":3}`, + } { + _, session, err := EncryptCompletionsNearCloud([]byte(body), testNearModelKey(t, pubHex)) + if err != nil { + t.Errorf("%s: %v", body, err) + continue + } + session.Zero() + } +} + +// A NEAR completions stream must end with [DONE], as a chat stream must. +func TestCompletionsStreamEndMarker(t *testing.T) { + session, err := NewNearCloudSession() + if err != nil { + t.Fatal(err) + } + defer session.Zero() + if err := CheckSSEEndMarker(session, EndpointCompletions, false); err == nil { + t.Error("a completions stream without [DONE] was accepted") + } + if err := CheckSSEEndMarker(session, EndpointCompletions, true); err != nil { + t.Errorf("a completions stream with [DONE]: %v", err) + } +} + +// choices[].text of a completions response is decrypted, and a plaintext +// text fails closed, for a JSON response and for an SSE chunk. +func TestDecryptCompletionsResponse(t *testing.T) { + session, err := NewNearCloudSession() + if err != nil { + t.Fatal(err) + } + defer session.Zero() + pub, err := hex.DecodeString(session.ClientEd25519PubHex()) + if err != nil { + t.Fatal(err) + } + recipient, err := Ed25519PubToX25519(pub) + if err != nil { + t.Fatal(err) + } + ct, err := EncryptXChaCha20([]byte(" hello"), recipient) + if err != nil { + t.Fatal(err) + } + encrypted := `{"id":"x","object":"text_completion","choices":[{"index":0,"text":"` + ct + `","finish_reason":"stop"}]}` + plain := `{"id":"x","object":"text_completion","choices":[{"index":0,"text":" hello","finish_reason":"stop"}]}` + + out, err := DecryptNonStreamResponseForEndpoint([]byte(encrypted), session, EndpointCompletions) + if err != nil || !strings.Contains(string(out), `"text":" hello"`) { + t.Fatalf("non-stream: %s, %v", out, err) + } + chunk, err := DecryptSSEChunk(encrypted, session, EndpointCompletions) + if err != nil || !strings.Contains(chunk, `"text":" hello"`) { + t.Fatalf("SSE chunk: %s, %v", chunk, err) + } + if _, err := DecryptNonStreamResponseForEndpoint([]byte(plain), session, EndpointCompletions); !errors.Is(err, ErrDecryptionFailed) { + t.Errorf("plaintext non-stream text: %v, want ErrDecryptionFailed", err) + } + if _, err := DecryptSSEChunk(plain, session, EndpointCompletions); !errors.Is(err, ErrDecryptionFailed) { + t.Errorf("plaintext SSE text: %v, want ErrDecryptionFailed", err) + } + empty := `{"choices":[{"index":0,"text":"","finish_reason":"length","logprobs":null,"matched_stop":null}]}` + if _, err := DecryptSSEChunk(empty, session, EndpointCompletions); err != nil { + t.Errorf("empty text chunk: %v", err) + } + // A choice without text, a content key with a value, or an unknown key + // fails closed: its content would pass outside the decrypted text. + for _, bad := range []string{ + `{"choices":[{"index":0,"finish_reason":"stop"}]}`, + `{"choices":[{"index":0,"text":"` + ct + `","logprobs":{"tokens":[" hello"]}}]}`, + `{"choices":[{"index":0,"text":"` + ct + `","token_ids":[1,2,3]}]}`, + `{"choices":[{"index":0,"text":"` + ct + `","prompt_token_ids":[4]}]}`, + `{"choices":[{"index":0,"text":"` + ct + `","new_field":"x"}]}`, + } { + if _, err := DecryptNonStreamResponseForEndpoint([]byte(bad), session, EndpointCompletions); !errors.Is(err, ErrDecryptionFailed) { + t.Errorf("non-stream %s: %v, want ErrDecryptionFailed", bad, err) + } + if _, err := DecryptSSEChunk(bad, session, EndpointCompletions); !errors.Is(err, ErrDecryptionFailed) { + t.Errorf("SSE %s: %v, want ErrDecryptionFailed", bad, err) + } + } +} diff --git a/internal/e2ee/nearcloud.go b/internal/e2ee/nearcloud.go index a08a0796..d4290dfe 100644 --- a/internal/e2ee/nearcloud.go +++ b/internal/e2ee/nearcloud.go @@ -12,12 +12,15 @@ import ( "errors" "fmt" "io" + "slices" "strings" "sync/atomic" "filippo.io/edwards25519" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/hkdf" + + "github.com/13rac1/teep/internal/jsonstrict" ) // NearCloudSession holds ephemeral Ed25519/X25519 key material for one @@ -151,7 +154,7 @@ func (s *NearCloudSession) IsResponseFieldEncrypted(fieldPath string, endpoint E EncFieldLogprobsContentToken, EncFieldLogprobsContentBytes, EncFieldLogprobsRefusalToken, EncFieldLogprobsRefusalBytes, EncFieldB64JSON, EncFieldRevisedPrompt, - EncFieldEmbedding, EncFieldRerankDocumentText: + EncFieldEmbedding, EncFieldRerankDocumentText, EncFieldCompletionText: return true } @@ -751,6 +754,107 @@ func EncryptEmbeddingsNearCloud(body []byte, modelKey NearModelKey) ([]byte, *Ne return out, session, nil } +// ErrCompletionsShape marks a completions request that NEAR's E2EE cannot +// carry; the request is not sent. +var ErrCompletionsShape = errors.New("completions request shape not supported with E2EE") + +// completionsAllowedFields are the completions request fields that carry no +// user text and make no response field carry prompt or completion content. +// Any other field fails with ErrCompletionsShape, so a new upstream field +// fails closed. +var completionsAllowedFields = map[string]bool{ + "model": true, "prompt": true, "max_tokens": true, "min_tokens": true, "temperature": true, + "top_p": true, "top_k": true, "min_p": true, "n": true, "best_of": true, "stream": true, + "stream_options": true, "stop": true, "seed": true, "presence_penalty": true, + "frequency_penalty": true, "repetition_penalty": true, "logit_bias": true, "user": true, +} + +// completionsOffFields are request fields that would send user text in +// plaintext, or make the response carry prompt or completion tokens outside +// choices[].text. They are accepted only with the listed "off" values: an +// integer 0 still turns logprobs on. +var completionsOffFields = map[string][]string{ + "suffix": {"null"}, + "logprobs": {"null"}, + "prompt_logprobs": {"null"}, + "echo": {"null", "false"}, + "return_token_ids": {"null", "false"}, +} + +// EncryptCompletionsNearCloud creates a NearCloud E2EE session and encrypts +// the "prompt" field of a completions request, a JSON string or an array of +// JSON strings. The model key must come from verified attestation. A prompt +// of token ids, an empty prompt array, a field outside +// completionsAllowedFields, and a field of completionsOffFields with a value +// that is not "off", fail with ErrCompletionsShape. +func EncryptCompletionsNearCloud(body []byte, modelKey NearModelKey) (out []byte, _ *NearCloudSession, err error) { + // A map keeps only the last of duplicate members, and the upstream can + // read another one: "logprobs":5,"logprobs":null would pass the check + // below as null. Duplicates at any depth are refused. + if err := jsonstrict.ValidateUniqueFields(body); err != nil { + return nil, nil, fmt.Errorf("%w: %w", ErrCompletionsShape, err) + } + session, full, err := newNearCloudSessionAndBody(body, modelKey, "completions") + if err != nil { + return nil, nil, err + } + // Every error return below zeroes the session; the returns set the + // named results to nil, so the local session is the one zeroed. + defer func() { + if err != nil { + session.Zero() + } + }() + for key, raw := range full { + if completionsAllowedFields[key] { + continue + } + off, known := completionsOffFields[key] + if !known || !slices.Contains(off, string(bytes.TrimSpace(raw))) { + return nil, nil, fmt.Errorf("%w: %s", ErrCompletionsShape, key) + } + } + promptRaw, ok := full["prompt"] + if !ok || IsJSONNull(promptRaw) { + return nil, nil, fmt.Errorf("%w: missing prompt", ErrCompletionsShape) + } + encrypt := func(item json.RawMessage, what string) (json.RawMessage, error) { + var s string + if err := json.Unmarshal(item, &s); err != nil { + return nil, fmt.Errorf("%w: %s is not a string", ErrCompletionsShape, what) + } + ct, err := EncryptXChaCha20([]byte(s), session.ModelX25519Pub()) + if err != nil { + return nil, fmt.Errorf("encrypt %s: %w", what, err) + } + enc, _ := json.Marshal(ct) //nolint:errchkjson // strings always marshal + return enc, nil + } + if trimmed := bytes.TrimSpace(promptRaw); len(trimmed) > 0 && trimmed[0] == '[' { + var arr []json.RawMessage + if err := json.Unmarshal(promptRaw, &arr); err != nil { + return nil, nil, fmt.Errorf("parse prompt array: %w", err) + } + if len(arr) == 0 { + return nil, nil, fmt.Errorf("%w: empty prompt array", ErrCompletionsShape) + } + for i := range arr { + if arr[i], err = encrypt(arr[i], fmt.Sprintf("prompt[%d]", i)); err != nil { + return nil, nil, err + } + } + full["prompt"], _ = json.Marshal(arr) //nolint:errchkjson // re-marshaling previously-unmarshaled JSON + } else { + enc, err := encrypt(promptRaw, "prompt") + if err != nil { + return nil, nil, err + } + full["prompt"] = enc + } + out, _ = json.Marshal(full) //nolint:errchkjson // re-marshaling previously-unmarshaled JSON + return out, session, nil +} + // EncryptRerankNearCloud creates a NearCloud E2EE session and encrypts the // "query" and "documents" fields of a rerank request. The model key must come from verified attestation. // diff --git a/internal/e2ee/relay.go b/internal/e2ee/relay.go index 82cfd424..c7ba1b2a 100644 --- a/internal/e2ee/relay.go +++ b/internal/e2ee/relay.go @@ -586,10 +586,51 @@ func applyDecryptedJSON(plaintext []byte) json.RawMessage { return b } +// decryptCompletionsChunk decrypts choices[].text of a /v1/completions SSE +// chunk, which is required to be encrypted, and re-serializes the chunk. +func decryptCompletionsChunk(full map[string]json.RawMessage, choices []map[string]json.RawMessage, session Decryptor) (string, error) { + for i := range choices { + if _, err := decryptCompletionChoice(choices[i], session, i); err != nil { + return "", err + } + } + choicesOut, _ := json.Marshal(choices) //nolint:errchkjson // re-marshaling previously-unmarshaled JSON + full["choices"] = choicesOut + out, _ := json.Marshal(full) //nolint:errchkjson // re-marshaling previously-unmarshaled JSON + return string(out), nil +} + +// completionChoiceKeys are the plaintext keys a completions choice may hold +// besides text: structural metadata only. +var completionChoiceKeys = map[string]bool{"index": true, "finish_reason": true, "stop_reason": true, "matched_stop": true} + +// completionChoiceNullKeys may appear in a completions choice only as null: +// a value would carry prompt or completion content outside choices[].text. +var completionChoiceNullKeys = map[string]bool{"logprobs": true, "prompt_logprobs": true, "token_ids": true, "prompt_token_ids": true} + +// decryptCompletionChoice decrypts choices[i].text of a completions response, +// which must be present and encrypted. Any key other than text and the +// structural metadata fails closed, and so does a non-null content key. +func decryptCompletionChoice(choice map[string]json.RawMessage, session Decryptor, i int) (bool, error) { + if _, ok := choice["text"]; !ok { + return false, fmt.Errorf("%w: choice[%d] has no text", ErrDecryptionFailed, i) + } + for key, raw := range choice { + switch { + case key == "text" || completionChoiceKeys[key]: + case completionChoiceNullKeys[key] && IsJSONNull(raw): + default: + return false, fmt.Errorf("%w: choice[%d].%s is not covered by E2EE", ErrDecryptionFailed, i, key) + } + } + return decryptMaybeEncryptedStringField(choice, "text", session, fmt.Sprintf("choice[%d]", i), EncFieldCompletionText, EndpointCompletions) +} + // DecryptSSEChunk parses one SSE data JSON payload, decrypts all encrypted -// fields in the delta object, and returns the JSON with plaintext substituted. -// The endpoint parameter identifies the proxy route kind; currently only EndpointChat is supported. -// Actual provider paths: /v1/chat/completions (NearCloud) or /api/v1/chat/completions (Venice). +// fields in the delta object (choices[].text for EndpointCompletions), and +// returns the JSON with plaintext substituted. +// Actual provider paths: /v1/chat/completions and /v1/completions (NEAR) or +// /api/v1/chat/completions (Venice). func DecryptSSEChunk(data string, session Decryptor, endpoint EndpointType) (string, error) { var full map[string]json.RawMessage if err := json.Unmarshal([]byte(data), &full); err != nil { @@ -617,6 +658,10 @@ func DecryptSSEChunk(data string, session Decryptor, endpoint EndpointType) (str return data, nil } + if endpoint == EndpointCompletions { + return decryptCompletionsChunk(full, choices, session) + } + deltaRaw, ok := choices[0]["delta"] if !ok { return data, nil @@ -857,7 +902,7 @@ func DecryptNonStreamResponseForEndpoint(body []byte, session Decryptor, endpoin // the allowlist and dispatch are co-located rather than split across two // separate guards. switch endpoint { - case EndpointChat: + case EndpointChat, EndpointCompletions: // choices[] already handled above; no data[] to decrypt. case EndpointImages: // Image generation endpoint path: /v1/images/generations. @@ -940,6 +985,13 @@ func decryptResponseChoices(choicesRaw json.RawMessage, session Decryptor, endpo changed := false for i, choice := range choices { + if endpoint == EndpointCompletions { + c, err := decryptCompletionChoice(choice, session, i) + if err := setChanged(&changed, c, err); err != nil { + return nil, err + } + continue + } msgRaw, ok := choice["message"] if !ok { continue diff --git a/internal/e2ee/session.go b/internal/e2ee/session.go index a7df610f..c75ed42c 100644 --- a/internal/e2ee/session.go +++ b/internal/e2ee/session.go @@ -32,6 +32,8 @@ const ( EndpointResponses EndpointType = "responses" // EndpointSpeech is /v1/audio/speech. EndpointSpeech EndpointType = "speech" + // EndpointCompletions is /v1/completions. + EndpointCompletions EndpointType = "completions" ) // EncField* constants identify the dot-notation sub-field paths within inference @@ -60,7 +62,12 @@ const ( EncFieldRevisedPrompt = "revised_prompt" EncFieldEmbedding = "embedding" EncFieldRerankDocumentText = "results[].document.text" - EncFieldScore = "score" + // EncFieldCompletionText is choices[].text of /v1/completions. The NEAR + // policy requires it encrypted on every endpoint; no other endpoint uses + // "text" as a policy path (chat uses content[].text, rerank + // results[].document.text). + EncFieldCompletionText = "text" + EncFieldScore = "score" ) // Decryptor is implemented by field-level E2EE sessions used by relay diff --git a/internal/e2ee/sse.go b/internal/e2ee/sse.go index 6a2bf8fe..c3d7c1e2 100644 --- a/internal/e2ee/sse.go +++ b/internal/e2ee/sse.go @@ -64,7 +64,7 @@ func CheckSSEEvent(line string) error { // CheckSSEEndMarker requires the NEAR chat completion marker. Other protocols // retain their own completion rules, including EHBP frame authentication. func CheckSSEEndMarker(session Decryptor, endpoint EndpointType, seen bool) error { - if _, near := session.(*NearCloudSession); near && endpoint == EndpointChat && !seen { + if _, near := session.(*NearCloudSession); near && (endpoint == EndpointChat || endpoint == EndpointCompletions) && !seen { return errors.New("NEAR SSE stream ended without [DONE]") } // E2EE v2 requires the [DONE] marker. The marker is not authenticated: diff --git a/internal/provider/neardirect/e2ee.go b/internal/provider/neardirect/e2ee.go index 567f863e..951edcbf 100644 --- a/internal/provider/neardirect/e2ee.go +++ b/internal/provider/neardirect/e2ee.go @@ -49,6 +49,8 @@ func (n *E2EE) EncryptRequestWithModelKey(body []byte, modelKey e2ee.NearModelKe encrypt = e2ee.EncryptRerankNearCloud case e2ee.EndpointScore: encrypt = e2ee.EncryptScoreNearCloud + case e2ee.EndpointCompletions: + encrypt = e2ee.EncryptCompletionsNearCloud default: return e2ee.EncryptResult{}, fmt.Errorf("NearAI E2EE not supported for endpoint %q", endpoint) } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index f3c5f563..17ea4c81 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -139,6 +139,10 @@ type Provider struct { // (e.g. "/v1/score"). Empty means unsupported. ScorePath string + // CompletionsPath is the upstream API path for text completions + // (e.g. "/v1/completions"). Empty means unsupported. + CompletionsPath string + // ResponsesPath is the upstream API path for responses // (e.g. "/v1/responses"). Empty means unsupported. ResponsesPath string diff --git a/internal/proxy/near_completions_test.go b/internal/proxy/near_completions_test.go new file mode 100644 index 00000000..f4e85eaa --- /dev/null +++ b/internal/proxy/near_completions_test.go @@ -0,0 +1,74 @@ +package proxy + +import ( + "encoding/hex" + "io" + "net/http" + "strings" + "testing" + + "github.com/13rac1/teep/internal/attestation" + "github.com/13rac1/teep/internal/e2ee" + "github.com/13rac1/teep/internal/tlsct/testtls" +) + +// A NearDirect completion is end-to-end encrypted: the upstream receives no +// plaintext prompt, the encrypted choices[].text is decrypted for the client, +// and e2ee_usable is confirmed. +func TestNearDirectCompletionsE2EE(t *testing.T) { + testtls.RunWithFallbackRoot(t, func(t *testing.T, authority *testtls.Authority) { + t.Helper() + upstream, fp := newTLSBindingTestServerWithHandler(t, authority, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if strings.Contains(string(body), "Say hello") { + t.Error("the plaintext prompt reached the upstream") + } + if r.URL.Path != "/v1/completions" { + t.Errorf("path = %s", r.URL.Path) + } + public, err := hex.DecodeString(r.Header.Get("X-Client-Pub-Key")) + if err != nil { + t.Error(err) + return + } + recipient, err := e2ee.Ed25519PubToX25519(public) + if err != nil { + t.Error(err) + return + } + ct, err := e2ee.EncryptXChaCha20([]byte(" hello"), recipient) + if err != nil { + t.Error(err) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"c","object":"text_completion","choices":[{"index":0,"text":"`+ct+`","finish_reason":"stop"}]}`) + })) + server, input, first := nearCompletionAuthorization(t, "neardirect", upstream.URL, fp) + request := *input + request.body = []byte(`{"model":"model","prompt":"Say hello","max_tokens":5}`) + request.path = "/v1/completions" + request.endpoint = e2ee.EndpointCompletions + rec := newInferenceRecorder() + out, err := server.inferAuthorized(t.Context(), rec, &request) + if err != nil || out.status != "ok" { + t.Fatalf("inferAuthorized: %v, status %q", err, out.status) + } + if !strings.Contains(rec.Body.String(), `"text":" hello"`) { + t.Fatalf("body = %s", rec.Body.String()) + } + current, ok := server.authorizations.acquire(input.key) + if !ok || current.generation != first.generation { + t.Fatal("authorization changed") + } + var usable bool + for _, f := range current.report.Factors { + if f.Name == attestation.FactorE2EEUsable { + usable = f.Status == attestation.Pass + } + } + if !usable { + t.Error("an E2EE completion did not confirm e2ee_usable") + } + }) +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 59f840ad..63e59b79 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -565,6 +565,7 @@ func New(cfg *config.Config) (*Server, error) { s.mux.HandleFunc("POST /v1/images/generations", s.handleEndpoint(&imagesEndpoint)) s.mux.HandleFunc("POST /v1/rerank", s.handleEndpoint(&rerankEndpoint)) s.mux.HandleFunc("POST /v1/score", s.handleEndpoint(&scoreEndpoint)) + s.mux.HandleFunc("POST /v1/completions", s.handleEndpoint(&completionsEndpoint)) s.mux.HandleFunc("POST /v1/responses", s.handleEndpoint(&responsesEndpoint)) s.mux.HandleFunc("POST /v1/audio/speech", s.handleEndpoint(&speechEndpoint)) s.mux.HandleFunc("GET /v1/{$}", handleV1Help) @@ -746,6 +747,9 @@ func fromConfig( p.ImagesPath = "/v1/images/generations" p.RerankPath = "/v1/rerank" p.ScorePath = "/v1/score" + // NEAR model hosts encrypt completions; NEAR's gateway does not, so + // nearcloud has no completions route. + p.CompletionsPath = "/v1/completions" rdVerifier := neardirect.ReportDataVerifier{} attester := neardirect.NewAttester(cp.BaseURL, cp.APIKey, offline) p.Attester = attester @@ -1687,6 +1691,14 @@ var ( return "", false }, } + completionsEndpoint = endpointConfig{ + name: "completions", + endpointType: e2ee.EndpointCompletions, + endpointPath: func(p *provider.Provider) string { return p.CompletionsPath }, + unsupported: "completions", + parseRequest: parseChatRequest, // reads only model and stream + contentType: "application/json", + } responsesEndpoint = endpointConfig{ name: "responses", endpointType: e2ee.EndpointResponses,