Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/api_support.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,15 @@ These are the OpenAI-style endpoints that clients use for inference:

`/v1/models` is a proxy-aggregated endpoint that returns the combined model list from all configured providers. Each model's `id` field is rewritten to `provider:upstreamID` (e.g. `venice:e2ee-qwen3-5-122b-a10b`, `neardirect:Qwen/Qwen3-VL-30B-A3B-Instruct`) so clients can route requests to the correct provider. NanoGPT lists only its TEE models, whose IDs start with `TEE/`; its other models serve no attestation. It is not included in the per-provider matrices below because it is handled entirely by the proxy, does not forward requests to individual providers, and is not E2EE-encrypted (GET request, no sensitive data).

Teep adds a `teep` member to each model and replaces any `teep` member that the provider sends. It holds the E2EE setting of the model's provider in the running config (`e2ee_enabled`), and the endpoints that the provider's E2EE covers and does not cover (`e2ee_covers`, `e2ee_does_not_cover`). These are the same lists that the dashboard shows. The coverage is the provider's: it does not state that each model serves every covered endpoint. With E2EE enabled, teep refuses an endpoint in `e2ee_does_not_cover` (see [Why teep refused my request](#why-teep-refused-my-request)). With E2EE off, both lists are empty, because teep then refuses no endpoint for E2EE coverage. Teep sends no request to the provider to build the member. For NearDirect with E2EE enabled:

```json
{"id": "neardirect:z-ai/glm-5.3-flash", "object": "model", "...": "...",
"teep": {"e2ee_enabled": true,
"e2ee_covers": ["/v1/chat/completions", "/v1/completions", "/v1/embeddings", "/v1/images/generations", "/v1/rerank", "/v1/score"],
"e2ee_does_not_cover": ["/v1/responses", "/v1/images/edits", "/v1/audio/transcriptions", "/v1/privacy/classify", "/v1/privacy/redact"]}}
```

### Operational Status, Monitoring, and Teep-Specific Endpoints

These are teep runtime/observability endpoints, not OpenAI-compatible inference APIs:
Expand Down
47 changes: 39 additions & 8 deletions internal/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -3304,13 +3304,14 @@ func (s *Server) fetchModels() []json.RawMessage {
slog.WarnContext(ctx, "model listing failed", "provider", prov.Name, "err", err)
return
}
teep := modelTeepMember(prov)
for _, raw := range models {
prefixed, err := prefixModelID(name, raw)
annotated, err := annotateModel(name, teep, raw)
if err != nil {
slog.WarnContext(ctx, "model ID prefix failed", "provider", name, "err", err)
slog.WarnContext(ctx, "model entry rewrite failed", "provider", name, "err", err)
continue
}
results[i] = append(results[i], prefixed)
results[i] = append(results[i], annotated)
}
}(i, name, prov)
}
Expand All @@ -3328,7 +3329,8 @@ func (s *Server) fetchModels() []json.RawMessage {
// handleModels returns available models from all configured providers in
// deterministic (sorted) provider order. Each model's "id" field is rewritten
// to "provider:upstreamID" so clients can route requests back to the correct
// provider. All other upstream model fields are preserved semantically.
// provider, and a "teep" member states the provider's E2EE coverage. All
// other upstream model fields are preserved semantically.
// Partial-success: a provider that fails listing is logged and skipped.
// Results are cached for modelsCacheTTL to avoid redundant upstream fetches.
// Concurrent requests coalesce via singleflight to prevent thundering herd.
Expand All @@ -3351,10 +3353,38 @@ func (s *Server) handleModels(w http.ResponseWriter, r *http.Request) {
writeModelsResponse(ctx, w, models)
}

// prefixModelID rewrites the "id" field of a JSON model object to
// "providerName:originalID", preserving all other fields. Returns an error if
// the object cannot be parsed or the "id" field is missing or not a string.
func prefixModelID(providerName string, raw json.RawMessage) (json.RawMessage, error) {
// teepModelInfo is the "teep" member of each /v1/models entry: whether E2EE
// is enabled for the entry's provider, and the endpoints that its E2EE covers
// and does not cover, from the same declarations as the dashboard. Both
// lists are empty when E2EE is off, because then teep refuses no endpoint for
// coverage. The coverage is the provider's: it does not state that each
// listed model serves every covered endpoint.
type teepModelInfo struct {
E2EEEnabled bool `json:"e2ee_enabled"`
E2EECovers []string `json:"e2ee_covers"`
E2EEDoesNotCover []string `json:"e2ee_does_not_cover"`
}

// modelTeepMember returns the encoded "teep" member for prov's models.
func modelTeepMember(prov *provider.Provider) json.RawMessage {
covered, notCovered := e2eeCoverage(prov)
// Empty, not nil, slices: the lists encode as [] and never as null.
info := teepModelInfo{E2EEEnabled: prov.E2EE, E2EECovers: []string{}, E2EEDoesNotCover: []string{}}
info.E2EECovers = append(info.E2EECovers, covered...)
info.E2EEDoesNotCover = append(info.E2EEDoesNotCover, notCovered...)
b, err := json.Marshal(info)
if err != nil {
panic("encode teep model member: " + err.Error()) // a struct of a bool and strings always encodes
}
return b
}

// annotateModel rewrites the "id" field of a JSON model object to
// "providerName:originalID" and sets its "teep" member to teep, replacing any
// "teep" member from the upstream, so only teep states the coverage. It
// preserves all other fields. Returns an error if the object cannot be parsed
// or the "id" field is missing or not a string.
func annotateModel(providerName string, teep, raw json.RawMessage) (json.RawMessage, error) {
var obj map[string]json.RawMessage
if err := json.Unmarshal(raw, &obj); err != nil {
return nil, err
Expand All @@ -3372,6 +3402,7 @@ func prefixModelID(providerName string, raw json.RawMessage) (json.RawMessage, e
return nil, err
}
obj["id"] = prefixed
obj["teep"] = teep
return json.Marshal(obj)
}

Expand Down
60 changes: 44 additions & 16 deletions internal/proxy/proxy_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync/atomic"
"testing"
Expand Down Expand Up @@ -198,54 +199,81 @@ func TestVerifyTinfoilSupplyChain_NonTinfoilFormat(t *testing.T) {
}

// --------------------------------------------------------------------------
// prefixModelID
// annotateModel
// --------------------------------------------------------------------------

func TestPrefixModelID(t *testing.T) {
func TestAnnotateModel(t *testing.T) {
teep := json.RawMessage(`{"e2ee_enabled":false,"e2ee_covers":[],"e2ee_does_not_cover":[]}`)
tests := []struct {
provName string
model json.RawMessage
wantID string
}{
{"venice", json.RawMessage(`{"id":"qwen3-32b","object":"model"}`), "venice:qwen3-32b"},
{"tinfoil_v3_cloud", json.RawMessage(`{"id":"llama3-3-70b"}`), "tinfoil_v3_cloud:llama3-3-70b"},
// An upstream "teep" member is replaced: only teep states the coverage.
{"venice", json.RawMessage(`{"id":"m","teep":{"e2ee_enabled":true}}`), "venice:m"},
}
for _, tc := range tests {
result, err := prefixModelID(tc.provName, tc.model)
result, err := annotateModel(tc.provName, teep, tc.model)
if err != nil {
t.Errorf("prefixModelID(%q, %s): %v", tc.provName, tc.model, err)
t.Errorf("annotateModel(%q, %s): %v", tc.provName, tc.model, err)
continue
}
var obj map[string]json.RawMessage
var obj struct {
ID string `json:"id"`
Teep json.RawMessage `json:"teep"`
}
if err := json.Unmarshal(result, &obj); err != nil {
t.Errorf("unmarshal result: %v", err)
continue
}
var got string
if err := json.Unmarshal(obj["id"], &got); err != nil {
t.Errorf("unmarshal id: %v", err)
continue
}
if got != tc.wantID {
t.Errorf("prefixModelID(%q, %s) id = %q, want %q", tc.provName, tc.model, got, tc.wantID)
if obj.ID != tc.wantID || string(obj.Teep) != string(teep) {
t.Errorf("annotateModel(%q, %s) = id %q teep %s, want %q %s", tc.provName, tc.model, obj.ID, obj.Teep, tc.wantID, teep)
}
}
}

func TestPrefixModelID_MissingID(t *testing.T) {
_, err := prefixModelID("test", json.RawMessage(`{"object":"model"}`))
func TestAnnotateModel_MissingID(t *testing.T) {
_, err := annotateModel("test", json.RawMessage(`{}`), json.RawMessage(`{"object":"model"}`))
if err == nil {
t.Error("expected error for missing id field")
}
}

func TestPrefixModelID_InvalidJSON(t *testing.T) {
_, err := prefixModelID("test", json.RawMessage(`not json`))
func TestAnnotateModel_InvalidJSON(t *testing.T) {
_, err := annotateModel("test", json.RawMessage(`{}`), json.RawMessage(`not json`))
if err == nil {
t.Error("expected error for invalid JSON")
}
}

// The "teep" member holds the provider's E2EE setting and the same coverage
// lists as the dashboard; with E2EE off both lists are empty arrays.
func TestModelTeepMember(t *testing.T) {
s := newCoverageServer(t, "neardirect")
prov := s.providers["neardirect"]
var on teepModelInfo
if err := json.Unmarshal(modelTeepMember(prov), &on); err != nil {
t.Fatal(err)
}
dash := s.buildDashboardData().Providers["neardirect"]
if !on.E2EEEnabled {
t.Error("E2EE on: e2ee_enabled is false")
}
if len(on.E2EECovers) == 0 || !slices.Equal(on.E2EECovers, dash.E2EECovers) {
t.Errorf("E2EE on: e2ee_covers = %v, dashboard = %v", on.E2EECovers, dash.E2EECovers)
}
if len(on.E2EEDoesNotCover) == 0 || !slices.Equal(on.E2EEDoesNotCover, dash.E2EEDoesNotCover) {
t.Errorf("E2EE on: e2ee_does_not_cover = %v, dashboard = %v", on.E2EEDoesNotCover, dash.E2EEDoesNotCover)
}
off := *prov
off.E2EE = false
if got := string(modelTeepMember(&off)); got != `{"e2ee_enabled":false,"e2ee_covers":[],"e2ee_does_not_cover":[]}` {
t.Errorf("E2EE off: teep = %s", got)
}
}

// --------------------------------------------------------------------------
// extractMultipartField — new branches not covered by relay_internal_test.go
// --------------------------------------------------------------------------
Expand Down
10 changes: 10 additions & 0 deletions internal/proxy/proxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1280,6 +1280,11 @@ func TestHandleModels(t *testing.T) {
Object string `json:"object"`
OwnedBy string `json:"owned_by"`
Type string `json:"type"`
Teep struct {
E2EEEnabled *bool `json:"e2ee_enabled"`
E2EECovers []string `json:"e2ee_covers"`
E2EEDoesNotCover []string `json:"e2ee_does_not_cover"`
} `json:"teep"`
}
ids := map[string]bool{}
for _, raw := range result.Data {
Expand All @@ -1298,6 +1303,11 @@ func TestHandleModels(t *testing.T) {
if m.Created != 1727966436 {
t.Errorf("model %q: created = %d, want 1727966436", m.ID, m.Created)
}
// The test config has E2EE off: teep refuses no endpoint for coverage.
if m.Teep.E2EEEnabled == nil || *m.Teep.E2EEEnabled || m.Teep.E2EECovers == nil || len(m.Teep.E2EECovers) != 0 ||
m.Teep.E2EEDoesNotCover == nil || len(m.Teep.E2EEDoesNotCover) != 0 {
t.Errorf("model %q: teep = %+v, want E2EE off and two empty lists", m.ID, m.Teep)
}
}
if !ids["venice:e2ee-test-model"] {
t.Error("venice:e2ee-test-model not in response")
Expand Down
Loading