From ebcb05969d4ddd88f9382b5938aa8c64718a0747 Mon Sep 17 00:00:00 2001 From: kittors Date: Fri, 14 Aug 2026 11:45:24 +0800 Subject: [PATCH 1/2] feat(codex): converge device fingerprints and add TLS fingerprint support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several people sharing one Codex OAuth account each send their own installation/session/thread identifiers, so upstream counts them as separate devices and sessions and applies device quota limits to the account. Convergence rewrites those identifiers to account-derived stable values before the request leaves the proxy, in four strengths (off/device/session/full, default session). Derived values are deterministic, so an account keeps the same installation identity across restarts instead of looking freshly reinstalled. Header and body rewriting share one resolved id set, because a turn id that disagrees between the x-codex-turn-metadata header and client_metadata would be a fingerprint of its own. Only identifiers the client actually sent are rewritten: adding a field upstream was never going to receive would widen the fingerprint rather than converge it. The websocket path is narrowed to device scope, since Session_id there is pinned to Conversation_id and prompt_cache_key for prompt cache matching. Also adds internal/tlsfingerprint, which gives upstream requests a ClientHello matching a real client (chrome/firefox/safari/edge/ios/android/randomized) instead of Go's distinctive default. It manages HTTP/2 itself because Go only routes crypto/tls connections to its HTTP/2 stack, and tunnels CONNECT itself because http.Transport would otherwise keep the tunnelled connection for its own TLS. Trust settings (CA bundle, skip-verify) are threaded through so enabling fingerprinting cannot silently drop them. Off by default: the right profile depends on what the operator has verified against the upstream. Verification: ./scripts/ci-pr.sh (gofmt, secret scan, structure check, go vet, go test ./..., golangci-lint, go build) — all green. Co-Authored-By: Claude Opus 5 --- .../config/codex_convergence_config_test.go | 103 +++++ internal/config/identity_fingerprint.go | 81 +++- .../management/settings/runtimeconfig/spec.go | 6 +- internal/runtime/executor/codex_auth.go | 11 + internal/runtime/executor/codex_executor.go | 6 + .../executor/codex_fingerprint_convergence.go | 425 +++++++++++++++++ .../codex_fingerprint_convergence_test.go | 367 +++++++++++++++ .../executor/codex_websockets_helpers.go | 6 + internal/runtime/executor/proxy_helpers.go | 9 +- .../executor/tls_fingerprint_transport.go | 108 +++++ internal/tlsfingerprint/profile.go | 94 ++++ internal/tlsfingerprint/proxy_test.go | 240 ++++++++++ internal/tlsfingerprint/roundtripper.go | 431 ++++++++++++++++++ internal/tlsfingerprint/roundtripper_test.go | 273 +++++++++++ 14 files changed, 2155 insertions(+), 5 deletions(-) create mode 100644 internal/config/codex_convergence_config_test.go create mode 100644 internal/runtime/executor/codex_fingerprint_convergence.go create mode 100644 internal/runtime/executor/codex_fingerprint_convergence_test.go create mode 100644 internal/runtime/executor/tls_fingerprint_transport.go create mode 100644 internal/tlsfingerprint/profile.go create mode 100644 internal/tlsfingerprint/proxy_test.go create mode 100644 internal/tlsfingerprint/roundtripper.go create mode 100644 internal/tlsfingerprint/roundtripper_test.go diff --git a/internal/config/codex_convergence_config_test.go b/internal/config/codex_convergence_config_test.go new file mode 100644 index 000000000..b45fda7a7 --- /dev/null +++ b/internal/config/codex_convergence_config_test.go @@ -0,0 +1,103 @@ +package config + +import ( + "encoding/json" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestNormalizeCodexIdentityFingerprintConvergenceDefaults(t *testing.T) { + got := NormalizeCodexIdentityFingerprint(CodexIdentityFingerprintConfig{}) + if got.ConvergenceMode != CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want the session default", got.ConvergenceMode) + } + + got = NormalizeCodexIdentityFingerprint(CodexIdentityFingerprintConfig{ConvergenceMode: "nonsense"}) + if got.ConvergenceMode != CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want an invalid value to fall back", got.ConvergenceMode) + } + + got = NormalizeCodexIdentityFingerprint(CodexIdentityFingerprintConfig{ConvergenceMode: " OFF "}) + if got.ConvergenceMode != CodexFingerprintConvergenceOff { + t.Fatalf("convergence mode = %q, want off to survive trimming and case", got.ConvergenceMode) + } +} + +func TestCodexIdentityFingerprintConvergenceRoundTrip(t *testing.T) { + raw := []byte(` +enabled: true +convergence-mode: full +installation-id: 11111111-2222-3333-4444-555555555555 +tls-fingerprint: + enabled: true + profile: firefox +`) + var fromYAML CodexIdentityFingerprintConfig + if err := yaml.Unmarshal(raw, &fromYAML); err != nil { + t.Fatalf("yaml unmarshal: %v", err) + } + if fromYAML.ConvergenceMode != CodexFingerprintConvergenceFull { + t.Fatalf("convergence mode = %q, want full", fromYAML.ConvergenceMode) + } + if fromYAML.InstallationID != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("installation id = %q, want the configured value", fromYAML.InstallationID) + } + if !fromYAML.TLSFingerprint.Enabled || fromYAML.TLSFingerprint.Profile != "firefox" { + t.Fatalf("tls fingerprint = %+v, want enabled firefox", fromYAML.TLSFingerprint) + } + + encoded, err := json.Marshal(NormalizeCodexIdentityFingerprint(fromYAML)) + if err != nil { + t.Fatalf("json marshal: %v", err) + } + var fromJSON CodexIdentityFingerprintConfig + if err = json.Unmarshal(encoded, &fromJSON); err != nil { + t.Fatalf("json unmarshal: %v", err) + } + if fromJSON.ConvergenceMode != CodexFingerprintConvergenceFull { + t.Fatalf("convergence mode did not survive the JSON round trip: %q", fromJSON.ConvergenceMode) + } + if !fromJSON.TLSFingerprint.Enabled || fromJSON.TLSFingerprint.Profile != "firefox" { + t.Fatalf("tls fingerprint did not survive the JSON round trip: %+v", fromJSON.TLSFingerprint) + } +} + +func TestCleanTLSFingerprintDropsUnknownProfile(t *testing.T) { + got := CleanTLSFingerprint(TLSFingerprintConfig{Enabled: true, Profile: "netscape"}) + if got.Profile != "" { + t.Fatalf("profile = %q, want an unknown client name dropped", got.Profile) + } + if !got.Enabled { + t.Fatal("dropping an unknown profile must not silently disable the feature") + } + + got = CleanTLSFingerprint(TLSFingerprintConfig{Enabled: true, Profile: " ChRoMe "}) + if got.Profile != "chrome" { + t.Fatalf("profile = %q, want it normalized to chrome", got.Profile) + } +} + +// Legacy runtime payloads predate these fields; they must still be recognised as +// the old "disabled default" so upgrading does not resurrect a stale config. +func TestCodexLegacyDefaultDetectionWithNewFields(t *testing.T) { + legacy := CodexIdentityFingerprintConfig{} + if !codexLegacyDefaultDisabled(legacy) { + t.Fatal("an empty legacy payload must still be treated as the disabled default") + } + + configured := CodexIdentityFingerprintConfig{ConvergenceMode: CodexFingerprintConvergenceOff} + if codexLegacyDefaultDisabled(configured) { + t.Fatal("an explicitly configured convergence mode must not be discarded as legacy") + } + + pinned := CodexIdentityFingerprintConfig{InstallationID: "abc"} + if codexLegacyDefaultDisabled(pinned) { + t.Fatal("a pinned installation id must not be discarded as legacy") + } + + tlsOn := CodexIdentityFingerprintConfig{TLSFingerprint: TLSFingerprintConfig{Enabled: true}} + if codexLegacyDefaultDisabled(tlsOn) { + t.Fatal("an enabled TLS fingerprint must not be discarded as legacy") + } +} diff --git a/internal/config/identity_fingerprint.go b/internal/config/identity_fingerprint.go index dfdf2f854..c0fa90e24 100644 --- a/internal/config/identity_fingerprint.go +++ b/internal/config/identity_fingerprint.go @@ -4,6 +4,7 @@ import ( "encoding/json" "strings" + "github.com/router-for-me/CLIProxyAPI/v6/internal/tlsfingerprint" "gopkg.in/yaml.v3" ) @@ -17,6 +18,21 @@ const ( DefaultCodexFingerprintBetaFeatures = "" DefaultCodexFingerprintSessionMode = "per-request" + // Codex device fingerprint convergence modes. Upstream counts distinct + // installation/session/thread identifiers to derive per-account device and + // session quotas, so several people sharing one OAuth account each burn a + // separate device slot. Convergence rewrites those identifiers to + // account-stable values before the request leaves the proxy. + CodexFingerprintConvergenceOff = "off" + CodexFingerprintConvergenceDevice = "device" + CodexFingerprintConvergenceSession = "session" + CodexFingerprintConvergenceFull = "full" + + // DefaultCodexFingerprintConvergenceMode converges installation and session + // while keeping one thread per real client session, which is the shape a + // single user spawning sub-agents produces upstream. + DefaultCodexFingerprintConvergenceMode = CodexFingerprintConvergenceSession + DefaultClaudeFingerprintCLIVersion = "2.1.161" DefaultClaudeFingerprintEntrypoint = "cli" DefaultClaudeFingerprintAnthropicBeta = "claude-code-20250219,oauth-2025-04-20,interleaved-thinking-2025-05-14,prompt-caching-scope-2026-01-05,effort-2025-11-24,context-management-2025-06-27,extended-cache-ttl-2025-04-11" @@ -53,7 +69,40 @@ type CodexIdentityFingerprintConfig struct { SessionMode string `yaml:"session-mode,omitempty" json:"session-mode,omitempty"` SessionID string `yaml:"session-id,omitempty" json:"session-id,omitempty"` CustomHeaders map[string]string `yaml:"custom-headers,omitempty" json:"custom-headers,omitempty"` - enabledSet bool + // ConvergenceMode selects how aggressively per-client device identifiers are + // rewritten to account-stable values: off, device, session or full. + ConvergenceMode string `yaml:"convergence-mode,omitempty" json:"convergence-mode,omitempty"` + // InstallationID pins the converged x-codex-installation-id. Leave empty to + // derive a stable value from the account key; set it to replay an + // installation id captured from a real Codex client. + InstallationID string `yaml:"installation-id,omitempty" json:"installation-id,omitempty"` + // TLSFingerprint shapes the ClientHello sent to the Codex upstream. + TLSFingerprint TLSFingerprintConfig `yaml:"tls-fingerprint,omitempty" json:"tls-fingerprint,omitempty"` + enabledSet bool +} + +// TLSFingerprintConfig selects the TLS ClientHello presented to an upstream. +// +// This is off by default: a ClientHello that does not match the client the +// User-Agent claims to be is its own inconsistency, and the correct profile +// depends on what the operator has verified against the upstream. Turn it on +// after confirming the profile with a fingerprint echo service. +type TLSFingerprintConfig struct { + Enabled bool `yaml:"enabled" json:"enabled"` + // Profile names a client to imitate: chrome, firefox, safari, edge, ios, + // android or randomized. Empty selects the built-in default. + Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` +} + +// CleanTLSFingerprint normalizes a TLS fingerprint block, dropping a profile +// name that does not identify a known client. +func CleanTLSFingerprint(in TLSFingerprintConfig) TLSFingerprintConfig { + out := in + out.Profile = strings.ToLower(strings.TrimSpace(out.Profile)) + if out.Profile != "" && !tlsfingerprint.IsValidProfile(out.Profile) { + out.Profile = "" + } + return out } // DefaultCodexIdentityFingerprint returns the recommended Codex identity template. @@ -67,6 +116,20 @@ func DefaultCodexIdentityFingerprint() CodexIdentityFingerprintConfig { BetaFeatures: DefaultCodexFingerprintBetaFeatures, SessionMode: DefaultCodexFingerprintSessionMode, CustomHeaders: map[string]string{}, + + ConvergenceMode: DefaultCodexFingerprintConvergenceMode, + } +} + +// IsValidCodexFingerprintConvergenceMode reports whether mode is one of the +// four supported convergence strengths. +func IsValidCodexFingerprintConvergenceMode(mode string) bool { + switch mode { + case CodexFingerprintConvergenceOff, CodexFingerprintConvergenceDevice, + CodexFingerprintConvergenceSession, CodexFingerprintConvergenceFull: + return true + default: + return false } } @@ -235,6 +298,9 @@ func NormalizeCodexIdentityFingerprint(in CodexIdentityFingerprintConfig) CodexI if out.SessionMode != "server-stable" && out.SessionMode != "fixed" && out.SessionMode != "per-request" { out.SessionMode = DefaultCodexFingerprintSessionMode } + if !IsValidCodexFingerprintConvergenceMode(out.ConvergenceMode) { + out.ConvergenceMode = DefaultCodexFingerprintConvergenceMode + } return out } @@ -253,6 +319,12 @@ func CleanCodexIdentityFingerprint(in CodexIdentityFingerprintConfig) CodexIdent if out.SessionMode != "" && out.SessionMode != "server-stable" && out.SessionMode != "fixed" && out.SessionMode != "per-request" { out.SessionMode = DefaultCodexFingerprintSessionMode } + out.ConvergenceMode = strings.TrimSpace(strings.ToLower(out.ConvergenceMode)) + if out.ConvergenceMode != "" && !IsValidCodexFingerprintConvergenceMode(out.ConvergenceMode) { + out.ConvergenceMode = DefaultCodexFingerprintConvergenceMode + } + out.InstallationID = strings.TrimSpace(out.InstallationID) + out.TLSFingerprint = CleanTLSFingerprint(out.TLSFingerprint) out.CustomHeaders = cleanIdentityFingerprintHeaders(out.CustomHeaders) return out } @@ -419,7 +491,9 @@ func defaultXAIIdentityFingerprintEnabled(in XAIIdentityFingerprintConfig) XAIId } func codexLegacyDefaultDisabled(fp CodexIdentityFingerprintConfig) bool { - if fp.Enabled || strings.TrimSpace(fp.SessionID) != "" || len(fp.CustomHeaders) > 0 { + if fp.Enabled || strings.TrimSpace(fp.SessionID) != "" || + strings.TrimSpace(fp.InstallationID) != "" || fp.TLSFingerprint.Enabled || + strings.TrimSpace(fp.TLSFingerprint.Profile) != "" || len(fp.CustomHeaders) > 0 { return false } defaults := DefaultCodexIdentityFingerprint() @@ -428,7 +502,8 @@ func codexLegacyDefaultDisabled(fp CodexIdentityFingerprintConfig) bool { emptyOrEqual(fp.Originator, defaults.Originator) && emptyOrEqual(fp.WebsocketBeta, defaults.WebsocketBeta) && emptyOrEqual(fp.BetaFeatures, defaults.BetaFeatures) && - emptyOrEqual(fp.SessionMode, defaults.SessionMode) + emptyOrEqual(fp.SessionMode, defaults.SessionMode) && + emptyOrEqual(fp.ConvergenceMode, defaults.ConvergenceMode) } func claudeLegacyDefaultDisabled(fp ClaudeIdentityFingerprintConfig) bool { diff --git a/internal/management/settings/runtimeconfig/spec.go b/internal/management/settings/runtimeconfig/spec.go index a51e6753c..def447ccb 100644 --- a/internal/management/settings/runtimeconfig/spec.go +++ b/internal/management/settings/runtimeconfig/spec.go @@ -400,7 +400,11 @@ func codexIdentityFingerprintMeaningful(fp config.CodexIdentityFingerprintConfig strings.TrimSpace(clean.Version) != "" || strings.TrimSpace(clean.Originator) != "" || strings.TrimSpace(clean.WebsocketBeta) != "" || - strings.TrimSpace(clean.SessionMode) != "" + strings.TrimSpace(clean.SessionMode) != "" || + strings.TrimSpace(clean.ConvergenceMode) != "" || + strings.TrimSpace(clean.InstallationID) != "" || + clean.TLSFingerprint.Enabled || + strings.TrimSpace(clean.TLSFingerprint.Profile) != "" } func claudeIdentityFingerprintMeaningful(fp config.ClaudeIdentityFingerprintConfig) bool { diff --git a/internal/runtime/executor/codex_auth.go b/internal/runtime/executor/codex_auth.go index 1fe12a812..271c43274 100644 --- a/internal/runtime/executor/codex_auth.go +++ b/internal/runtime/executor/codex_auth.go @@ -45,6 +45,17 @@ func applyCodexHeaders(r *http.Request, cfg *config.Config, auth *cliproxyauth.A misc.EnsureHeader(r.Header, ginHeaders, "User-Agent", codexUserAgent) } + // Device fingerprint convergence runs after the client headers above have + // been copied in, so it rewrites the values that actually reach upstream. + // The executor resolves the id set before translating the body and stores it + // on the context; requests that never pass through that path (token probes, + // PrepareRequest) resolve their own set here. + convergedIDs := codexConvergedIDsFromContext(r.Context()) + if convergedIDs == nil { + convergedIDs = resolveCodexConvergedIDs(cfg, auth, ginHeaders) + } + applyCodexConvergenceHeaders(r.Header, convergedIDs, ginHeaders) + // Upstream codex-tui behavior: only attach Session_id when the UA indicates a desktop client. if strings.Contains(r.Header.Get("User-Agent"), "Mac OS") && strings.TrimSpace(r.Header.Get("Session_id")) == "" { r.Header.Set("Session_id", uuid.NewString()) diff --git a/internal/runtime/executor/codex_executor.go b/internal/runtime/executor/codex_executor.go index ea259db7c..241698fac 100644 --- a/internal/runtime/executor/codex_executor.go +++ b/internal/runtime/executor/codex_executor.go @@ -120,6 +120,7 @@ func (e *CodexExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth, re return e.executeCompact(ctx, auth, req, opts) } req, opts = maybeStripCodexHistoryDataURLImagesOnRequest(req, opts) + ctx, convergedIDs := e.prepareCodexConvergence(ctx, auth) execCtx := newExecutionContext(ctx, e.Identifier(), e.cfg, auth, req, opts, ExecutionOptions{ TargetFormat: sdktranslator.FromString("codex"), }) @@ -158,6 +159,7 @@ func (e *CodexExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth, re "safety_identifier", }) body = maybeEnsureCodexImageGenerationTool(body, auth, execCtx.BaseModel, codexAdmissionHeadersFromContext(execCtx.Context)) + body = applyCodexConvergenceClientMetadata(body, convergedIDs) url := strings.TrimSuffix(baseURL, "/") + "/responses" httpReq, err := e.cacheHelper(execCtx.Context, auth, execCtx.SourceFormat, url, req, body) @@ -250,6 +252,7 @@ func (e *CodexExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth, re } func (e *CodexExecutor) executeCompact(ctx context.Context, auth *cliproxyauth.Auth, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (resp cliproxyexecutor.Response, err error) { + ctx, convergedIDs := e.prepareCodexConvergence(ctx, auth) execCtx := newExecutionContext(ctx, e.Identifier(), e.cfg, auth, req, opts, ExecutionOptions{ TargetFormat: sdktranslator.FromString("openai-response"), }) @@ -277,6 +280,7 @@ func (e *CodexExecutor) executeCompact(ctx context.Context, auth *cliproxyauth.A body = ensureTranslatedCodexModel(body, execCtx.BaseModel) body = sanitizeCodexResponsesRequest(body) body, _ = sjson.DeleteBytes(body, "stream") + body = applyCodexConvergenceClientMetadata(body, convergedIDs) url := strings.TrimSuffix(baseURL, "/") + "/responses/compact" httpReq, err := e.cacheHelper(execCtx.Context, auth, execCtx.SourceFormat, url, req, body) @@ -329,6 +333,7 @@ func (e *CodexExecutor) ExecuteStream(ctx context.Context, auth *cliproxyauth.Au // Shrink multi-MB Desktop history data URLs before translation/sanitize so later // body-level passes never see the full base64 history. req, opts = maybeStripCodexHistoryDataURLImagesOnRequest(req, opts) + ctx, convergedIDs := e.prepareCodexConvergence(ctx, auth) execCtx := newExecutionContext(ctx, e.Identifier(), e.cfg, auth, req, opts, ExecutionOptions{ TargetFormat: sdktranslator.FromString("codex"), TranslateAsStream: true, @@ -366,6 +371,7 @@ func (e *CodexExecutor) ExecuteStream(ctx context.Context, auth *cliproxyauth.Au }) body = ensureTranslatedCodexModel(body, execCtx.BaseModel) body = maybeEnsureCodexImageGenerationTool(body, auth, execCtx.BaseModel, codexAdmissionHeadersFromContext(execCtx.Context)) + body = applyCodexConvergenceClientMetadata(body, convergedIDs) url := strings.TrimSuffix(baseURL, "/") + "/responses" httpReq, err := e.cacheHelper(execCtx.Context, auth, execCtx.SourceFormat, url, req, body) diff --git a/internal/runtime/executor/codex_fingerprint_convergence.go b/internal/runtime/executor/codex_fingerprint_convergence.go new file mode 100644 index 000000000..7f3f837c8 --- /dev/null +++ b/internal/runtime/executor/codex_fingerprint_convergence.go @@ -0,0 +1,425 @@ +package executor + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/google/uuid" + "github.com/router-for-me/CLIProxyAPI/v6/internal/config" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/auth" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// Codex device fingerprint convergence. +// +// A Codex client stamps every request with identifiers that upstream reads as +// "which machine" and "which conversation": installation id, session id, thread +// id and window id. When several people share one OAuth account through this +// proxy, each of their clients contributes its own set, so upstream sees a +// crowd of devices and sessions on a single account and applies device/session +// quota limits accordingly. +// +// Convergence rewrites those identifiers to values derived from the account +// itself, so the traffic reads as one installation instead of N. The four modes +// trade fidelity for aggressiveness; see config.CodexFingerprintConvergence*. +// +// Identifiers are derived (not random) so they survive restarts: the same +// account always produces the same installation and session id, which is what +// makes the device look persistent rather than freshly reinstalled per boot. + +// codexConvergedIDs is the identifier set applied to one outbound request. +// +// Header rewriting and body rewriting must share a single instance: turn id is +// generated fresh per request, and a mismatch between the header copy and the +// client_metadata copy is itself a fingerprint. +type codexConvergedIDs struct { + mode string + installationID string + sessionID string + threadID string + turnID string + windowID string + // turnStartedAtUnixMs is stamped from the same clock that produced turnID + // (a time-ordered UUIDv7). Keeping the client's original timestamp next to a + // server-generated turn id would leave the two disagreeing about when the + // turn started, which is a discrepancy a real client never produces. + turnStartedAtUnixMs int64 +} + +type codexConvergedIDsContextKeyType struct{} + +var codexConvergedIDsContextKey codexConvergedIDsContextKeyType + +// withCodexConvergedIDs stores a resolved identifier set on the context so the +// body rewrite in the executor and the header rewrite in applyCodexHeaders act +// on the same values. +func withCodexConvergedIDs(ctx context.Context, ids *codexConvergedIDs) context.Context { + if ctx == nil || ids == nil { + return ctx + } + return context.WithValue(ctx, codexConvergedIDsContextKey, ids) +} + +func codexConvergedIDsFromContext(ctx context.Context) *codexConvergedIDs { + if ctx == nil { + return nil + } + ids, _ := ctx.Value(codexConvergedIDsContextKey).(*codexConvergedIDs) + return ids +} + +// codexConvergenceMode reports the effective convergence strength for this auth. +// Convergence rides on the Codex identity fingerprint switch: turning that off +// means "leave my outbound identity alone", which has to include device ids. +func codexConvergenceMode(cfg *config.Config, auth *cliproxyauth.Auth) string { + if cfg == nil || !cfg.IdentityFingerprint.Codex.Enabled { + return config.CodexFingerprintConvergenceOff + } + // API-key credentials are not OAuth accounts and carry no device quota, so + // rewriting their identifiers would only add noise. + if auth != nil && auth.Attributes != nil { + if strings.TrimSpace(auth.Attributes["api_key"]) != "" { + return config.CodexFingerprintConvergenceOff + } + } + mode := strings.TrimSpace(strings.ToLower(cfg.IdentityFingerprint.Codex.ConvergenceMode)) + if mode == "" { + mode = config.DefaultCodexFingerprintConvergenceMode + } + if !config.IsValidCodexFingerprintConvergenceMode(mode) { + mode = config.DefaultCodexFingerprintConvergenceMode + } + return mode +} + +// deriveStableUUIDv4 turns a seed into a fixed UUIDv4-shaped string. The same +// seed always yields the same value, which is what makes a derived installation +// id look like a real one that persists across restarts. +func deriveStableUUIDv4(seed string) string { + h := sha256.Sum256([]byte(seed)) + b := h[:16] + b[6] = (b[6] & 0x0f) | 0x40 // version 4 + b[8] = (b[8] & 0x3f) | 0x80 // variant 1 + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + binary.BigEndian.Uint32(b[0:4]), + binary.BigEndian.Uint16(b[4:6]), + binary.BigEndian.Uint16(b[6:8]), + binary.BigEndian.Uint16(b[8:10]), + b[10:16]) +} + +// codexConvergenceScope returns the stable per-account seed component. It +// reuses the identity fingerprint account key so a credential keeps its device +// identity after moving between tenants, and falls back to the broader session +// isolation scope for auths that carry no resolvable subject. +func codexConvergenceScope(auth *cliproxyauth.Auth) string { + if accountKey, _ := identityFingerprintAccount(auth); strings.TrimSpace(accountKey) != "" { + return strings.TrimSpace(accountKey) + } + return sessionIsolationScope(auth) +} + +// resolveConvergedInstallationID prefers an operator-pinned installation id so a +// value captured from a real Codex client can be replayed; otherwise it derives +// one from the account scope. +func resolveConvergedInstallationID(cfg *config.Config, scope string) string { + if cfg != nil { + if pinned := strings.TrimSpace(cfg.IdentityFingerprint.Codex.InstallationID); pinned != "" { + return pinned + } + } + if scope == "" { + return "" + } + return deriveStableUUIDv4("clirelay:codex-install-id:v1:" + scope) +} + +func resolveConvergedSessionID(scope string) string { + if scope == "" { + return "" + } + return deriveStableUUIDv4("clirelay:codex-session-id:v1:" + scope) +} + +// resolveConvergedThreadID derives one thread per real client session, so a +// shared account still looks like a single user running several agents rather +// than a single endless conversation. +func resolveConvergedThreadID(scope, clientSessionID string) string { + if scope == "" || clientSessionID == "" { + return "" + } + return deriveStableUUIDv4("clirelay:codex-thread-id:v1:" + scope + ":" + clientSessionID) +} + +// extractClientSessionID reads the client's own session identifier before any +// rewrite. Codex CLI sends the hyphenated form; the underscore form is accepted +// as a fallback because some clients and earlier proxy layers emit it. +func extractClientSessionID(h http.Header) string { + if h == nil { + return "" + } + for _, key := range []string{"session-id", "session_id"} { + if v := strings.TrimSpace(h.Get(key)); v != "" { + return v + } + } + return "" +} + +// resolveCodexConvergedIDs computes the identifier set for one request. +// It returns nil when convergence is off or no stable account scope exists, +// in which case callers must leave the client identifiers untouched. +// +// The result carries a freshly generated turn id, so callers must resolve once +// per request and share the result between header and body rewriting. +func resolveCodexConvergedIDs(cfg *config.Config, auth *cliproxyauth.Auth, clientHeaders http.Header) *codexConvergedIDs { + mode := codexConvergenceMode(cfg, auth) + if mode == config.CodexFingerprintConvergenceOff { + return nil + } + scope := codexConvergenceScope(auth) + installationID := resolveConvergedInstallationID(cfg, scope) + if installationID == "" { + return nil + } + + ids := &codexConvergedIDs{mode: mode, installationID: installationID} + if mode == config.CodexFingerprintConvergenceDevice { + return ids + } + + ids.sessionID = resolveConvergedSessionID(scope) + if ids.sessionID == "" { + // Without a stable session id the session/full modes cannot converge + // anything beyond the device; degrade instead of emitting empty headers. + ids.mode = config.CodexFingerprintConvergenceDevice + return ids + } + switch mode { + case config.CodexFingerprintConvergenceSession: + ids.threadID = resolveConvergedThreadID(scope, extractClientSessionID(clientHeaders)) + if ids.threadID == "" { + ids.threadID = ids.sessionID + } + case config.CodexFingerprintConvergenceFull: + ids.threadID = ids.sessionID + } + turnStartedAt := time.Now() + ids.turnID = newCodexTurnID() + ids.turnStartedAtUnixMs = turnStartedAt.UnixMilli() + ids.windowID = ids.threadID + ":0" + return ids +} + +// prepareCodexConvergence resolves the identifier set once per request and +// returns a context carrying it. Header rewriting in applyCodexHeaders reads it +// back from there, so the headers and the request body agree on the per-request +// turn id instead of each generating its own. +func (e *CodexExecutor) prepareCodexConvergence(ctx context.Context, auth *cliproxyauth.Auth) (context.Context, *codexConvergedIDs) { + ids := resolveCodexConvergedIDs(e.cfg, auth, identityFingerprintHeadersFromContext(ctx)) + if ids == nil { + return ctx, nil + } + return withCodexConvergedIDs(ctx, ids), ids +} + +// newCodexTurnID mirrors the time-ordered turn identifier a real client emits. +func newCodexTurnID() string { + if v, err := uuid.NewV7(); err == nil { + return v.String() + } + return uuid.NewString() +} + +// applyCodexConvergenceHeaders rewrites the device identifiers on the outbound +// request. It must run after client headers have been copied in, so the values +// it writes are the ones that survive to the wire. +// +// clientHeaders is the original inbound request. Identifiers this proxy does not +// otherwise forward are only emitted when the client sent them: this layer +// converges what upstream would have seen anyway and must not introduce a header +// upstream was never going to receive, which would add fingerprint surface +// rather than remove it. The turn metadata payload is exempt because +// applyCodexHeaders forwards it verbatim, so its embedded identifiers do reach +// upstream and are the main leak this feature closes. +func applyCodexConvergenceHeaders(h http.Header, ids *codexConvergedIDs, clientHeaders http.Header) { + if h == nil || ids == nil || ids.installationID == "" { + return + } + + setIfClientSent(h, clientHeaders, "X-Codex-Installation-Id", ids.installationID) + + if ids.mode == config.CodexFingerprintConvergenceDevice { + rewriteCodexTurnMetadataHeader(h, ids.turnMetadataFields()) + return + } + + setIfClientSent(h, clientHeaders, "X-Codex-Window-Id", ids.windowID) + setIfClientSent(h, clientHeaders, "Thread-Id", ids.threadID) + // x-client-request-id is forwarded verbatim by applyCodexHeaders, so it + // reaches upstream and must be converged whether or not the client set it. + h.Set("X-Client-Request-Id", ids.threadID) + // Session_id always goes out: without convergence the per-request session + // mode stamps a fresh random id on every call, which makes one account look + // like an endless stream of new sessions. Both spellings are pinned so a + // client that used the other form cannot re-expose its original value. + h.Set("Session-Id", ids.sessionID) + h.Set("Session_id", ids.sessionID) + + rewriteCodexTurnMetadataHeader(h, ids.turnMetadataFields()) +} + +// setIfClientSent writes value only when the inbound request carried that +// header, so convergence never widens what upstream can observe. +func setIfClientSent(h, clientHeaders http.Header, key, value string) { + if value == "" { + return + } + if strings.TrimSpace(h.Get(key)) == "" { + if clientHeaders == nil || strings.TrimSpace(clientHeaders.Get(key)) == "" { + return + } + } + h.Set(key, value) +} + +// deviceOnly narrows an identifier set to device-level convergence. +// +// The websocket path uses this: there Session_id is deliberately pinned to +// Conversation_id and prompt_cache_key so upstream can match the prompt cache, +// and rewriting the session or thread would break that three-way association for +// a saving that only matters on the HTTP path. Converging the installation is +// safe because nothing else is keyed off it. +func (ids *codexConvergedIDs) deviceOnly() *codexConvergedIDs { + if ids == nil { + return nil + } + narrowed := *ids + narrowed.mode = config.CodexFingerprintConvergenceDevice + return &narrowed +} + +// turnMetadataFields returns the turn-metadata entries this mode converges. +// Device mode only claims the installation, so it must not touch the session, +// thread or turn fields the client sent. +func (ids *codexConvergedIDs) turnMetadataFields() map[string]any { + fields := map[string]any{"installation_id": ids.installationID} + if ids.mode == config.CodexFingerprintConvergenceDevice { + return fields + } + fields["session_id"] = ids.sessionID + fields["thread_id"] = ids.threadID + fields["turn_id"] = ids.turnID + fields["window_id"] = ids.windowID + fields["turn_started_at_unix_ms"] = ids.turnStartedAtUnixMs + return fields +} + +// rewriteCodexTurnMetadataHeader replaces the given fields inside the +// x-codex-turn-metadata JSON payload while preserving every other field the +// client sent (sandbox, thread_source and similar), because dropping them would +// itself change the fingerprint. +func rewriteCodexTurnMetadataHeader(h http.Header, fields map[string]any) { + raw := strings.TrimSpace(h.Get("X-Codex-Turn-Metadata")) + rebuilt, ok := rewriteCodexTurnMetadataJSON(raw, fields) + if !ok { + return + } + h.Set("X-Codex-Turn-Metadata", rebuilt) +} + +// rewriteCodexTurnMetadataJSON applies fields to a turn-metadata JSON document. +// It reports false when the payload is absent or not parseable, so callers leave +// the original value untouched rather than replacing it with a partial rewrite. +// +// Only keys the client already sent are replaced. Convergence exists to shrink +// what upstream can distinguish, so adding a field the client omitted would work +// against the goal even when the field is one a real client usually sends. +func rewriteCodexTurnMetadataJSON(raw string, fields map[string]any) (string, bool) { + raw = strings.TrimSpace(raw) + if raw == "" || !gjson.Valid(raw) { + return "", false + } + rebuilt := raw + for _, key := range sortedCodexMetadataKeys(fields) { + if !gjson.Get(rebuilt, key).Exists() { + continue + } + next, err := sjson.Set(rebuilt, key, fields[key]) + if err != nil { + return "", false + } + rebuilt = next + } + return rebuilt, true +} + +// applyCodexConvergenceClientMetadata rewrites the identifiers duplicated in the +// request body. Fields are only replaced when the client already sent +// client_metadata: fabricating the object for clients that never send it would +// add a fingerprint rather than converge one. +func applyCodexConvergenceClientMetadata(body []byte, ids *codexConvergedIDs) []byte { + if len(body) == 0 || ids == nil || ids.installationID == "" { + return body + } + if !gjson.GetBytes(body, "client_metadata").IsObject() { + return body + } + + fields := map[string]any{ + "client_metadata.x-codex-installation-id": ids.installationID, + } + if ids.mode != config.CodexFingerprintConvergenceDevice { + fields["client_metadata.session_id"] = ids.sessionID + fields["client_metadata.thread_id"] = ids.threadID + fields["client_metadata.turn_id"] = ids.turnID + fields["client_metadata.x-codex-window-id"] = ids.windowID + } + + for _, key := range sortedCodexMetadataKeys(fields) { + // Only rewrite identifiers the client actually sent: adding a field it + // never included would widen the fingerprint instead of converging it. + if !gjson.GetBytes(body, key).Exists() { + continue + } + next, err := sjson.SetBytes(body, key, fields[key]) + if err != nil { + return body + } + body = next + } + return rewriteCodexClientMetadataEmbeddedTurnMetadata(body, ids) +} + +// rewriteCodexClientMetadataEmbeddedTurnMetadata patches the turn metadata that +// clients embed inside client_metadata as a JSON string. +func rewriteCodexClientMetadataEmbeddedTurnMetadata(body []byte, ids *codexConvergedIDs) []byte { + raw := gjson.GetBytes(body, "client_metadata.x-codex-turn-metadata").String() + rebuilt, ok := rewriteCodexTurnMetadataJSON(raw, ids.turnMetadataFields()) + if !ok { + return body + } + next, err := sjson.SetBytes(body, "client_metadata.x-codex-turn-metadata", rebuilt) + if err != nil { + return body + } + return next +} + +// sortedCodexMetadataKeys keeps rewrite order deterministic so the resulting +// JSON field order does not vary between otherwise identical requests. +func sortedCodexMetadataKeys(fields map[string]any) []string { + keys := make([]string, 0, len(fields)) + for key := range fields { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/internal/runtime/executor/codex_fingerprint_convergence_test.go b/internal/runtime/executor/codex_fingerprint_convergence_test.go new file mode 100644 index 000000000..7f11a0d15 --- /dev/null +++ b/internal/runtime/executor/codex_fingerprint_convergence_test.go @@ -0,0 +1,367 @@ +package executor + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v6/internal/config" + "github.com/router-for-me/CLIProxyAPI/v6/internal/util" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/auth" + "github.com/tidwall/gjson" +) + +func codexConvergenceTestConfig(mode string) *config.Config { + return &config.Config{ + IdentityFingerprint: config.IdentityFingerprintConfig{ + Codex: config.CodexIdentityFingerprintConfig{ + Enabled: true, + UserAgent: "codex_cli_rs/0.144.1", + Originator: "codex_cli_rs", + SessionMode: "per-request", + ConvergenceMode: mode, + }, + }, + } +} + +func codexConvergenceTestAuth(id string) *cliproxyauth.Auth { + return &cliproxyauth.Auth{ + ID: id, + Provider: "codex", + Metadata: map[string]any{ + "access_token": "codex-token", + "account_id": id + "-account", + }, + } +} + +func TestResolveCodexConvergedIDsIsStableAcrossRequests(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + auth := codexConvergenceTestAuth("codex-stable") + + headers := http.Header{} + headers.Set("session-id", "client-session-a") + + first := resolveCodexConvergedIDs(cfg, auth, headers) + second := resolveCodexConvergedIDs(cfg, auth, headers) + if first == nil || second == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if first.installationID != second.installationID { + t.Fatalf("installation id drifted between requests: %q vs %q", first.installationID, second.installationID) + } + if first.sessionID != second.sessionID { + t.Fatalf("session id drifted between requests: %q vs %q", first.sessionID, second.sessionID) + } + if first.threadID != second.threadID { + t.Fatalf("thread id drifted for the same client session: %q vs %q", first.threadID, second.threadID) + } + // Turn id is per-request by design; a constant turn id would be the anomaly. + if first.turnID == second.turnID { + t.Fatal("turn id was reused across requests, want a fresh value each turn") + } +} + +func TestResolveCodexConvergedIDsSeparatesAccounts(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + headers := http.Header{} + headers.Set("session-id", "shared-client-session") + + a := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-a"), headers) + b := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-b"), headers) + if a == nil || b == nil { + t.Fatal("convergence returned nil for accounts with resolvable scopes") + } + if a.installationID == b.installationID { + t.Fatal("two accounts collapsed onto one installation id") + } + if a.sessionID == b.sessionID { + t.Fatal("two accounts collapsed onto one session id") + } +} + +func TestResolveCodexConvergedIDsDerivesThreadPerClientSession(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + auth := codexConvergenceTestAuth("codex-threads") + + first := http.Header{} + first.Set("session-id", "client-session-1") + second := http.Header{} + second.Set("session-id", "client-session-2") + + a := resolveCodexConvergedIDs(cfg, auth, first) + b := resolveCodexConvergedIDs(cfg, auth, second) + if a == nil || b == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if a.sessionID != b.sessionID { + t.Fatal("session mode must converge both clients onto one session id") + } + if a.threadID == b.threadID { + t.Fatal("distinct client sessions must map to distinct thread ids") + } +} + +func TestResolveCodexConvergedIDsFullCollapsesThread(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceFull) + auth := codexConvergenceTestAuth("codex-full") + + first := http.Header{} + first.Set("session-id", "client-session-1") + second := http.Header{} + second.Set("session-id", "client-session-2") + + a := resolveCodexConvergedIDs(cfg, auth, first) + b := resolveCodexConvergedIDs(cfg, auth, second) + if a == nil || b == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if a.threadID != b.threadID || a.threadID != a.sessionID { + t.Fatalf("full mode must collapse every client onto the session thread: %q vs %q", a.threadID, b.threadID) + } +} + +func TestResolveCodexConvergedIDsOffAndAPIKey(t *testing.T) { + auth := codexConvergenceTestAuth("codex-off") + if ids := resolveCodexConvergedIDs(codexConvergenceTestConfig(config.CodexFingerprintConvergenceOff), auth, nil); ids != nil { + t.Fatal("off mode must not converge anything") + } + + disabled := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + disabled.IdentityFingerprint.Codex.Enabled = false + if ids := resolveCodexConvergedIDs(disabled, auth, nil); ids != nil { + t.Fatal("disabling the Codex identity fingerprint must also disable convergence") + } + + apiKeyAuth := codexConvergenceTestAuth("codex-api-key") + apiKeyAuth.Attributes = map[string]string{"api_key": "sk-test"} + if ids := resolveCodexConvergedIDs(codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession), apiKeyAuth, nil); ids != nil { + t.Fatal("API key credentials carry no device quota and must not be converged") + } +} + +func TestResolveCodexConvergedIDsHonoursPinnedInstallationID(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceDevice) + cfg.IdentityFingerprint.Codex.InstallationID = "11111111-2222-3333-4444-555555555555" + + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-pinned"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if ids.installationID != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("installation id = %q, want the operator-pinned value", ids.installationID) + } + if ids.sessionID != "" || ids.threadID != "" { + t.Fatal("device mode must leave session and thread identifiers untouched") + } +} + +func TestApplyCodexHeadersConvergesTurnMetadata(t *testing.T) { + gin.SetMode(gin.TestMode) + ginCtx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ginCtx.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + ginCtx.Request.Header.Set("session-id", "client-session") + ginCtx.Request.Header.Set("X-Codex-Turn-Metadata", + `{"installation_id":"client-install","session_id":"client-session","thread_id":"client-thread","turn_id":"client-turn","window_id":"client-window","sandbox":"workspace-write"}`) + ginCtx.Request.Header.Set("X-Client-Request-Id", "client-request") + + req := httptest.NewRequest(http.MethodPost, "https://chatgpt.com/backend-api/codex/responses", nil) + req = req.WithContext(context.WithValue(req.Context(), util.ContextKeyGin, ginCtx)) + + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + auth := codexConvergenceTestAuth("codex-headers") + applyCodexHeaders(req, cfg, auth, "token", true) + + metadata := req.Header.Get("X-Codex-Turn-Metadata") + for _, field := range []string{"installation_id", "session_id", "thread_id", "turn_id", "window_id"} { + got := gjson.Get(metadata, field).String() + if got == "" { + t.Fatalf("turn metadata %s was dropped", field) + } + if got == "client-"+shortFieldName(field) { + t.Fatalf("turn metadata %s still carries the client value %q", field, got) + } + } + // Unrelated fields must survive: dropping them would itself change the fingerprint. + if got := gjson.Get(metadata, "sandbox").String(); got != "workspace-write" { + t.Fatalf("sandbox = %q, want the client value to be preserved", got) + } + if got := req.Header.Get("X-Client-Request-Id"); got == "client-request" || got == "" { + t.Fatalf("X-Client-Request-Id = %q, want a converged value", got) + } + + ids := resolveCodexConvergedIDs(cfg, auth, ginCtx.Request.Header) + if got := req.Header.Get("Session_id"); got != ids.sessionID { + t.Fatalf("Session_id = %q, want converged %q", got, ids.sessionID) + } + if got := req.Header.Get("Session-Id"); got != ids.sessionID { + t.Fatalf("Session-Id = %q, want converged %q", got, ids.sessionID) + } + if got := gjson.Get(metadata, "installation_id").String(); got != ids.installationID { + t.Fatalf("turn metadata installation_id = %q, want converged %q", got, ids.installationID) + } +} + +// shortFieldName maps a turn metadata field to the suffix used by the client +// placeholder values in the test above. +func shortFieldName(field string) string { + switch field { + case "installation_id": + return "install" + case "session_id": + return "session" + case "thread_id": + return "thread" + case "turn_id": + return "turn" + case "window_id": + return "window" + default: + return field + } +} + +func TestApplyCodexHeadersDoesNotIntroduceUnsentIdentifiers(t *testing.T) { + gin.SetMode(gin.TestMode) + ginCtx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ginCtx.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + ginCtx.Request.Header.Set("session-id", "client-session") + + req := httptest.NewRequest(http.MethodPost, "https://chatgpt.com/backend-api/codex/responses", nil) + req = req.WithContext(context.WithValue(req.Context(), util.ContextKeyGin, ginCtx)) + + applyCodexHeaders(req, codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession), + codexConvergenceTestAuth("codex-nosurface"), "token", true) + + // The client sent none of these, so convergence must not hand upstream a + // header it would never have received. + for _, key := range []string{"X-Codex-Installation-Id", "X-Codex-Window-Id", "Thread-Id"} { + if got := req.Header.Get(key); got != "" { + t.Fatalf("%s = %q, want no header when the client sent none", key, got) + } + } +} + +func TestApplyCodexConvergenceClientMetadataRewritesOnlyPresentFields(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-body"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + body := []byte(`{"model":"gpt-5","client_metadata":{"session_id":"client-session","x-codex-installation-id":"client-install","cli_version":"0.144.1"}}`) + got := applyCodexConvergenceClientMetadata(body, ids) + + if v := gjson.GetBytes(got, "client_metadata.session_id").String(); v != ids.sessionID { + t.Fatalf("client_metadata.session_id = %q, want converged %q", v, ids.sessionID) + } + if v := gjson.GetBytes(got, "client_metadata.x-codex-installation-id").String(); v != ids.installationID { + t.Fatalf("client_metadata installation id = %q, want converged %q", v, ids.installationID) + } + if v := gjson.GetBytes(got, "client_metadata.cli_version").String(); v != "0.144.1" { + t.Fatalf("unrelated client_metadata field was altered: %q", v) + } + // thread_id was absent from the client payload and must stay absent. + if gjson.GetBytes(got, "client_metadata.thread_id").Exists() { + t.Fatal("convergence added a client_metadata field the client never sent") + } + if v := gjson.GetBytes(got, "model").String(); v != "gpt-5" { + t.Fatalf("model = %q, want the request to be otherwise untouched", v) + } +} + +func TestApplyCodexConvergenceLeavesOmittedTurnMetadataFieldsAbsent(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-partial"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + headers := http.Header{} + headers.Set("X-Codex-Turn-Metadata", `{"installation_id":"client-install","sandbox":"read-only"}`) + applyCodexConvergenceHeaders(headers, ids, http.Header{}) + + metadata := headers.Get("X-Codex-Turn-Metadata") + if got := gjson.Get(metadata, "installation_id").String(); got != ids.installationID { + t.Fatalf("installation_id = %q, want the converged value", got) + } + // The client omitted these; convergence must not introduce them. + for _, field := range []string{"session_id", "thread_id", "turn_id", "window_id", "turn_started_at_unix_ms"} { + if gjson.Get(metadata, field).Exists() { + t.Fatalf("turn metadata gained %s, which the client never sent", field) + } + } + if got := gjson.Get(metadata, "sandbox").String(); got != "read-only" { + t.Fatalf("sandbox = %q, want unrelated fields preserved", got) + } +} + +func TestApplyCodexConvergenceClientMetadataSkipsBodiesWithoutIt(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-nobody"), nil) + + body := []byte(`{"model":"gpt-5","input":[]}`) + got := applyCodexConvergenceClientMetadata(body, ids) + if string(got) != string(body) { + t.Fatalf("body without client_metadata was modified: %s", got) + } + if gjson.GetBytes(got, "client_metadata").Exists() { + t.Fatal("convergence fabricated a client_metadata object") + } +} + +func TestApplyCodexConvergenceHeadersSharesTurnIDWithBody(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-shared"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + headers := http.Header{} + headers.Set("X-Codex-Turn-Metadata", `{"turn_id":"client-turn"}`) + applyCodexConvergenceHeaders(headers, ids, http.Header{}) + + body := []byte(`{"client_metadata":{"turn_id":"client-turn"}}`) + body = applyCodexConvergenceClientMetadata(body, ids) + + headerTurn := gjson.Get(headers.Get("X-Codex-Turn-Metadata"), "turn_id").String() + bodyTurn := gjson.GetBytes(body, "client_metadata.turn_id").String() + if headerTurn == "" || headerTurn != bodyTurn { + t.Fatalf("header turn id %q and body turn id %q must match", headerTurn, bodyTurn) + } +} + +func TestDeviceOnlyLeavesSessionUntouched(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-ws"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + headers := http.Header{} + headers.Set("Session_id", "prompt-cache-key") + headers.Set("Conversation_id", "prompt-cache-key") + applyCodexConvergenceHeaders(headers, ids.deviceOnly(), http.Header{}) + + if got := headers.Get("Session_id"); got != "prompt-cache-key" { + t.Fatalf("Session_id = %q, want the prompt cache association preserved", got) + } + if got := headers.Get("Conversation_id"); got != "prompt-cache-key" { + t.Fatalf("Conversation_id = %q, want it preserved", got) + } +} + +func TestCodexConvergenceDefaultsToSessionMode(t *testing.T) { + cfg := codexConvergenceTestConfig("") + if got := codexConvergenceMode(cfg, codexConvergenceTestAuth("codex-default")); got != config.CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want the session default", got) + } + + cfg = codexConvergenceTestConfig("nonsense") + if got := codexConvergenceMode(cfg, codexConvergenceTestAuth("codex-bogus")); got != config.CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want an unrecognised value to fall back to the default", got) + } +} diff --git a/internal/runtime/executor/codex_websockets_helpers.go b/internal/runtime/executor/codex_websockets_helpers.go index e810bd06a..8a247d173 100644 --- a/internal/runtime/executor/codex_websockets_helpers.go +++ b/internal/runtime/executor/codex_websockets_helpers.go @@ -207,6 +207,12 @@ func applyCodexWebsocketHeaders(ctx context.Context, headers http.Header, cfg *c misc.EnsureHeader(headers, ginHeaders, "User-Agent", codexUserAgent) } + // Device fingerprint convergence, narrowed to the installation identifier: + // the websocket handshake pins Session_id to Conversation_id and + // prompt_cache_key above, so converging the session here would break prompt + // cache matching. + applyCodexConvergenceHeaders(headers, resolveCodexConvergedIDs(cfg, auth, ginHeaders).deviceOnly(), ginHeaders) + // Match upstream: only attach Session_id when UA indicates a desktop client, and do not forward UA over websocket. if strings.Contains(headers.Get("User-Agent"), "Mac OS") && strings.TrimSpace(headers.Get("Session_id")) == "" { headers.Set("Session_id", uuid.NewString()) diff --git a/internal/runtime/executor/proxy_helpers.go b/internal/runtime/executor/proxy_helpers.go index d6cea9826..303980bd4 100644 --- a/internal/runtime/executor/proxy_helpers.go +++ b/internal/runtime/executor/proxy_helpers.go @@ -65,8 +65,15 @@ func newProxyAwareHTTPClient(ctx context.Context, cfg *config.Config, auth *clip } recordRequestLogEgressRoute(ctx, cfg, auth, proxyURL) + // A TLS fingerprint transport dials and tunnels on its own, because the + // ClientHello has to be emitted by utls rather than by http.Transport. It + // therefore replaces the transport selection below instead of wrapping it. + if rt := tlsFingerprintRoundTripper(cfg, auth, proxyURL); rt != nil { + httpClient.Transport = rt + } + // If we have a proxy URL configured, set up the transport - if proxyURL != "" { + if proxyURL != "" && httpClient.Transport == nil { transport := cachedProxyTransport(proxyURL, cfgToSDKCfg(cfg)) if transport != nil { httpClient.Transport = transport diff --git a/internal/runtime/executor/tls_fingerprint_transport.go b/internal/runtime/executor/tls_fingerprint_transport.go new file mode 100644 index 000000000..2552a2976 --- /dev/null +++ b/internal/runtime/executor/tls_fingerprint_transport.go @@ -0,0 +1,108 @@ +package executor + +import ( + "net/http" + "strings" + "sync" + + "github.com/router-for-me/CLIProxyAPI/v6/internal/config" + "github.com/router-for-me/CLIProxyAPI/v6/internal/tlsfingerprint" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/auth" + log "github.com/sirupsen/logrus" +) + +// TLS fingerprint transports are cached process-wide for the same reason plain +// transports are: they own connection pools, so building one per request would +// force a fresh TLS handshake on every call. + +type tlsFingerprintTransportKey struct { + profile string + proxyURL string + preferIPv4 bool + insecureSkipVerify bool + caCert string + // caCertStat changes when the bundle on disk is replaced, so a rotated + // certificate produces a new transport instead of reusing one pinned to the + // old trust root. + caCertStat string +} + +var tlsFingerprintTransports = struct { + sync.Mutex + entries map[tlsFingerprintTransportKey]http.RoundTripper +}{entries: map[tlsFingerprintTransportKey]http.RoundTripper{}} + +// codexTLSFingerprintConfig returns the TLS fingerprint settings that apply to +// this auth, and whether fingerprinting is active for it. +// +// The settings live under the Codex identity fingerprint block and only apply +// to Codex traffic: a ClientHello has to match the client the rest of the +// request claims to be, so it cannot be configured independently of the +// identity headers. +func codexTLSFingerprintConfig(cfg *config.Config, auth *cliproxyauth.Auth) (config.TLSFingerprintConfig, bool) { + if cfg == nil || auth == nil { + return config.TLSFingerprintConfig{}, false + } + if !strings.EqualFold(strings.TrimSpace(auth.Provider), "codex") { + return config.TLSFingerprintConfig{}, false + } + tlsCfg := cfg.IdentityFingerprint.Codex.TLSFingerprint + if !tlsCfg.Enabled { + return config.TLSFingerprintConfig{}, false + } + return tlsCfg, true +} + +// tlsFingerprintRoundTripper returns a cached fingerprinting round tripper for +// this auth, or nil when fingerprinting does not apply. +// +// A construction failure is logged and returns nil so the caller falls back to +// the standard transport: a misconfigured profile must not take upstream +// traffic down, it only means the ClientHello stays the Go default. +func tlsFingerprintRoundTripper(cfg *config.Config, auth *cliproxyauth.Auth, proxyURL string) http.RoundTripper { + tlsCfg, ok := codexTLSFingerprintConfig(cfg, auth) + if !ok { + return nil + } + + profile := tlsfingerprint.NormalizeProfileName(tlsCfg.Profile) + if profile == "" { + profile = tlsfingerprint.DefaultProfile + } + key := tlsFingerprintTransportKey{ + profile: profile, + proxyURL: strings.TrimSpace(proxyURL), + } + // The fingerprint transport does its own TLS, so it has to honour the same + // trust settings the standard transport applies via util.ApplyTLSConfig; + // otherwise enabling fingerprinting would silently drop a configured CA + // bundle or an operator's decision to skip verification. + if sdkCfg := cfgToSDKCfg(cfg); sdkCfg != nil { + key.preferIPv4 = sdkCfg.PreferIPv4 + key.insecureSkipVerify = sdkCfg.InsecureSkipVerify + key.caCert = strings.TrimSpace(sdkCfg.CACert) + key.caCertStat = caCertStatFingerprint(key.caCert) + } + + tlsFingerprintTransports.Lock() + defer tlsFingerprintTransports.Unlock() + if cached, exists := tlsFingerprintTransports.entries[key]; exists { + return cached + } + + rt, err := tlsfingerprint.New(tlsfingerprint.Options{ + Profile: profile, + ProxyURL: key.proxyURL, + PreferIPv4: key.preferIPv4, + InsecureSkipVerify: key.insecureSkipVerify, + CACertPath: key.caCert, + }) + if err != nil { + log.WithError(err).Warn("tls fingerprint: falling back to the default transport") + // Cached as nil so a broken profile is not re-reported on every request. + tlsFingerprintTransports.entries[key] = nil + return nil + } + tlsFingerprintTransports.entries[key] = rt + return rt +} diff --git a/internal/tlsfingerprint/profile.go b/internal/tlsfingerprint/profile.go new file mode 100644 index 000000000..547bbb5c5 --- /dev/null +++ b/internal/tlsfingerprint/profile.go @@ -0,0 +1,94 @@ +// Package tlsfingerprint gives upstream requests a configurable TLS ClientHello +// fingerprint. +// +// Go's crypto/tls emits a ClientHello that no shipping browser or CLI produces, +// so JA3/JA4 based filtering can single out proxy traffic before a single byte +// of the request is seen. Sending a ClientHello that matches a real client +// removes that signal. +// +// Profiles are deliberately named after real clients rather than raw cipher +// lists: the point is to match a client that exists in the wild, and utls keeps +// the underlying byte sequences current as those clients ship new versions. +package tlsfingerprint + +import ( + "sort" + "strings" + + utls "github.com/refraction-networking/utls" +) + +// Profile names accepted by configuration. These are the stable identifiers +// exposed to operators; the utls ClientHelloID behind each one may advance as +// the library tracks new browser releases. +const ( + ProfileChrome = "chrome" + ProfileFirefox = "firefox" + ProfileSafari = "safari" + ProfileEdge = "edge" + ProfileIOS = "ios" + ProfileAndroid = "android" + ProfileRandomized = "randomized" +) + +// DefaultProfile is used when TLS fingerprinting is enabled without naming a +// profile. Chrome is the most common ClientHello on the public internet, so it +// is the least remarkable choice. +const DefaultProfile = ProfileChrome + +// profiles maps configuration names to utls ClientHello templates. +// +// The _Auto variants track the newest version utls implements, which is what +// keeps a profile from ageing into a fingerprint of its own once the real +// client has moved on. +var profiles = map[string]utls.ClientHelloID{ + ProfileChrome: utls.HelloChrome_Auto, + ProfileFirefox: utls.HelloFirefox_Auto, + ProfileSafari: utls.HelloSafari_Auto, + ProfileEdge: utls.HelloEdge_Auto, + ProfileIOS: utls.HelloIOS_Auto, + ProfileAndroid: utls.HelloAndroid_11_OkHttp, + // Randomized generates a fresh, internally consistent ClientHello per + // connection. It defeats fingerprint matching but is itself unusual, since + // no real client varies its ClientHello between connections. + ProfileRandomized: utls.HelloRandomizedALPN, +} + +// ResolveProfile maps a configured profile name to its ClientHello template. +// It reports false for unknown names so callers can fail loudly at config load +// rather than silently falling back to a fingerprint the operator did not ask +// for. +func ResolveProfile(name string) (utls.ClientHelloID, bool) { + normalized := NormalizeProfileName(name) + if normalized == "" { + return profiles[DefaultProfile], true + } + id, ok := profiles[normalized] + return id, ok +} + +// NormalizeProfileName trims and lowercases a profile name for comparison. +func NormalizeProfileName(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} + +// IsValidProfile reports whether name identifies a known profile. An empty name +// is valid and selects DefaultProfile. +func IsValidProfile(name string) bool { + if NormalizeProfileName(name) == "" { + return true + } + _, ok := ResolveProfile(name) + return ok +} + +// AvailableProfiles lists the supported profile names in a stable order, for +// configuration validation messages and management surfaces. +func AvailableProfiles() []string { + names := make([]string, 0, len(profiles)) + for name := range profiles { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/tlsfingerprint/proxy_test.go b/internal/tlsfingerprint/proxy_test.go new file mode 100644 index 000000000..d5ea0d4cb --- /dev/null +++ b/internal/tlsfingerprint/proxy_test.go @@ -0,0 +1,240 @@ +package tlsfingerprint + +import ( + "bufio" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// connectProxy is a minimal HTTP CONNECT proxy used to exercise the tunnelling +// path, which http.Transport would normally own. +type connectProxy struct { + listener net.Listener + wg sync.WaitGroup + + mu sync.Mutex + requireAuth string + observedTargets []string + observedAuthHdrs []string +} + +func newConnectProxy(t *testing.T, requireAuth string) *connectProxy { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + p := &connectProxy{listener: listener, requireAuth: requireAuth} + p.wg.Add(1) + go p.serve() + t.Cleanup(func() { + _ = listener.Close() + p.wg.Wait() + }) + return p +} + +func (p *connectProxy) addr() string { return p.listener.Addr().String() } + +func (p *connectProxy) targets() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.observedTargets...) +} + +func (p *connectProxy) authHeaders() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.observedAuthHdrs...) +} + +func (p *connectProxy) serve() { + defer p.wg.Done() + for { + clientConn, err := p.listener.Accept() + if err != nil { + return + } + p.wg.Add(1) + go func() { + defer p.wg.Done() + p.handle(clientConn) + }() + } +} + +func (p *connectProxy) handle(clientConn net.Conn) { + defer func() { _ = clientConn.Close() }() + + reader := bufio.NewReader(clientConn) + req, err := http.ReadRequest(reader) + if err != nil { + return + } + if req.Method != http.MethodConnect { + _, _ = io.WriteString(clientConn, "HTTP/1.1 405 Method Not Allowed\r\n\r\n") + return + } + + p.mu.Lock() + p.observedTargets = append(p.observedTargets, req.Host) + p.observedAuthHdrs = append(p.observedAuthHdrs, req.Header.Get("Proxy-Authorization")) + required := p.requireAuth + p.mu.Unlock() + + if required != "" && req.Header.Get("Proxy-Authorization") != required { + _, _ = io.WriteString(clientConn, "HTTP/1.1 407 Proxy Authentication Required\r\n\r\n") + return + } + + upstream, err := net.Dial("tcp", req.Host) + if err != nil { + _, _ = io.WriteString(clientConn, "HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + } + defer func() { _ = upstream.Close() }() + + if _, err = io.WriteString(clientConn, "HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + return + } + + var pipeWG sync.WaitGroup + pipeWG.Add(2) + go func() { + defer pipeWG.Done() + _, _ = io.Copy(upstream, reader) + }() + go func() { + defer pipeWG.Done() + _, _ = io.Copy(clientConn, upstream) + }() + pipeWG.Wait() +} + +func newTLSEchoServer(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Proto", r.Proto) + _, _ = io.WriteString(w, "tunnelled") + })) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + return server +} + +func TestRoundTripThroughHTTPConnectProxy(t *testing.T) { + origin := newTLSEchoServer(t) + proxy := newConnectProxy(t, "") + + rt, err := New(Options{ + Profile: ProfileChrome, + ProxyURL: "http://" + proxy.addr(), + InsecureSkipVerify: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip through CONNECT proxy: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if string(body) != "tunnelled" { + t.Fatalf("body = %q, want %q", body, "tunnelled") + } + // The TLS handshake must happen inside the tunnel, so the origin still sees + // a fingerprinted HTTP/2 connection rather than something the proxy re-made. + if got := resp.Header.Get("X-Proto"); got != "HTTP/2.0" { + t.Fatalf("origin saw %q, want HTTP/2 negotiated inside the tunnel", got) + } + + targets := proxy.targets() + if len(targets) != 1 { + t.Fatalf("proxy saw %d CONNECT requests, want 1", len(targets)) + } + originHost := strings.TrimPrefix(origin.URL, "https://") + if targets[0] != originHost { + t.Fatalf("CONNECT target = %q, want %q", targets[0], originHost) + } +} + +func TestRoundTripSendsProxyCredentials(t *testing.T) { + origin := newTLSEchoServer(t) + + credentialed := &http.Request{Header: make(http.Header)} + credentialed.SetBasicAuth("alice", "s3cret") + expected := credentialed.Header.Get("Authorization") + + proxy := newConnectProxy(t, expected) + + rt, err := New(Options{ + Profile: ProfileChrome, + ProxyURL: "http://alice:s3cret@" + proxy.addr(), + InsecureSkipVerify: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip through authenticated proxy: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + headers := proxy.authHeaders() + if len(headers) != 1 || headers[0] != expected { + t.Fatalf("Proxy-Authorization = %v, want the configured credentials", headers) + } +} + +func TestRoundTripSurfacesProxyRejection(t *testing.T) { + origin := newTLSEchoServer(t) + proxy := newConnectProxy(t, "Basic expected-but-not-sent") + + rt, err := New(Options{ + Profile: ProfileChrome, + ProxyURL: "http://" + proxy.addr(), + InsecureSkipVerify: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + t.Fatal("a rejected CONNECT must surface as an error, not a silent fallback") + } + if !strings.Contains(err.Error(), "407") { + t.Fatalf("error = %v, want it to carry the proxy status", err) + } +} diff --git a/internal/tlsfingerprint/roundtripper.go b/internal/tlsfingerprint/roundtripper.go new file mode 100644 index 000000000..0ee421b00 --- /dev/null +++ b/internal/tlsfingerprint/roundtripper.go @@ -0,0 +1,431 @@ +package tlsfingerprint + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + utls "github.com/refraction-networking/utls" + "golang.org/x/net/http2" + "golang.org/x/net/proxy" +) + +// Options configures a fingerprinting RoundTripper. +type Options struct { + // Profile names the ClientHello template; empty selects DefaultProfile. + Profile string + // ProxyURL routes connections through an HTTP, HTTPS or SOCKS5 proxy. + // Empty dials directly. + ProxyURL string + // PreferIPv4 restricts dialing to IPv4, matching the rest of the proxy's + // egress behaviour. + PreferIPv4 bool + // DialTimeout bounds TCP establishment and the TLS handshake. + DialTimeout time.Duration + // InsecureSkipVerify disables certificate verification. It exists only for + // operators terminating upstream TLS on an inspection proxy. + InsecureSkipVerify bool + // CACertPath adds a PEM bundle as the trust root, for the same inspection + // proxy setups. Empty uses the system pool. + CACertPath string +} + +const defaultDialTimeout = 30 * time.Second + +// RoundTripper performs HTTPS requests over connections whose ClientHello +// matches a configured client profile. +// +// It manages HTTP/2 connections itself rather than delegating to +// http.Transport: Go only routes a connection to its HTTP/2 stack when the +// connection is a *crypto/tls.Conn, and a utls connection never is. Handing a +// utls connection to http.Transport would therefore speak HTTP/1.1 on a +// connection where the server already agreed to HTTP/2 via ALPN. +type RoundTripper struct { + opts Options + helloID utls.ClientHelloID + // rootCAs is resolved once at construction so a missing or malformed bundle + // is reported at setup instead of on every handshake. + rootCAs *x509.CertPool + + mu sync.Mutex + // h2Conns caches one HTTP/2 connection per host. HTTP/2 multiplexes, so a + // single connection carries concurrent requests. + h2Conns map[string]*http2.ClientConn + // dialing serializes connection setup per host so a burst of requests to a + // cold host opens one connection instead of one per request. + dialing map[string]*sync.Cond +} + +// New builds a RoundTripper for the given options. It returns an error for an +// unknown profile or an unparseable proxy URL so misconfiguration surfaces at +// setup instead of as a per-request failure. +func New(opts Options) (*RoundTripper, error) { + helloID, ok := ResolveProfile(opts.Profile) + if !ok { + return nil, fmt.Errorf("tlsfingerprint: unknown profile %q, want one of %s", + opts.Profile, strings.Join(AvailableProfiles(), ", ")) + } + if proxyURL := strings.TrimSpace(opts.ProxyURL); proxyURL != "" { + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("tlsfingerprint: parse proxy URL: %w", err) + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https", "socks5", "socks5h": + default: + return nil, fmt.Errorf("tlsfingerprint: unsupported proxy scheme %q", parsed.Scheme) + } + } + if opts.DialTimeout <= 0 { + opts.DialTimeout = defaultDialTimeout + } + + var rootCAs *x509.CertPool + if caPath := strings.TrimSpace(opts.CACertPath); caPath != "" { + pem, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("tlsfingerprint: read CA cert %s: %w", caPath, err) + } + rootCAs = x509.NewCertPool() + if !rootCAs.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("tlsfingerprint: parse CA cert %s", caPath) + } + } + + return &RoundTripper{ + opts: opts, + helloID: helloID, + rootCAs: rootCAs, + h2Conns: make(map[string]*http2.ClientConn), + dialing: make(map[string]*sync.Cond), + }, nil +} + +// RoundTrip implements http.RoundTripper. +func (t *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL == nil { + return nil, errors.New("tlsfingerprint: request has no URL") + } + if !strings.EqualFold(req.URL.Scheme, "https") { + // Plaintext requests carry no ClientHello, so there is nothing to + // fingerprint and no reason for this transport to handle them. + return nil, fmt.Errorf("tlsfingerprint: scheme %q is not supported, want https", req.URL.Scheme) + } + + host := req.URL.Hostname() + addr := canonicalAddr(req.URL) + + h2Conn, err := t.h2ConnFor(req.Context(), host, addr) + if err != nil { + return nil, err + } + if h2Conn != nil { + resp, errRT := h2Conn.RoundTrip(req) + if errRT != nil { + t.dropH2Conn(host, h2Conn) + return nil, errRT + } + return resp, nil + } + + // The server declined HTTP/2 for this profile's ALPN list. Fall back to a + // single-use HTTP/1.1 exchange; without Go's connection pool behind it there + // is no keep-alive here, which is acceptable because every provider this + // proxy targets negotiates HTTP/2. + return t.roundTripHTTP1(req, host, addr) +} + +// h2ConnFor returns a usable HTTP/2 connection for host, or nil when the +// handshake settled on HTTP/1.1. +func (t *RoundTripper) h2ConnFor(ctx context.Context, host, addr string) (*http2.ClientConn, error) { + t.mu.Lock() + for { + if conn, ok := t.h2Conns[host]; ok { + if conn.CanTakeNewRequest() { + t.mu.Unlock() + return conn, nil + } + delete(t.h2Conns, host) + } + cond, inFlight := t.dialing[host] + if !inFlight { + break + } + // Another goroutine is dialing this host; wait for it and re-check. + cond.Wait() + } + + cond := sync.NewCond(&t.mu) + t.dialing[host] = cond + t.mu.Unlock() + + conn, err := t.dialH2(ctx, host, addr) + + t.mu.Lock() + delete(t.dialing, host) + cond.Broadcast() + if err == nil && conn != nil { + t.h2Conns[host] = conn + } + t.mu.Unlock() + + return conn, err +} + +// dialH2 establishes a fingerprinted TLS connection and, when ALPN selected +// HTTP/2, wraps it in an HTTP/2 client connection. A nil connection with a nil +// error means the peer chose HTTP/1.1. +func (t *RoundTripper) dialH2(ctx context.Context, host, addr string) (*http2.ClientConn, error) { + tlsConn, err := t.dialTLS(ctx, host, addr) + if err != nil { + return nil, err + } + if tlsConn.ConnectionState().NegotiatedProtocol != http2.NextProtoTLS { + // Closed here because the HTTP/1.1 path re-dials: reusing this + // connection would require threading it back out through the cache, + // which is not worth it for a path the target providers never take. + _ = tlsConn.Close() + return nil, nil + } + h2Transport := &http2.Transport{} + h2Conn, err := h2Transport.NewClientConn(tlsConn) + if err != nil { + _ = tlsConn.Close() + return nil, err + } + return h2Conn, nil +} + +// dialTLS opens a TCP connection through the configured egress path and +// performs the fingerprinted TLS handshake on top of it. +func (t *RoundTripper) dialTLS(ctx context.Context, host, addr string) (*utls.UConn, error) { + rawConn, err := t.dialTCP(ctx, addr) + if err != nil { + return nil, err + } + + if deadline, ok := ctx.Deadline(); ok { + _ = rawConn.SetDeadline(deadline) + } else { + _ = rawConn.SetDeadline(time.Now().Add(t.opts.DialTimeout)) + } + + tlsConn := utls.UClient(rawConn, &utls.Config{ + ServerName: host, + InsecureSkipVerify: t.opts.InsecureSkipVerify, + RootCAs: t.rootCAs, + }, t.helloID) + if err = tlsConn.HandshakeContext(ctx); err != nil { + _ = rawConn.Close() + return nil, err + } + // Clear the handshake deadline: streaming responses must not inherit it. + _ = rawConn.SetDeadline(time.Time{}) + return tlsConn, nil +} + +// dialTCP establishes the underlying TCP connection, tunnelling through a proxy +// when one is configured. +func (t *RoundTripper) dialTCP(ctx context.Context, addr string) (net.Conn, error) { + proxyURL := strings.TrimSpace(t.opts.ProxyURL) + if proxyURL == "" { + return t.baseDialer().DialContext(ctx, t.network(), addr) + } + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("tlsfingerprint: parse proxy URL: %w", err) + } + switch strings.ToLower(parsed.Scheme) { + case "socks5", "socks5h": + return t.dialViaSOCKS5(ctx, parsed, addr) + case "http", "https": + return t.dialViaHTTPProxy(ctx, parsed, addr) + default: + return nil, fmt.Errorf("tlsfingerprint: unsupported proxy scheme %q", parsed.Scheme) + } +} + +func (t *RoundTripper) baseDialer() *net.Dialer { + return &net.Dialer{Timeout: t.opts.DialTimeout, KeepAlive: 30 * time.Second} +} + +func (t *RoundTripper) network() string { + if t.opts.PreferIPv4 { + return "tcp4" + } + return "tcp" +} + +func (t *RoundTripper) dialViaSOCKS5(ctx context.Context, proxyURL *url.URL, addr string) (net.Conn, error) { + var auth *proxy.Auth + if proxyURL.User != nil { + password, _ := proxyURL.User.Password() + auth = &proxy.Auth{User: proxyURL.User.Username(), Password: password} + } + dialer, err := proxy.SOCKS5(t.network(), proxyURL.Host, auth, t.baseDialer()) + if err != nil { + return nil, err + } + if contextDialer, ok := dialer.(proxy.ContextDialer); ok { + return contextDialer.DialContext(ctx, t.network(), addr) + } + return dialer.Dial(t.network(), addr) +} + +// dialViaHTTPProxy opens a CONNECT tunnel. http.Transport would normally handle +// this, but it only exposes the tunnelled connection to its own TLS stack, so +// the tunnel has to be established here for the utls handshake to run on top. +func (t *RoundTripper) dialViaHTTPProxy(ctx context.Context, proxyURL *url.URL, addr string) (net.Conn, error) { + proxyAddr := proxyURL.Host + if proxyURL.Port() == "" { + if strings.EqualFold(proxyURL.Scheme, "https") { + proxyAddr = net.JoinHostPort(proxyURL.Hostname(), "443") + } else { + proxyAddr = net.JoinHostPort(proxyURL.Hostname(), "80") + } + } + + conn, err := t.baseDialer().DialContext(ctx, t.network(), proxyAddr) + if err != nil { + return nil, err + } + + // An HTTPS proxy speaks TLS on the hop to the proxy itself. That handshake + // uses the standard library on purpose: the fingerprint that matters is the + // one the origin server sees, inside the tunnel. + if strings.EqualFold(proxyURL.Scheme, "https") { + proxyTLS := tls.Client(conn, &tls.Config{ServerName: proxyURL.Hostname()}) + if err = proxyTLS.HandshakeContext(ctx); err != nil { + _ = conn.Close() + return nil, err + } + conn = proxyTLS + } + + connectReq := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: make(http.Header), + } + if proxyURL.User != nil { + password, _ := proxyURL.User.Password() + connectReq.Header.Set("Proxy-Authorization", basicProxyAuth(proxyURL.User.Username(), password)) + } + + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } else { + _ = conn.SetDeadline(time.Now().Add(t.opts.DialTimeout)) + } + if err = connectReq.Write(conn); err != nil { + _ = conn.Close() + return nil, err + } + + // Buffered reads must stop at the end of the CONNECT response: anything the + // reader buffers past it belongs to the TLS handshake that follows. + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, connectReq) + if err != nil { + _ = conn.Close() + return nil, err + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _ = conn.Close() + return nil, fmt.Errorf("tlsfingerprint: proxy CONNECT to %s failed: %s", addr, resp.Status) + } + if reader.Buffered() > 0 { + _ = conn.Close() + return nil, errors.New("tlsfingerprint: proxy sent data before the tunnel was established") + } + _ = conn.SetDeadline(time.Time{}) + return conn, nil +} + +// roundTripHTTP1 serves one request over a dedicated fingerprinted connection. +func (t *RoundTripper) roundTripHTTP1(req *http.Request, host, addr string) (*http.Response, error) { + tlsConn, err := t.dialTLS(req.Context(), host, addr) + if err != nil { + return nil, err + } + if err = req.Write(tlsConn); err != nil { + _ = tlsConn.Close() + return nil, err + } + resp, err := http.ReadResponse(bufio.NewReader(tlsConn), req) + if err != nil { + _ = tlsConn.Close() + return nil, err + } + // The caller closes the body; closing the connection with it prevents the + // socket leaking, since this path keeps no pool. + resp.Body = &connClosingBody{ReadCloser: resp.Body, conn: tlsConn} + return resp, nil +} + +// connClosingBody ties a response body to the connection that produced it, so +// closing the body releases the socket on this pool-less path. +type connClosingBody struct { + io.ReadCloser + conn net.Conn +} + +func (b *connClosingBody) Close() error { + err := b.ReadCloser.Close() + if errConn := b.conn.Close(); err == nil { + err = errConn + } + return err +} + +func (t *RoundTripper) dropH2Conn(host string, conn *http2.ClientConn) { + t.mu.Lock() + if cached, ok := t.h2Conns[host]; ok && cached == conn { + delete(t.h2Conns, host) + } + t.mu.Unlock() +} + +// CloseIdleConnections implements the optional http.Transport behaviour used by +// http.Client.CloseIdleConnections. +func (t *RoundTripper) CloseIdleConnections() { + t.mu.Lock() + conns := make([]*http2.ClientConn, 0, len(t.h2Conns)) + for host, conn := range t.h2Conns { + conns = append(conns, conn) + delete(t.h2Conns, host) + } + t.mu.Unlock() + for _, conn := range conns { + _ = conn.Close() + } +} + +// canonicalAddr returns host:port, defaulting to the HTTPS port. +func canonicalAddr(u *url.URL) string { + host := u.Hostname() + port := u.Port() + if port == "" { + port = "443" + } + return net.JoinHostPort(host, port) +} + +func basicProxyAuth(username, password string) string { + req := &http.Request{Header: make(http.Header)} + req.SetBasicAuth(username, password) + return req.Header.Get("Authorization") +} diff --git a/internal/tlsfingerprint/roundtripper_test.go b/internal/tlsfingerprint/roundtripper_test.go new file mode 100644 index 000000000..42a6c6284 --- /dev/null +++ b/internal/tlsfingerprint/roundtripper_test.go @@ -0,0 +1,273 @@ +package tlsfingerprint + +import ( + "crypto/tls" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveProfile(t *testing.T) { + for _, name := range AvailableProfiles() { + if _, ok := ResolveProfile(name); !ok { + t.Fatalf("advertised profile %q does not resolve", name) + } + } + if _, ok := ResolveProfile(""); !ok { + t.Fatal("empty profile must resolve to the default") + } + if _, ok := ResolveProfile(" ChRoMe "); !ok { + t.Fatal("profile lookup must ignore case and surrounding space") + } + if _, ok := ResolveProfile("netscape"); ok { + t.Fatal("unknown profile must not resolve") + } + if !IsValidProfile("") || !IsValidProfile("firefox") || IsValidProfile("netscape") { + t.Fatal("IsValidProfile disagrees with ResolveProfile") + } +} + +func TestNewRejectsBadConfiguration(t *testing.T) { + if _, err := New(Options{Profile: "netscape"}); err == nil { + t.Fatal("an unknown profile must fail at construction, not per request") + } + if _, err := New(Options{ProxyURL: "ftp://proxy.invalid:21"}); err == nil { + t.Fatal("an unsupported proxy scheme must fail at construction") + } + if _, err := New(Options{ProxyURL: "://not a url"}); err == nil { + t.Fatal("an unparseable proxy URL must fail at construction") + } + if _, err := New(Options{Profile: "chrome", ProxyURL: "socks5://127.0.0.1:1080"}); err != nil { + t.Fatalf("valid options rejected: %v", err) + } +} + +// TestRoundTripHonoursCustomCA guards against the fingerprint transport quietly +// ignoring trust settings the standard transport applies: it does its own TLS, +// so a configured CA bundle has to be threaded through explicitly. +func TestRoundTripHonoursCustomCA(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + + caPath := filepath.Join(t.TempDir(), "ca.pem") + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + if err := os.WriteFile(caPath, pemBytes, 0o600); err != nil { + t.Fatalf("write CA bundle: %v", err) + } + + // Without the bundle the self-signed certificate must be rejected. + strict, err := New(Options{Profile: ProfileChrome}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer strict.CloseIdleConnections() + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := strict.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + t.Fatal("an untrusted certificate must fail verification") + } + + trusting, err := New(Options{Profile: ProfileChrome, CACertPath: caPath}) + if err != nil { + t.Fatalf("New with CA bundle: %v", err) + } + defer trusting.CloseIdleConnections() + req, err = http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err = trusting.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip with the configured CA: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() +} + +func TestNewRejectsUnusableCABundle(t *testing.T) { + if _, err := New(Options{Profile: ProfileChrome, CACertPath: filepath.Join(t.TempDir(), "missing.pem")}); err == nil { + t.Fatal("a missing CA bundle must fail at construction") + } + + junk := filepath.Join(t.TempDir(), "junk.pem") + if err := os.WriteFile(junk, []byte("not a certificate"), 0o600); err != nil { + t.Fatalf("write junk bundle: %v", err) + } + if _, err := New(Options{Profile: ProfileChrome, CACertPath: junk}); err == nil { + t.Fatal("an unparseable CA bundle must fail at construction") + } +} + +func TestRoundTripRejectsPlaintext(t *testing.T) { + rt, err := New(Options{Profile: ProfileChrome}) + if err != nil { + t.Fatalf("New: %v", err) + } + req := httptest.NewRequest(http.MethodGet, "http://example.invalid/", nil) + resp, err := rt.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + t.Fatal("plaintext requests carry no ClientHello and must be refused") + } +} + +// TestRoundTripAgainstTLSServer drives a real handshake so the ClientHello, +// ALPN negotiation and HTTP/2 framing are exercised end to end rather than +// mocked. +func TestRoundTripAgainstTLSServer(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Proto", r.Proto) + _, _ = io.WriteString(w, "ok") + })) + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + + rt, err := New(Options{Profile: ProfileChrome, InsecureSkipVerify: true}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if string(body) != "ok" { + t.Fatalf("body = %q, want %q", body, "ok") + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + if got := resp.Header.Get("X-Proto"); got != "HTTP/2.0" { + t.Fatalf("server saw %q, want the connection to negotiate HTTP/2", got) + } +} + +// TestRoundTripReusesHTTP2Connection verifies the per-host cache: a second +// request must multiplex over the connection the first one opened rather than +// repeating the handshake. +func TestRoundTripReusesHTTP2Connection(t *testing.T) { + var handshakes int + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + server.EnableHTTP2 = true + server.TLS = &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + handshakes++ + return nil, nil + }, + } + server.StartTLS() + defer server.Close() + + rt, err := New(Options{Profile: ProfileFirefox, InsecureSkipVerify: true}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + for i := 0; i < 3; i++ { + req, errReq := http.NewRequest(http.MethodGet, server.URL, nil) + if errReq != nil { + t.Fatalf("NewRequest: %v", errReq) + } + resp, errRT := rt.RoundTrip(req) + if errRT != nil { + t.Fatalf("RoundTrip %d: %v", i, errRT) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + + if handshakes != 1 { + t.Fatalf("handshakes = %d, want the connection reused across requests", handshakes) + } +} + +// TestRoundTripSendsProfileSpecificClientHello confirms the transport actually +// changes the wire bytes: two profiles must not produce the same ClientHello, +// and neither should look like Go's default. +func TestRoundTripSendsProfileSpecificClientHello(t *testing.T) { + capture := func(profile string) *tls.ClientHelloInfo { + var captured *tls.ClientHelloInfo + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + server.TLS = &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + copied := *hello + captured = &copied + return nil, nil + }, + } + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + + rt, err := New(Options{Profile: profile, InsecureSkipVerify: true}) + if err != nil { + t.Fatalf("New(%s): %v", profile, err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip(%s): %v", profile, err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if captured == nil { + t.Fatalf("no ClientHello captured for %s", profile) + } + return captured + } + + chrome := capture(ProfileChrome) + firefox := capture(ProfileFirefox) + + if joinUint16(chrome.CipherSuites) == joinUint16(firefox.CipherSuites) { + t.Fatal("chrome and firefox produced identical cipher suite lists") + } + for _, hello := range []*tls.ClientHelloInfo{chrome, firefox} { + if len(hello.SupportedProtos) == 0 || hello.SupportedProtos[0] != "h2" { + t.Fatalf("ALPN = %v, want h2 offered first", hello.SupportedProtos) + } + } +} + +func joinUint16(values []uint16) string { + parts := make([]string, 0, len(values)) + for _, v := range values { + parts = append(parts, string(rune(v))) + } + return strings.Join(parts, ",") +} From b6b137fed2afafad4540614606f54e5d9f0fed9d Mon Sep 17 00:00:00 2001 From: kittors Date: Fri, 14 Aug 2026 12:02:38 +0800 Subject: [PATCH 2/2] chore: raise Go toolchain to 1.26.6 to clear stdlib advisories govulncheck reports six standard library vulnerabilities against go1.26.5 (encoding/xml and encoding/asn1 recursion depth, net/http idna punycode handling among them), all fixed in 1.26.6. They predate this branch; the vulncheck required check now blocks any PR until the toolchain moves. go.mod, the PR workflow and the deploy workflow are bumped together, as the repository requires these to stay in sync. Co-Authored-By: Claude Opus 5 --- .github/workflows/deploy.yml | 2 +- .github/workflows/pr-test-build.yml | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 83169f723..cbaf77b14 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -57,7 +57,7 @@ jobs: - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version: '1.26.5' + go-version: '1.26.6' cache: true cache-dependency-path: go.sum diff --git a/.github/workflows/pr-test-build.yml b/.github/workflows/pr-test-build.yml index dd301bf0e..82a9935b2 100644 --- a/.github/workflows/pr-test-build.yml +++ b/.github/workflows/pr-test-build.yml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true env: - GO_VERSION: '1.26.5' + GO_VERSION: '1.26.6' GOLANGCI_LINT_VERSION: v1.64.5 jobs: diff --git a/go.mod b/go.mod index 85fec1008..9ed13370e 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/router-for-me/CLIProxyAPI/v6 go 1.26.0 -toolchain go1.26.5 +toolchain go1.26.6 require ( clirelay.local/updater v0.0.0-00010101000000-000000000000