diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 83169f723..cbaf77b14 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -57,7 +57,7 @@ jobs: - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version: '1.26.5' + go-version: '1.26.6' cache: true cache-dependency-path: go.sum diff --git a/.github/workflows/pr-test-build.yml b/.github/workflows/pr-test-build.yml index dd301bf0e..82a9935b2 100644 --- a/.github/workflows/pr-test-build.yml +++ b/.github/workflows/pr-test-build.yml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true env: - GO_VERSION: '1.26.5' + GO_VERSION: '1.26.6' GOLANGCI_LINT_VERSION: v1.64.5 jobs: diff --git a/go.mod b/go.mod index 85fec1008..9ed13370e 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/router-for-me/CLIProxyAPI/v6 go 1.26.0 -toolchain go1.26.5 +toolchain go1.26.6 require ( clirelay.local/updater v0.0.0-00010101000000-000000000000 diff --git a/internal/config/codex_convergence_config_test.go b/internal/config/codex_convergence_config_test.go new file mode 100644 index 000000000..b45fda7a7 --- /dev/null +++ b/internal/config/codex_convergence_config_test.go @@ -0,0 +1,103 @@ +package config + +import ( + "encoding/json" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestNormalizeCodexIdentityFingerprintConvergenceDefaults(t *testing.T) { + got := NormalizeCodexIdentityFingerprint(CodexIdentityFingerprintConfig{}) + if got.ConvergenceMode != CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want the session default", got.ConvergenceMode) + } + + got = NormalizeCodexIdentityFingerprint(CodexIdentityFingerprintConfig{ConvergenceMode: "nonsense"}) + if got.ConvergenceMode != CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want an invalid value to fall back", got.ConvergenceMode) + } + + got = NormalizeCodexIdentityFingerprint(CodexIdentityFingerprintConfig{ConvergenceMode: " OFF "}) + if got.ConvergenceMode != CodexFingerprintConvergenceOff { + t.Fatalf("convergence mode = %q, want off to survive trimming and case", got.ConvergenceMode) + } +} + +func TestCodexIdentityFingerprintConvergenceRoundTrip(t *testing.T) { + raw := []byte(` +enabled: true +convergence-mode: full +installation-id: 11111111-2222-3333-4444-555555555555 +tls-fingerprint: + enabled: true + profile: firefox +`) + var fromYAML CodexIdentityFingerprintConfig + if err := yaml.Unmarshal(raw, &fromYAML); err != nil { + t.Fatalf("yaml unmarshal: %v", err) + } + if fromYAML.ConvergenceMode != CodexFingerprintConvergenceFull { + t.Fatalf("convergence mode = %q, want full", fromYAML.ConvergenceMode) + } + if fromYAML.InstallationID != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("installation id = %q, want the configured value", fromYAML.InstallationID) + } + if !fromYAML.TLSFingerprint.Enabled || fromYAML.TLSFingerprint.Profile != "firefox" { + t.Fatalf("tls fingerprint = %+v, want enabled firefox", fromYAML.TLSFingerprint) + } + + encoded, err := json.Marshal(NormalizeCodexIdentityFingerprint(fromYAML)) + if err != nil { + t.Fatalf("json marshal: %v", err) + } + var fromJSON CodexIdentityFingerprintConfig + if err = json.Unmarshal(encoded, &fromJSON); err != nil { + t.Fatalf("json unmarshal: %v", err) + } + if fromJSON.ConvergenceMode != CodexFingerprintConvergenceFull { + t.Fatalf("convergence mode did not survive the JSON round trip: %q", fromJSON.ConvergenceMode) + } + if !fromJSON.TLSFingerprint.Enabled || fromJSON.TLSFingerprint.Profile != "firefox" { + t.Fatalf("tls fingerprint did not survive the JSON round trip: %+v", fromJSON.TLSFingerprint) + } +} + +func TestCleanTLSFingerprintDropsUnknownProfile(t *testing.T) { + got := CleanTLSFingerprint(TLSFingerprintConfig{Enabled: true, Profile: "netscape"}) + if got.Profile != "" { + t.Fatalf("profile = %q, want an unknown client name dropped", got.Profile) + } + if !got.Enabled { + t.Fatal("dropping an unknown profile must not silently disable the feature") + } + + got = CleanTLSFingerprint(TLSFingerprintConfig{Enabled: true, Profile: " ChRoMe "}) + if got.Profile != "chrome" { + t.Fatalf("profile = %q, want it normalized to chrome", got.Profile) + } +} + +// Legacy runtime payloads predate these fields; they must still be recognised as +// the old "disabled default" so upgrading does not resurrect a stale config. +func TestCodexLegacyDefaultDetectionWithNewFields(t *testing.T) { + legacy := CodexIdentityFingerprintConfig{} + if !codexLegacyDefaultDisabled(legacy) { + t.Fatal("an empty legacy payload must still be treated as the disabled default") + } + + configured := CodexIdentityFingerprintConfig{ConvergenceMode: CodexFingerprintConvergenceOff} + if codexLegacyDefaultDisabled(configured) { + t.Fatal("an explicitly configured convergence mode must not be discarded as legacy") + } + + pinned := CodexIdentityFingerprintConfig{InstallationID: "abc"} + if codexLegacyDefaultDisabled(pinned) { + t.Fatal("a pinned installation id must not be discarded as legacy") + } + + tlsOn := CodexIdentityFingerprintConfig{TLSFingerprint: TLSFingerprintConfig{Enabled: true}} + if codexLegacyDefaultDisabled(tlsOn) { + t.Fatal("an enabled TLS fingerprint must not be discarded as legacy") + } +} diff --git a/internal/config/identity_fingerprint.go b/internal/config/identity_fingerprint.go index dfdf2f854..c0fa90e24 100644 --- a/internal/config/identity_fingerprint.go +++ b/internal/config/identity_fingerprint.go @@ -4,6 +4,7 @@ import ( "encoding/json" "strings" + "github.com/router-for-me/CLIProxyAPI/v6/internal/tlsfingerprint" "gopkg.in/yaml.v3" ) @@ -17,6 +18,21 @@ const ( DefaultCodexFingerprintBetaFeatures = "" DefaultCodexFingerprintSessionMode = "per-request" + // Codex device fingerprint convergence modes. Upstream counts distinct + // installation/session/thread identifiers to derive per-account device and + // session quotas, so several people sharing one OAuth account each burn a + // separate device slot. Convergence rewrites those identifiers to + // account-stable values before the request leaves the proxy. + CodexFingerprintConvergenceOff = "off" + CodexFingerprintConvergenceDevice = "device" + CodexFingerprintConvergenceSession = "session" + CodexFingerprintConvergenceFull = "full" + + // DefaultCodexFingerprintConvergenceMode converges installation and session + // while keeping one thread per real client session, which is the shape a + // single user spawning sub-agents produces upstream. + DefaultCodexFingerprintConvergenceMode = CodexFingerprintConvergenceSession + DefaultClaudeFingerprintCLIVersion = "2.1.161" DefaultClaudeFingerprintEntrypoint = "cli" DefaultClaudeFingerprintAnthropicBeta = "claude-code-20250219,oauth-2025-04-20,interleaved-thinking-2025-05-14,prompt-caching-scope-2026-01-05,effort-2025-11-24,context-management-2025-06-27,extended-cache-ttl-2025-04-11" @@ -53,7 +69,40 @@ type CodexIdentityFingerprintConfig struct { SessionMode string `yaml:"session-mode,omitempty" json:"session-mode,omitempty"` SessionID string `yaml:"session-id,omitempty" json:"session-id,omitempty"` CustomHeaders map[string]string `yaml:"custom-headers,omitempty" json:"custom-headers,omitempty"` - enabledSet bool + // ConvergenceMode selects how aggressively per-client device identifiers are + // rewritten to account-stable values: off, device, session or full. + ConvergenceMode string `yaml:"convergence-mode,omitempty" json:"convergence-mode,omitempty"` + // InstallationID pins the converged x-codex-installation-id. Leave empty to + // derive a stable value from the account key; set it to replay an + // installation id captured from a real Codex client. + InstallationID string `yaml:"installation-id,omitempty" json:"installation-id,omitempty"` + // TLSFingerprint shapes the ClientHello sent to the Codex upstream. + TLSFingerprint TLSFingerprintConfig `yaml:"tls-fingerprint,omitempty" json:"tls-fingerprint,omitempty"` + enabledSet bool +} + +// TLSFingerprintConfig selects the TLS ClientHello presented to an upstream. +// +// This is off by default: a ClientHello that does not match the client the +// User-Agent claims to be is its own inconsistency, and the correct profile +// depends on what the operator has verified against the upstream. Turn it on +// after confirming the profile with a fingerprint echo service. +type TLSFingerprintConfig struct { + Enabled bool `yaml:"enabled" json:"enabled"` + // Profile names a client to imitate: chrome, firefox, safari, edge, ios, + // android or randomized. Empty selects the built-in default. + Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` +} + +// CleanTLSFingerprint normalizes a TLS fingerprint block, dropping a profile +// name that does not identify a known client. +func CleanTLSFingerprint(in TLSFingerprintConfig) TLSFingerprintConfig { + out := in + out.Profile = strings.ToLower(strings.TrimSpace(out.Profile)) + if out.Profile != "" && !tlsfingerprint.IsValidProfile(out.Profile) { + out.Profile = "" + } + return out } // DefaultCodexIdentityFingerprint returns the recommended Codex identity template. @@ -67,6 +116,20 @@ func DefaultCodexIdentityFingerprint() CodexIdentityFingerprintConfig { BetaFeatures: DefaultCodexFingerprintBetaFeatures, SessionMode: DefaultCodexFingerprintSessionMode, CustomHeaders: map[string]string{}, + + ConvergenceMode: DefaultCodexFingerprintConvergenceMode, + } +} + +// IsValidCodexFingerprintConvergenceMode reports whether mode is one of the +// four supported convergence strengths. +func IsValidCodexFingerprintConvergenceMode(mode string) bool { + switch mode { + case CodexFingerprintConvergenceOff, CodexFingerprintConvergenceDevice, + CodexFingerprintConvergenceSession, CodexFingerprintConvergenceFull: + return true + default: + return false } } @@ -235,6 +298,9 @@ func NormalizeCodexIdentityFingerprint(in CodexIdentityFingerprintConfig) CodexI if out.SessionMode != "server-stable" && out.SessionMode != "fixed" && out.SessionMode != "per-request" { out.SessionMode = DefaultCodexFingerprintSessionMode } + if !IsValidCodexFingerprintConvergenceMode(out.ConvergenceMode) { + out.ConvergenceMode = DefaultCodexFingerprintConvergenceMode + } return out } @@ -253,6 +319,12 @@ func CleanCodexIdentityFingerprint(in CodexIdentityFingerprintConfig) CodexIdent if out.SessionMode != "" && out.SessionMode != "server-stable" && out.SessionMode != "fixed" && out.SessionMode != "per-request" { out.SessionMode = DefaultCodexFingerprintSessionMode } + out.ConvergenceMode = strings.TrimSpace(strings.ToLower(out.ConvergenceMode)) + if out.ConvergenceMode != "" && !IsValidCodexFingerprintConvergenceMode(out.ConvergenceMode) { + out.ConvergenceMode = DefaultCodexFingerprintConvergenceMode + } + out.InstallationID = strings.TrimSpace(out.InstallationID) + out.TLSFingerprint = CleanTLSFingerprint(out.TLSFingerprint) out.CustomHeaders = cleanIdentityFingerprintHeaders(out.CustomHeaders) return out } @@ -419,7 +491,9 @@ func defaultXAIIdentityFingerprintEnabled(in XAIIdentityFingerprintConfig) XAIId } func codexLegacyDefaultDisabled(fp CodexIdentityFingerprintConfig) bool { - if fp.Enabled || strings.TrimSpace(fp.SessionID) != "" || len(fp.CustomHeaders) > 0 { + if fp.Enabled || strings.TrimSpace(fp.SessionID) != "" || + strings.TrimSpace(fp.InstallationID) != "" || fp.TLSFingerprint.Enabled || + strings.TrimSpace(fp.TLSFingerprint.Profile) != "" || len(fp.CustomHeaders) > 0 { return false } defaults := DefaultCodexIdentityFingerprint() @@ -428,7 +502,8 @@ func codexLegacyDefaultDisabled(fp CodexIdentityFingerprintConfig) bool { emptyOrEqual(fp.Originator, defaults.Originator) && emptyOrEqual(fp.WebsocketBeta, defaults.WebsocketBeta) && emptyOrEqual(fp.BetaFeatures, defaults.BetaFeatures) && - emptyOrEqual(fp.SessionMode, defaults.SessionMode) + emptyOrEqual(fp.SessionMode, defaults.SessionMode) && + emptyOrEqual(fp.ConvergenceMode, defaults.ConvergenceMode) } func claudeLegacyDefaultDisabled(fp ClaudeIdentityFingerprintConfig) bool { diff --git a/internal/management/settings/runtimeconfig/spec.go b/internal/management/settings/runtimeconfig/spec.go index a51e6753c..def447ccb 100644 --- a/internal/management/settings/runtimeconfig/spec.go +++ b/internal/management/settings/runtimeconfig/spec.go @@ -400,7 +400,11 @@ func codexIdentityFingerprintMeaningful(fp config.CodexIdentityFingerprintConfig strings.TrimSpace(clean.Version) != "" || strings.TrimSpace(clean.Originator) != "" || strings.TrimSpace(clean.WebsocketBeta) != "" || - strings.TrimSpace(clean.SessionMode) != "" + strings.TrimSpace(clean.SessionMode) != "" || + strings.TrimSpace(clean.ConvergenceMode) != "" || + strings.TrimSpace(clean.InstallationID) != "" || + clean.TLSFingerprint.Enabled || + strings.TrimSpace(clean.TLSFingerprint.Profile) != "" } func claudeIdentityFingerprintMeaningful(fp config.ClaudeIdentityFingerprintConfig) bool { diff --git a/internal/runtime/executor/codex_auth.go b/internal/runtime/executor/codex_auth.go index 1fe12a812..271c43274 100644 --- a/internal/runtime/executor/codex_auth.go +++ b/internal/runtime/executor/codex_auth.go @@ -45,6 +45,17 @@ func applyCodexHeaders(r *http.Request, cfg *config.Config, auth *cliproxyauth.A misc.EnsureHeader(r.Header, ginHeaders, "User-Agent", codexUserAgent) } + // Device fingerprint convergence runs after the client headers above have + // been copied in, so it rewrites the values that actually reach upstream. + // The executor resolves the id set before translating the body and stores it + // on the context; requests that never pass through that path (token probes, + // PrepareRequest) resolve their own set here. + convergedIDs := codexConvergedIDsFromContext(r.Context()) + if convergedIDs == nil { + convergedIDs = resolveCodexConvergedIDs(cfg, auth, ginHeaders) + } + applyCodexConvergenceHeaders(r.Header, convergedIDs, ginHeaders) + // Upstream codex-tui behavior: only attach Session_id when the UA indicates a desktop client. if strings.Contains(r.Header.Get("User-Agent"), "Mac OS") && strings.TrimSpace(r.Header.Get("Session_id")) == "" { r.Header.Set("Session_id", uuid.NewString()) diff --git a/internal/runtime/executor/codex_executor.go b/internal/runtime/executor/codex_executor.go index ea259db7c..241698fac 100644 --- a/internal/runtime/executor/codex_executor.go +++ b/internal/runtime/executor/codex_executor.go @@ -120,6 +120,7 @@ func (e *CodexExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth, re return e.executeCompact(ctx, auth, req, opts) } req, opts = maybeStripCodexHistoryDataURLImagesOnRequest(req, opts) + ctx, convergedIDs := e.prepareCodexConvergence(ctx, auth) execCtx := newExecutionContext(ctx, e.Identifier(), e.cfg, auth, req, opts, ExecutionOptions{ TargetFormat: sdktranslator.FromString("codex"), }) @@ -158,6 +159,7 @@ func (e *CodexExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth, re "safety_identifier", }) body = maybeEnsureCodexImageGenerationTool(body, auth, execCtx.BaseModel, codexAdmissionHeadersFromContext(execCtx.Context)) + body = applyCodexConvergenceClientMetadata(body, convergedIDs) url := strings.TrimSuffix(baseURL, "/") + "/responses" httpReq, err := e.cacheHelper(execCtx.Context, auth, execCtx.SourceFormat, url, req, body) @@ -250,6 +252,7 @@ func (e *CodexExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth, re } func (e *CodexExecutor) executeCompact(ctx context.Context, auth *cliproxyauth.Auth, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (resp cliproxyexecutor.Response, err error) { + ctx, convergedIDs := e.prepareCodexConvergence(ctx, auth) execCtx := newExecutionContext(ctx, e.Identifier(), e.cfg, auth, req, opts, ExecutionOptions{ TargetFormat: sdktranslator.FromString("openai-response"), }) @@ -277,6 +280,7 @@ func (e *CodexExecutor) executeCompact(ctx context.Context, auth *cliproxyauth.A body = ensureTranslatedCodexModel(body, execCtx.BaseModel) body = sanitizeCodexResponsesRequest(body) body, _ = sjson.DeleteBytes(body, "stream") + body = applyCodexConvergenceClientMetadata(body, convergedIDs) url := strings.TrimSuffix(baseURL, "/") + "/responses/compact" httpReq, err := e.cacheHelper(execCtx.Context, auth, execCtx.SourceFormat, url, req, body) @@ -329,6 +333,7 @@ func (e *CodexExecutor) ExecuteStream(ctx context.Context, auth *cliproxyauth.Au // Shrink multi-MB Desktop history data URLs before translation/sanitize so later // body-level passes never see the full base64 history. req, opts = maybeStripCodexHistoryDataURLImagesOnRequest(req, opts) + ctx, convergedIDs := e.prepareCodexConvergence(ctx, auth) execCtx := newExecutionContext(ctx, e.Identifier(), e.cfg, auth, req, opts, ExecutionOptions{ TargetFormat: sdktranslator.FromString("codex"), TranslateAsStream: true, @@ -366,6 +371,7 @@ func (e *CodexExecutor) ExecuteStream(ctx context.Context, auth *cliproxyauth.Au }) body = ensureTranslatedCodexModel(body, execCtx.BaseModel) body = maybeEnsureCodexImageGenerationTool(body, auth, execCtx.BaseModel, codexAdmissionHeadersFromContext(execCtx.Context)) + body = applyCodexConvergenceClientMetadata(body, convergedIDs) url := strings.TrimSuffix(baseURL, "/") + "/responses" httpReq, err := e.cacheHelper(execCtx.Context, auth, execCtx.SourceFormat, url, req, body) diff --git a/internal/runtime/executor/codex_fingerprint_convergence.go b/internal/runtime/executor/codex_fingerprint_convergence.go new file mode 100644 index 000000000..7f3f837c8 --- /dev/null +++ b/internal/runtime/executor/codex_fingerprint_convergence.go @@ -0,0 +1,425 @@ +package executor + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/google/uuid" + "github.com/router-for-me/CLIProxyAPI/v6/internal/config" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/auth" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// Codex device fingerprint convergence. +// +// A Codex client stamps every request with identifiers that upstream reads as +// "which machine" and "which conversation": installation id, session id, thread +// id and window id. When several people share one OAuth account through this +// proxy, each of their clients contributes its own set, so upstream sees a +// crowd of devices and sessions on a single account and applies device/session +// quota limits accordingly. +// +// Convergence rewrites those identifiers to values derived from the account +// itself, so the traffic reads as one installation instead of N. The four modes +// trade fidelity for aggressiveness; see config.CodexFingerprintConvergence*. +// +// Identifiers are derived (not random) so they survive restarts: the same +// account always produces the same installation and session id, which is what +// makes the device look persistent rather than freshly reinstalled per boot. + +// codexConvergedIDs is the identifier set applied to one outbound request. +// +// Header rewriting and body rewriting must share a single instance: turn id is +// generated fresh per request, and a mismatch between the header copy and the +// client_metadata copy is itself a fingerprint. +type codexConvergedIDs struct { + mode string + installationID string + sessionID string + threadID string + turnID string + windowID string + // turnStartedAtUnixMs is stamped from the same clock that produced turnID + // (a time-ordered UUIDv7). Keeping the client's original timestamp next to a + // server-generated turn id would leave the two disagreeing about when the + // turn started, which is a discrepancy a real client never produces. + turnStartedAtUnixMs int64 +} + +type codexConvergedIDsContextKeyType struct{} + +var codexConvergedIDsContextKey codexConvergedIDsContextKeyType + +// withCodexConvergedIDs stores a resolved identifier set on the context so the +// body rewrite in the executor and the header rewrite in applyCodexHeaders act +// on the same values. +func withCodexConvergedIDs(ctx context.Context, ids *codexConvergedIDs) context.Context { + if ctx == nil || ids == nil { + return ctx + } + return context.WithValue(ctx, codexConvergedIDsContextKey, ids) +} + +func codexConvergedIDsFromContext(ctx context.Context) *codexConvergedIDs { + if ctx == nil { + return nil + } + ids, _ := ctx.Value(codexConvergedIDsContextKey).(*codexConvergedIDs) + return ids +} + +// codexConvergenceMode reports the effective convergence strength for this auth. +// Convergence rides on the Codex identity fingerprint switch: turning that off +// means "leave my outbound identity alone", which has to include device ids. +func codexConvergenceMode(cfg *config.Config, auth *cliproxyauth.Auth) string { + if cfg == nil || !cfg.IdentityFingerprint.Codex.Enabled { + return config.CodexFingerprintConvergenceOff + } + // API-key credentials are not OAuth accounts and carry no device quota, so + // rewriting their identifiers would only add noise. + if auth != nil && auth.Attributes != nil { + if strings.TrimSpace(auth.Attributes["api_key"]) != "" { + return config.CodexFingerprintConvergenceOff + } + } + mode := strings.TrimSpace(strings.ToLower(cfg.IdentityFingerprint.Codex.ConvergenceMode)) + if mode == "" { + mode = config.DefaultCodexFingerprintConvergenceMode + } + if !config.IsValidCodexFingerprintConvergenceMode(mode) { + mode = config.DefaultCodexFingerprintConvergenceMode + } + return mode +} + +// deriveStableUUIDv4 turns a seed into a fixed UUIDv4-shaped string. The same +// seed always yields the same value, which is what makes a derived installation +// id look like a real one that persists across restarts. +func deriveStableUUIDv4(seed string) string { + h := sha256.Sum256([]byte(seed)) + b := h[:16] + b[6] = (b[6] & 0x0f) | 0x40 // version 4 + b[8] = (b[8] & 0x3f) | 0x80 // variant 1 + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + binary.BigEndian.Uint32(b[0:4]), + binary.BigEndian.Uint16(b[4:6]), + binary.BigEndian.Uint16(b[6:8]), + binary.BigEndian.Uint16(b[8:10]), + b[10:16]) +} + +// codexConvergenceScope returns the stable per-account seed component. It +// reuses the identity fingerprint account key so a credential keeps its device +// identity after moving between tenants, and falls back to the broader session +// isolation scope for auths that carry no resolvable subject. +func codexConvergenceScope(auth *cliproxyauth.Auth) string { + if accountKey, _ := identityFingerprintAccount(auth); strings.TrimSpace(accountKey) != "" { + return strings.TrimSpace(accountKey) + } + return sessionIsolationScope(auth) +} + +// resolveConvergedInstallationID prefers an operator-pinned installation id so a +// value captured from a real Codex client can be replayed; otherwise it derives +// one from the account scope. +func resolveConvergedInstallationID(cfg *config.Config, scope string) string { + if cfg != nil { + if pinned := strings.TrimSpace(cfg.IdentityFingerprint.Codex.InstallationID); pinned != "" { + return pinned + } + } + if scope == "" { + return "" + } + return deriveStableUUIDv4("clirelay:codex-install-id:v1:" + scope) +} + +func resolveConvergedSessionID(scope string) string { + if scope == "" { + return "" + } + return deriveStableUUIDv4("clirelay:codex-session-id:v1:" + scope) +} + +// resolveConvergedThreadID derives one thread per real client session, so a +// shared account still looks like a single user running several agents rather +// than a single endless conversation. +func resolveConvergedThreadID(scope, clientSessionID string) string { + if scope == "" || clientSessionID == "" { + return "" + } + return deriveStableUUIDv4("clirelay:codex-thread-id:v1:" + scope + ":" + clientSessionID) +} + +// extractClientSessionID reads the client's own session identifier before any +// rewrite. Codex CLI sends the hyphenated form; the underscore form is accepted +// as a fallback because some clients and earlier proxy layers emit it. +func extractClientSessionID(h http.Header) string { + if h == nil { + return "" + } + for _, key := range []string{"session-id", "session_id"} { + if v := strings.TrimSpace(h.Get(key)); v != "" { + return v + } + } + return "" +} + +// resolveCodexConvergedIDs computes the identifier set for one request. +// It returns nil when convergence is off or no stable account scope exists, +// in which case callers must leave the client identifiers untouched. +// +// The result carries a freshly generated turn id, so callers must resolve once +// per request and share the result between header and body rewriting. +func resolveCodexConvergedIDs(cfg *config.Config, auth *cliproxyauth.Auth, clientHeaders http.Header) *codexConvergedIDs { + mode := codexConvergenceMode(cfg, auth) + if mode == config.CodexFingerprintConvergenceOff { + return nil + } + scope := codexConvergenceScope(auth) + installationID := resolveConvergedInstallationID(cfg, scope) + if installationID == "" { + return nil + } + + ids := &codexConvergedIDs{mode: mode, installationID: installationID} + if mode == config.CodexFingerprintConvergenceDevice { + return ids + } + + ids.sessionID = resolveConvergedSessionID(scope) + if ids.sessionID == "" { + // Without a stable session id the session/full modes cannot converge + // anything beyond the device; degrade instead of emitting empty headers. + ids.mode = config.CodexFingerprintConvergenceDevice + return ids + } + switch mode { + case config.CodexFingerprintConvergenceSession: + ids.threadID = resolveConvergedThreadID(scope, extractClientSessionID(clientHeaders)) + if ids.threadID == "" { + ids.threadID = ids.sessionID + } + case config.CodexFingerprintConvergenceFull: + ids.threadID = ids.sessionID + } + turnStartedAt := time.Now() + ids.turnID = newCodexTurnID() + ids.turnStartedAtUnixMs = turnStartedAt.UnixMilli() + ids.windowID = ids.threadID + ":0" + return ids +} + +// prepareCodexConvergence resolves the identifier set once per request and +// returns a context carrying it. Header rewriting in applyCodexHeaders reads it +// back from there, so the headers and the request body agree on the per-request +// turn id instead of each generating its own. +func (e *CodexExecutor) prepareCodexConvergence(ctx context.Context, auth *cliproxyauth.Auth) (context.Context, *codexConvergedIDs) { + ids := resolveCodexConvergedIDs(e.cfg, auth, identityFingerprintHeadersFromContext(ctx)) + if ids == nil { + return ctx, nil + } + return withCodexConvergedIDs(ctx, ids), ids +} + +// newCodexTurnID mirrors the time-ordered turn identifier a real client emits. +func newCodexTurnID() string { + if v, err := uuid.NewV7(); err == nil { + return v.String() + } + return uuid.NewString() +} + +// applyCodexConvergenceHeaders rewrites the device identifiers on the outbound +// request. It must run after client headers have been copied in, so the values +// it writes are the ones that survive to the wire. +// +// clientHeaders is the original inbound request. Identifiers this proxy does not +// otherwise forward are only emitted when the client sent them: this layer +// converges what upstream would have seen anyway and must not introduce a header +// upstream was never going to receive, which would add fingerprint surface +// rather than remove it. The turn metadata payload is exempt because +// applyCodexHeaders forwards it verbatim, so its embedded identifiers do reach +// upstream and are the main leak this feature closes. +func applyCodexConvergenceHeaders(h http.Header, ids *codexConvergedIDs, clientHeaders http.Header) { + if h == nil || ids == nil || ids.installationID == "" { + return + } + + setIfClientSent(h, clientHeaders, "X-Codex-Installation-Id", ids.installationID) + + if ids.mode == config.CodexFingerprintConvergenceDevice { + rewriteCodexTurnMetadataHeader(h, ids.turnMetadataFields()) + return + } + + setIfClientSent(h, clientHeaders, "X-Codex-Window-Id", ids.windowID) + setIfClientSent(h, clientHeaders, "Thread-Id", ids.threadID) + // x-client-request-id is forwarded verbatim by applyCodexHeaders, so it + // reaches upstream and must be converged whether or not the client set it. + h.Set("X-Client-Request-Id", ids.threadID) + // Session_id always goes out: without convergence the per-request session + // mode stamps a fresh random id on every call, which makes one account look + // like an endless stream of new sessions. Both spellings are pinned so a + // client that used the other form cannot re-expose its original value. + h.Set("Session-Id", ids.sessionID) + h.Set("Session_id", ids.sessionID) + + rewriteCodexTurnMetadataHeader(h, ids.turnMetadataFields()) +} + +// setIfClientSent writes value only when the inbound request carried that +// header, so convergence never widens what upstream can observe. +func setIfClientSent(h, clientHeaders http.Header, key, value string) { + if value == "" { + return + } + if strings.TrimSpace(h.Get(key)) == "" { + if clientHeaders == nil || strings.TrimSpace(clientHeaders.Get(key)) == "" { + return + } + } + h.Set(key, value) +} + +// deviceOnly narrows an identifier set to device-level convergence. +// +// The websocket path uses this: there Session_id is deliberately pinned to +// Conversation_id and prompt_cache_key so upstream can match the prompt cache, +// and rewriting the session or thread would break that three-way association for +// a saving that only matters on the HTTP path. Converging the installation is +// safe because nothing else is keyed off it. +func (ids *codexConvergedIDs) deviceOnly() *codexConvergedIDs { + if ids == nil { + return nil + } + narrowed := *ids + narrowed.mode = config.CodexFingerprintConvergenceDevice + return &narrowed +} + +// turnMetadataFields returns the turn-metadata entries this mode converges. +// Device mode only claims the installation, so it must not touch the session, +// thread or turn fields the client sent. +func (ids *codexConvergedIDs) turnMetadataFields() map[string]any { + fields := map[string]any{"installation_id": ids.installationID} + if ids.mode == config.CodexFingerprintConvergenceDevice { + return fields + } + fields["session_id"] = ids.sessionID + fields["thread_id"] = ids.threadID + fields["turn_id"] = ids.turnID + fields["window_id"] = ids.windowID + fields["turn_started_at_unix_ms"] = ids.turnStartedAtUnixMs + return fields +} + +// rewriteCodexTurnMetadataHeader replaces the given fields inside the +// x-codex-turn-metadata JSON payload while preserving every other field the +// client sent (sandbox, thread_source and similar), because dropping them would +// itself change the fingerprint. +func rewriteCodexTurnMetadataHeader(h http.Header, fields map[string]any) { + raw := strings.TrimSpace(h.Get("X-Codex-Turn-Metadata")) + rebuilt, ok := rewriteCodexTurnMetadataJSON(raw, fields) + if !ok { + return + } + h.Set("X-Codex-Turn-Metadata", rebuilt) +} + +// rewriteCodexTurnMetadataJSON applies fields to a turn-metadata JSON document. +// It reports false when the payload is absent or not parseable, so callers leave +// the original value untouched rather than replacing it with a partial rewrite. +// +// Only keys the client already sent are replaced. Convergence exists to shrink +// what upstream can distinguish, so adding a field the client omitted would work +// against the goal even when the field is one a real client usually sends. +func rewriteCodexTurnMetadataJSON(raw string, fields map[string]any) (string, bool) { + raw = strings.TrimSpace(raw) + if raw == "" || !gjson.Valid(raw) { + return "", false + } + rebuilt := raw + for _, key := range sortedCodexMetadataKeys(fields) { + if !gjson.Get(rebuilt, key).Exists() { + continue + } + next, err := sjson.Set(rebuilt, key, fields[key]) + if err != nil { + return "", false + } + rebuilt = next + } + return rebuilt, true +} + +// applyCodexConvergenceClientMetadata rewrites the identifiers duplicated in the +// request body. Fields are only replaced when the client already sent +// client_metadata: fabricating the object for clients that never send it would +// add a fingerprint rather than converge one. +func applyCodexConvergenceClientMetadata(body []byte, ids *codexConvergedIDs) []byte { + if len(body) == 0 || ids == nil || ids.installationID == "" { + return body + } + if !gjson.GetBytes(body, "client_metadata").IsObject() { + return body + } + + fields := map[string]any{ + "client_metadata.x-codex-installation-id": ids.installationID, + } + if ids.mode != config.CodexFingerprintConvergenceDevice { + fields["client_metadata.session_id"] = ids.sessionID + fields["client_metadata.thread_id"] = ids.threadID + fields["client_metadata.turn_id"] = ids.turnID + fields["client_metadata.x-codex-window-id"] = ids.windowID + } + + for _, key := range sortedCodexMetadataKeys(fields) { + // Only rewrite identifiers the client actually sent: adding a field it + // never included would widen the fingerprint instead of converging it. + if !gjson.GetBytes(body, key).Exists() { + continue + } + next, err := sjson.SetBytes(body, key, fields[key]) + if err != nil { + return body + } + body = next + } + return rewriteCodexClientMetadataEmbeddedTurnMetadata(body, ids) +} + +// rewriteCodexClientMetadataEmbeddedTurnMetadata patches the turn metadata that +// clients embed inside client_metadata as a JSON string. +func rewriteCodexClientMetadataEmbeddedTurnMetadata(body []byte, ids *codexConvergedIDs) []byte { + raw := gjson.GetBytes(body, "client_metadata.x-codex-turn-metadata").String() + rebuilt, ok := rewriteCodexTurnMetadataJSON(raw, ids.turnMetadataFields()) + if !ok { + return body + } + next, err := sjson.SetBytes(body, "client_metadata.x-codex-turn-metadata", rebuilt) + if err != nil { + return body + } + return next +} + +// sortedCodexMetadataKeys keeps rewrite order deterministic so the resulting +// JSON field order does not vary between otherwise identical requests. +func sortedCodexMetadataKeys(fields map[string]any) []string { + keys := make([]string, 0, len(fields)) + for key := range fields { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/internal/runtime/executor/codex_fingerprint_convergence_test.go b/internal/runtime/executor/codex_fingerprint_convergence_test.go new file mode 100644 index 000000000..7f11a0d15 --- /dev/null +++ b/internal/runtime/executor/codex_fingerprint_convergence_test.go @@ -0,0 +1,367 @@ +package executor + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v6/internal/config" + "github.com/router-for-me/CLIProxyAPI/v6/internal/util" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/auth" + "github.com/tidwall/gjson" +) + +func codexConvergenceTestConfig(mode string) *config.Config { + return &config.Config{ + IdentityFingerprint: config.IdentityFingerprintConfig{ + Codex: config.CodexIdentityFingerprintConfig{ + Enabled: true, + UserAgent: "codex_cli_rs/0.144.1", + Originator: "codex_cli_rs", + SessionMode: "per-request", + ConvergenceMode: mode, + }, + }, + } +} + +func codexConvergenceTestAuth(id string) *cliproxyauth.Auth { + return &cliproxyauth.Auth{ + ID: id, + Provider: "codex", + Metadata: map[string]any{ + "access_token": "codex-token", + "account_id": id + "-account", + }, + } +} + +func TestResolveCodexConvergedIDsIsStableAcrossRequests(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + auth := codexConvergenceTestAuth("codex-stable") + + headers := http.Header{} + headers.Set("session-id", "client-session-a") + + first := resolveCodexConvergedIDs(cfg, auth, headers) + second := resolveCodexConvergedIDs(cfg, auth, headers) + if first == nil || second == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if first.installationID != second.installationID { + t.Fatalf("installation id drifted between requests: %q vs %q", first.installationID, second.installationID) + } + if first.sessionID != second.sessionID { + t.Fatalf("session id drifted between requests: %q vs %q", first.sessionID, second.sessionID) + } + if first.threadID != second.threadID { + t.Fatalf("thread id drifted for the same client session: %q vs %q", first.threadID, second.threadID) + } + // Turn id is per-request by design; a constant turn id would be the anomaly. + if first.turnID == second.turnID { + t.Fatal("turn id was reused across requests, want a fresh value each turn") + } +} + +func TestResolveCodexConvergedIDsSeparatesAccounts(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + headers := http.Header{} + headers.Set("session-id", "shared-client-session") + + a := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-a"), headers) + b := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-b"), headers) + if a == nil || b == nil { + t.Fatal("convergence returned nil for accounts with resolvable scopes") + } + if a.installationID == b.installationID { + t.Fatal("two accounts collapsed onto one installation id") + } + if a.sessionID == b.sessionID { + t.Fatal("two accounts collapsed onto one session id") + } +} + +func TestResolveCodexConvergedIDsDerivesThreadPerClientSession(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + auth := codexConvergenceTestAuth("codex-threads") + + first := http.Header{} + first.Set("session-id", "client-session-1") + second := http.Header{} + second.Set("session-id", "client-session-2") + + a := resolveCodexConvergedIDs(cfg, auth, first) + b := resolveCodexConvergedIDs(cfg, auth, second) + if a == nil || b == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if a.sessionID != b.sessionID { + t.Fatal("session mode must converge both clients onto one session id") + } + if a.threadID == b.threadID { + t.Fatal("distinct client sessions must map to distinct thread ids") + } +} + +func TestResolveCodexConvergedIDsFullCollapsesThread(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceFull) + auth := codexConvergenceTestAuth("codex-full") + + first := http.Header{} + first.Set("session-id", "client-session-1") + second := http.Header{} + second.Set("session-id", "client-session-2") + + a := resolveCodexConvergedIDs(cfg, auth, first) + b := resolveCodexConvergedIDs(cfg, auth, second) + if a == nil || b == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if a.threadID != b.threadID || a.threadID != a.sessionID { + t.Fatalf("full mode must collapse every client onto the session thread: %q vs %q", a.threadID, b.threadID) + } +} + +func TestResolveCodexConvergedIDsOffAndAPIKey(t *testing.T) { + auth := codexConvergenceTestAuth("codex-off") + if ids := resolveCodexConvergedIDs(codexConvergenceTestConfig(config.CodexFingerprintConvergenceOff), auth, nil); ids != nil { + t.Fatal("off mode must not converge anything") + } + + disabled := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + disabled.IdentityFingerprint.Codex.Enabled = false + if ids := resolveCodexConvergedIDs(disabled, auth, nil); ids != nil { + t.Fatal("disabling the Codex identity fingerprint must also disable convergence") + } + + apiKeyAuth := codexConvergenceTestAuth("codex-api-key") + apiKeyAuth.Attributes = map[string]string{"api_key": "sk-test"} + if ids := resolveCodexConvergedIDs(codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession), apiKeyAuth, nil); ids != nil { + t.Fatal("API key credentials carry no device quota and must not be converged") + } +} + +func TestResolveCodexConvergedIDsHonoursPinnedInstallationID(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceDevice) + cfg.IdentityFingerprint.Codex.InstallationID = "11111111-2222-3333-4444-555555555555" + + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-pinned"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + if ids.installationID != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("installation id = %q, want the operator-pinned value", ids.installationID) + } + if ids.sessionID != "" || ids.threadID != "" { + t.Fatal("device mode must leave session and thread identifiers untouched") + } +} + +func TestApplyCodexHeadersConvergesTurnMetadata(t *testing.T) { + gin.SetMode(gin.TestMode) + ginCtx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ginCtx.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + ginCtx.Request.Header.Set("session-id", "client-session") + ginCtx.Request.Header.Set("X-Codex-Turn-Metadata", + `{"installation_id":"client-install","session_id":"client-session","thread_id":"client-thread","turn_id":"client-turn","window_id":"client-window","sandbox":"workspace-write"}`) + ginCtx.Request.Header.Set("X-Client-Request-Id", "client-request") + + req := httptest.NewRequest(http.MethodPost, "https://chatgpt.com/backend-api/codex/responses", nil) + req = req.WithContext(context.WithValue(req.Context(), util.ContextKeyGin, ginCtx)) + + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + auth := codexConvergenceTestAuth("codex-headers") + applyCodexHeaders(req, cfg, auth, "token", true) + + metadata := req.Header.Get("X-Codex-Turn-Metadata") + for _, field := range []string{"installation_id", "session_id", "thread_id", "turn_id", "window_id"} { + got := gjson.Get(metadata, field).String() + if got == "" { + t.Fatalf("turn metadata %s was dropped", field) + } + if got == "client-"+shortFieldName(field) { + t.Fatalf("turn metadata %s still carries the client value %q", field, got) + } + } + // Unrelated fields must survive: dropping them would itself change the fingerprint. + if got := gjson.Get(metadata, "sandbox").String(); got != "workspace-write" { + t.Fatalf("sandbox = %q, want the client value to be preserved", got) + } + if got := req.Header.Get("X-Client-Request-Id"); got == "client-request" || got == "" { + t.Fatalf("X-Client-Request-Id = %q, want a converged value", got) + } + + ids := resolveCodexConvergedIDs(cfg, auth, ginCtx.Request.Header) + if got := req.Header.Get("Session_id"); got != ids.sessionID { + t.Fatalf("Session_id = %q, want converged %q", got, ids.sessionID) + } + if got := req.Header.Get("Session-Id"); got != ids.sessionID { + t.Fatalf("Session-Id = %q, want converged %q", got, ids.sessionID) + } + if got := gjson.Get(metadata, "installation_id").String(); got != ids.installationID { + t.Fatalf("turn metadata installation_id = %q, want converged %q", got, ids.installationID) + } +} + +// shortFieldName maps a turn metadata field to the suffix used by the client +// placeholder values in the test above. +func shortFieldName(field string) string { + switch field { + case "installation_id": + return "install" + case "session_id": + return "session" + case "thread_id": + return "thread" + case "turn_id": + return "turn" + case "window_id": + return "window" + default: + return field + } +} + +func TestApplyCodexHeadersDoesNotIntroduceUnsentIdentifiers(t *testing.T) { + gin.SetMode(gin.TestMode) + ginCtx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ginCtx.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + ginCtx.Request.Header.Set("session-id", "client-session") + + req := httptest.NewRequest(http.MethodPost, "https://chatgpt.com/backend-api/codex/responses", nil) + req = req.WithContext(context.WithValue(req.Context(), util.ContextKeyGin, ginCtx)) + + applyCodexHeaders(req, codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession), + codexConvergenceTestAuth("codex-nosurface"), "token", true) + + // The client sent none of these, so convergence must not hand upstream a + // header it would never have received. + for _, key := range []string{"X-Codex-Installation-Id", "X-Codex-Window-Id", "Thread-Id"} { + if got := req.Header.Get(key); got != "" { + t.Fatalf("%s = %q, want no header when the client sent none", key, got) + } + } +} + +func TestApplyCodexConvergenceClientMetadataRewritesOnlyPresentFields(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-body"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + body := []byte(`{"model":"gpt-5","client_metadata":{"session_id":"client-session","x-codex-installation-id":"client-install","cli_version":"0.144.1"}}`) + got := applyCodexConvergenceClientMetadata(body, ids) + + if v := gjson.GetBytes(got, "client_metadata.session_id").String(); v != ids.sessionID { + t.Fatalf("client_metadata.session_id = %q, want converged %q", v, ids.sessionID) + } + if v := gjson.GetBytes(got, "client_metadata.x-codex-installation-id").String(); v != ids.installationID { + t.Fatalf("client_metadata installation id = %q, want converged %q", v, ids.installationID) + } + if v := gjson.GetBytes(got, "client_metadata.cli_version").String(); v != "0.144.1" { + t.Fatalf("unrelated client_metadata field was altered: %q", v) + } + // thread_id was absent from the client payload and must stay absent. + if gjson.GetBytes(got, "client_metadata.thread_id").Exists() { + t.Fatal("convergence added a client_metadata field the client never sent") + } + if v := gjson.GetBytes(got, "model").String(); v != "gpt-5" { + t.Fatalf("model = %q, want the request to be otherwise untouched", v) + } +} + +func TestApplyCodexConvergenceLeavesOmittedTurnMetadataFieldsAbsent(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-partial"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + headers := http.Header{} + headers.Set("X-Codex-Turn-Metadata", `{"installation_id":"client-install","sandbox":"read-only"}`) + applyCodexConvergenceHeaders(headers, ids, http.Header{}) + + metadata := headers.Get("X-Codex-Turn-Metadata") + if got := gjson.Get(metadata, "installation_id").String(); got != ids.installationID { + t.Fatalf("installation_id = %q, want the converged value", got) + } + // The client omitted these; convergence must not introduce them. + for _, field := range []string{"session_id", "thread_id", "turn_id", "window_id", "turn_started_at_unix_ms"} { + if gjson.Get(metadata, field).Exists() { + t.Fatalf("turn metadata gained %s, which the client never sent", field) + } + } + if got := gjson.Get(metadata, "sandbox").String(); got != "read-only" { + t.Fatalf("sandbox = %q, want unrelated fields preserved", got) + } +} + +func TestApplyCodexConvergenceClientMetadataSkipsBodiesWithoutIt(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-nobody"), nil) + + body := []byte(`{"model":"gpt-5","input":[]}`) + got := applyCodexConvergenceClientMetadata(body, ids) + if string(got) != string(body) { + t.Fatalf("body without client_metadata was modified: %s", got) + } + if gjson.GetBytes(got, "client_metadata").Exists() { + t.Fatal("convergence fabricated a client_metadata object") + } +} + +func TestApplyCodexConvergenceHeadersSharesTurnIDWithBody(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-shared"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + headers := http.Header{} + headers.Set("X-Codex-Turn-Metadata", `{"turn_id":"client-turn"}`) + applyCodexConvergenceHeaders(headers, ids, http.Header{}) + + body := []byte(`{"client_metadata":{"turn_id":"client-turn"}}`) + body = applyCodexConvergenceClientMetadata(body, ids) + + headerTurn := gjson.Get(headers.Get("X-Codex-Turn-Metadata"), "turn_id").String() + bodyTurn := gjson.GetBytes(body, "client_metadata.turn_id").String() + if headerTurn == "" || headerTurn != bodyTurn { + t.Fatalf("header turn id %q and body turn id %q must match", headerTurn, bodyTurn) + } +} + +func TestDeviceOnlyLeavesSessionUntouched(t *testing.T) { + cfg := codexConvergenceTestConfig(config.CodexFingerprintConvergenceSession) + ids := resolveCodexConvergedIDs(cfg, codexConvergenceTestAuth("codex-ws"), nil) + if ids == nil { + t.Fatal("convergence returned nil for an auth with a resolvable scope") + } + + headers := http.Header{} + headers.Set("Session_id", "prompt-cache-key") + headers.Set("Conversation_id", "prompt-cache-key") + applyCodexConvergenceHeaders(headers, ids.deviceOnly(), http.Header{}) + + if got := headers.Get("Session_id"); got != "prompt-cache-key" { + t.Fatalf("Session_id = %q, want the prompt cache association preserved", got) + } + if got := headers.Get("Conversation_id"); got != "prompt-cache-key" { + t.Fatalf("Conversation_id = %q, want it preserved", got) + } +} + +func TestCodexConvergenceDefaultsToSessionMode(t *testing.T) { + cfg := codexConvergenceTestConfig("") + if got := codexConvergenceMode(cfg, codexConvergenceTestAuth("codex-default")); got != config.CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want the session default", got) + } + + cfg = codexConvergenceTestConfig("nonsense") + if got := codexConvergenceMode(cfg, codexConvergenceTestAuth("codex-bogus")); got != config.CodexFingerprintConvergenceSession { + t.Fatalf("convergence mode = %q, want an unrecognised value to fall back to the default", got) + } +} diff --git a/internal/runtime/executor/codex_websockets_helpers.go b/internal/runtime/executor/codex_websockets_helpers.go index e810bd06a..8a247d173 100644 --- a/internal/runtime/executor/codex_websockets_helpers.go +++ b/internal/runtime/executor/codex_websockets_helpers.go @@ -207,6 +207,12 @@ func applyCodexWebsocketHeaders(ctx context.Context, headers http.Header, cfg *c misc.EnsureHeader(headers, ginHeaders, "User-Agent", codexUserAgent) } + // Device fingerprint convergence, narrowed to the installation identifier: + // the websocket handshake pins Session_id to Conversation_id and + // prompt_cache_key above, so converging the session here would break prompt + // cache matching. + applyCodexConvergenceHeaders(headers, resolveCodexConvergedIDs(cfg, auth, ginHeaders).deviceOnly(), ginHeaders) + // Match upstream: only attach Session_id when UA indicates a desktop client, and do not forward UA over websocket. if strings.Contains(headers.Get("User-Agent"), "Mac OS") && strings.TrimSpace(headers.Get("Session_id")) == "" { headers.Set("Session_id", uuid.NewString()) diff --git a/internal/runtime/executor/proxy_helpers.go b/internal/runtime/executor/proxy_helpers.go index d6cea9826..303980bd4 100644 --- a/internal/runtime/executor/proxy_helpers.go +++ b/internal/runtime/executor/proxy_helpers.go @@ -65,8 +65,15 @@ func newProxyAwareHTTPClient(ctx context.Context, cfg *config.Config, auth *clip } recordRequestLogEgressRoute(ctx, cfg, auth, proxyURL) + // A TLS fingerprint transport dials and tunnels on its own, because the + // ClientHello has to be emitted by utls rather than by http.Transport. It + // therefore replaces the transport selection below instead of wrapping it. + if rt := tlsFingerprintRoundTripper(cfg, auth, proxyURL); rt != nil { + httpClient.Transport = rt + } + // If we have a proxy URL configured, set up the transport - if proxyURL != "" { + if proxyURL != "" && httpClient.Transport == nil { transport := cachedProxyTransport(proxyURL, cfgToSDKCfg(cfg)) if transport != nil { httpClient.Transport = transport diff --git a/internal/runtime/executor/tls_fingerprint_transport.go b/internal/runtime/executor/tls_fingerprint_transport.go new file mode 100644 index 000000000..2552a2976 --- /dev/null +++ b/internal/runtime/executor/tls_fingerprint_transport.go @@ -0,0 +1,108 @@ +package executor + +import ( + "net/http" + "strings" + "sync" + + "github.com/router-for-me/CLIProxyAPI/v6/internal/config" + "github.com/router-for-me/CLIProxyAPI/v6/internal/tlsfingerprint" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v6/sdk/cliproxy/auth" + log "github.com/sirupsen/logrus" +) + +// TLS fingerprint transports are cached process-wide for the same reason plain +// transports are: they own connection pools, so building one per request would +// force a fresh TLS handshake on every call. + +type tlsFingerprintTransportKey struct { + profile string + proxyURL string + preferIPv4 bool + insecureSkipVerify bool + caCert string + // caCertStat changes when the bundle on disk is replaced, so a rotated + // certificate produces a new transport instead of reusing one pinned to the + // old trust root. + caCertStat string +} + +var tlsFingerprintTransports = struct { + sync.Mutex + entries map[tlsFingerprintTransportKey]http.RoundTripper +}{entries: map[tlsFingerprintTransportKey]http.RoundTripper{}} + +// codexTLSFingerprintConfig returns the TLS fingerprint settings that apply to +// this auth, and whether fingerprinting is active for it. +// +// The settings live under the Codex identity fingerprint block and only apply +// to Codex traffic: a ClientHello has to match the client the rest of the +// request claims to be, so it cannot be configured independently of the +// identity headers. +func codexTLSFingerprintConfig(cfg *config.Config, auth *cliproxyauth.Auth) (config.TLSFingerprintConfig, bool) { + if cfg == nil || auth == nil { + return config.TLSFingerprintConfig{}, false + } + if !strings.EqualFold(strings.TrimSpace(auth.Provider), "codex") { + return config.TLSFingerprintConfig{}, false + } + tlsCfg := cfg.IdentityFingerprint.Codex.TLSFingerprint + if !tlsCfg.Enabled { + return config.TLSFingerprintConfig{}, false + } + return tlsCfg, true +} + +// tlsFingerprintRoundTripper returns a cached fingerprinting round tripper for +// this auth, or nil when fingerprinting does not apply. +// +// A construction failure is logged and returns nil so the caller falls back to +// the standard transport: a misconfigured profile must not take upstream +// traffic down, it only means the ClientHello stays the Go default. +func tlsFingerprintRoundTripper(cfg *config.Config, auth *cliproxyauth.Auth, proxyURL string) http.RoundTripper { + tlsCfg, ok := codexTLSFingerprintConfig(cfg, auth) + if !ok { + return nil + } + + profile := tlsfingerprint.NormalizeProfileName(tlsCfg.Profile) + if profile == "" { + profile = tlsfingerprint.DefaultProfile + } + key := tlsFingerprintTransportKey{ + profile: profile, + proxyURL: strings.TrimSpace(proxyURL), + } + // The fingerprint transport does its own TLS, so it has to honour the same + // trust settings the standard transport applies via util.ApplyTLSConfig; + // otherwise enabling fingerprinting would silently drop a configured CA + // bundle or an operator's decision to skip verification. + if sdkCfg := cfgToSDKCfg(cfg); sdkCfg != nil { + key.preferIPv4 = sdkCfg.PreferIPv4 + key.insecureSkipVerify = sdkCfg.InsecureSkipVerify + key.caCert = strings.TrimSpace(sdkCfg.CACert) + key.caCertStat = caCertStatFingerprint(key.caCert) + } + + tlsFingerprintTransports.Lock() + defer tlsFingerprintTransports.Unlock() + if cached, exists := tlsFingerprintTransports.entries[key]; exists { + return cached + } + + rt, err := tlsfingerprint.New(tlsfingerprint.Options{ + Profile: profile, + ProxyURL: key.proxyURL, + PreferIPv4: key.preferIPv4, + InsecureSkipVerify: key.insecureSkipVerify, + CACertPath: key.caCert, + }) + if err != nil { + log.WithError(err).Warn("tls fingerprint: falling back to the default transport") + // Cached as nil so a broken profile is not re-reported on every request. + tlsFingerprintTransports.entries[key] = nil + return nil + } + tlsFingerprintTransports.entries[key] = rt + return rt +} diff --git a/internal/tlsfingerprint/profile.go b/internal/tlsfingerprint/profile.go new file mode 100644 index 000000000..547bbb5c5 --- /dev/null +++ b/internal/tlsfingerprint/profile.go @@ -0,0 +1,94 @@ +// Package tlsfingerprint gives upstream requests a configurable TLS ClientHello +// fingerprint. +// +// Go's crypto/tls emits a ClientHello that no shipping browser or CLI produces, +// so JA3/JA4 based filtering can single out proxy traffic before a single byte +// of the request is seen. Sending a ClientHello that matches a real client +// removes that signal. +// +// Profiles are deliberately named after real clients rather than raw cipher +// lists: the point is to match a client that exists in the wild, and utls keeps +// the underlying byte sequences current as those clients ship new versions. +package tlsfingerprint + +import ( + "sort" + "strings" + + utls "github.com/refraction-networking/utls" +) + +// Profile names accepted by configuration. These are the stable identifiers +// exposed to operators; the utls ClientHelloID behind each one may advance as +// the library tracks new browser releases. +const ( + ProfileChrome = "chrome" + ProfileFirefox = "firefox" + ProfileSafari = "safari" + ProfileEdge = "edge" + ProfileIOS = "ios" + ProfileAndroid = "android" + ProfileRandomized = "randomized" +) + +// DefaultProfile is used when TLS fingerprinting is enabled without naming a +// profile. Chrome is the most common ClientHello on the public internet, so it +// is the least remarkable choice. +const DefaultProfile = ProfileChrome + +// profiles maps configuration names to utls ClientHello templates. +// +// The _Auto variants track the newest version utls implements, which is what +// keeps a profile from ageing into a fingerprint of its own once the real +// client has moved on. +var profiles = map[string]utls.ClientHelloID{ + ProfileChrome: utls.HelloChrome_Auto, + ProfileFirefox: utls.HelloFirefox_Auto, + ProfileSafari: utls.HelloSafari_Auto, + ProfileEdge: utls.HelloEdge_Auto, + ProfileIOS: utls.HelloIOS_Auto, + ProfileAndroid: utls.HelloAndroid_11_OkHttp, + // Randomized generates a fresh, internally consistent ClientHello per + // connection. It defeats fingerprint matching but is itself unusual, since + // no real client varies its ClientHello between connections. + ProfileRandomized: utls.HelloRandomizedALPN, +} + +// ResolveProfile maps a configured profile name to its ClientHello template. +// It reports false for unknown names so callers can fail loudly at config load +// rather than silently falling back to a fingerprint the operator did not ask +// for. +func ResolveProfile(name string) (utls.ClientHelloID, bool) { + normalized := NormalizeProfileName(name) + if normalized == "" { + return profiles[DefaultProfile], true + } + id, ok := profiles[normalized] + return id, ok +} + +// NormalizeProfileName trims and lowercases a profile name for comparison. +func NormalizeProfileName(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} + +// IsValidProfile reports whether name identifies a known profile. An empty name +// is valid and selects DefaultProfile. +func IsValidProfile(name string) bool { + if NormalizeProfileName(name) == "" { + return true + } + _, ok := ResolveProfile(name) + return ok +} + +// AvailableProfiles lists the supported profile names in a stable order, for +// configuration validation messages and management surfaces. +func AvailableProfiles() []string { + names := make([]string, 0, len(profiles)) + for name := range profiles { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/tlsfingerprint/proxy_test.go b/internal/tlsfingerprint/proxy_test.go new file mode 100644 index 000000000..d5ea0d4cb --- /dev/null +++ b/internal/tlsfingerprint/proxy_test.go @@ -0,0 +1,240 @@ +package tlsfingerprint + +import ( + "bufio" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// connectProxy is a minimal HTTP CONNECT proxy used to exercise the tunnelling +// path, which http.Transport would normally own. +type connectProxy struct { + listener net.Listener + wg sync.WaitGroup + + mu sync.Mutex + requireAuth string + observedTargets []string + observedAuthHdrs []string +} + +func newConnectProxy(t *testing.T, requireAuth string) *connectProxy { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + p := &connectProxy{listener: listener, requireAuth: requireAuth} + p.wg.Add(1) + go p.serve() + t.Cleanup(func() { + _ = listener.Close() + p.wg.Wait() + }) + return p +} + +func (p *connectProxy) addr() string { return p.listener.Addr().String() } + +func (p *connectProxy) targets() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.observedTargets...) +} + +func (p *connectProxy) authHeaders() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.observedAuthHdrs...) +} + +func (p *connectProxy) serve() { + defer p.wg.Done() + for { + clientConn, err := p.listener.Accept() + if err != nil { + return + } + p.wg.Add(1) + go func() { + defer p.wg.Done() + p.handle(clientConn) + }() + } +} + +func (p *connectProxy) handle(clientConn net.Conn) { + defer func() { _ = clientConn.Close() }() + + reader := bufio.NewReader(clientConn) + req, err := http.ReadRequest(reader) + if err != nil { + return + } + if req.Method != http.MethodConnect { + _, _ = io.WriteString(clientConn, "HTTP/1.1 405 Method Not Allowed\r\n\r\n") + return + } + + p.mu.Lock() + p.observedTargets = append(p.observedTargets, req.Host) + p.observedAuthHdrs = append(p.observedAuthHdrs, req.Header.Get("Proxy-Authorization")) + required := p.requireAuth + p.mu.Unlock() + + if required != "" && req.Header.Get("Proxy-Authorization") != required { + _, _ = io.WriteString(clientConn, "HTTP/1.1 407 Proxy Authentication Required\r\n\r\n") + return + } + + upstream, err := net.Dial("tcp", req.Host) + if err != nil { + _, _ = io.WriteString(clientConn, "HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + } + defer func() { _ = upstream.Close() }() + + if _, err = io.WriteString(clientConn, "HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + return + } + + var pipeWG sync.WaitGroup + pipeWG.Add(2) + go func() { + defer pipeWG.Done() + _, _ = io.Copy(upstream, reader) + }() + go func() { + defer pipeWG.Done() + _, _ = io.Copy(clientConn, upstream) + }() + pipeWG.Wait() +} + +func newTLSEchoServer(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Proto", r.Proto) + _, _ = io.WriteString(w, "tunnelled") + })) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + return server +} + +func TestRoundTripThroughHTTPConnectProxy(t *testing.T) { + origin := newTLSEchoServer(t) + proxy := newConnectProxy(t, "") + + rt, err := New(Options{ + Profile: ProfileChrome, + ProxyURL: "http://" + proxy.addr(), + InsecureSkipVerify: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip through CONNECT proxy: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if string(body) != "tunnelled" { + t.Fatalf("body = %q, want %q", body, "tunnelled") + } + // The TLS handshake must happen inside the tunnel, so the origin still sees + // a fingerprinted HTTP/2 connection rather than something the proxy re-made. + if got := resp.Header.Get("X-Proto"); got != "HTTP/2.0" { + t.Fatalf("origin saw %q, want HTTP/2 negotiated inside the tunnel", got) + } + + targets := proxy.targets() + if len(targets) != 1 { + t.Fatalf("proxy saw %d CONNECT requests, want 1", len(targets)) + } + originHost := strings.TrimPrefix(origin.URL, "https://") + if targets[0] != originHost { + t.Fatalf("CONNECT target = %q, want %q", targets[0], originHost) + } +} + +func TestRoundTripSendsProxyCredentials(t *testing.T) { + origin := newTLSEchoServer(t) + + credentialed := &http.Request{Header: make(http.Header)} + credentialed.SetBasicAuth("alice", "s3cret") + expected := credentialed.Header.Get("Authorization") + + proxy := newConnectProxy(t, expected) + + rt, err := New(Options{ + Profile: ProfileChrome, + ProxyURL: "http://alice:s3cret@" + proxy.addr(), + InsecureSkipVerify: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip through authenticated proxy: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + headers := proxy.authHeaders() + if len(headers) != 1 || headers[0] != expected { + t.Fatalf("Proxy-Authorization = %v, want the configured credentials", headers) + } +} + +func TestRoundTripSurfacesProxyRejection(t *testing.T) { + origin := newTLSEchoServer(t) + proxy := newConnectProxy(t, "Basic expected-but-not-sent") + + rt, err := New(Options{ + Profile: ProfileChrome, + ProxyURL: "http://" + proxy.addr(), + InsecureSkipVerify: true, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + t.Fatal("a rejected CONNECT must surface as an error, not a silent fallback") + } + if !strings.Contains(err.Error(), "407") { + t.Fatalf("error = %v, want it to carry the proxy status", err) + } +} diff --git a/internal/tlsfingerprint/roundtripper.go b/internal/tlsfingerprint/roundtripper.go new file mode 100644 index 000000000..0ee421b00 --- /dev/null +++ b/internal/tlsfingerprint/roundtripper.go @@ -0,0 +1,431 @@ +package tlsfingerprint + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + utls "github.com/refraction-networking/utls" + "golang.org/x/net/http2" + "golang.org/x/net/proxy" +) + +// Options configures a fingerprinting RoundTripper. +type Options struct { + // Profile names the ClientHello template; empty selects DefaultProfile. + Profile string + // ProxyURL routes connections through an HTTP, HTTPS or SOCKS5 proxy. + // Empty dials directly. + ProxyURL string + // PreferIPv4 restricts dialing to IPv4, matching the rest of the proxy's + // egress behaviour. + PreferIPv4 bool + // DialTimeout bounds TCP establishment and the TLS handshake. + DialTimeout time.Duration + // InsecureSkipVerify disables certificate verification. It exists only for + // operators terminating upstream TLS on an inspection proxy. + InsecureSkipVerify bool + // CACertPath adds a PEM bundle as the trust root, for the same inspection + // proxy setups. Empty uses the system pool. + CACertPath string +} + +const defaultDialTimeout = 30 * time.Second + +// RoundTripper performs HTTPS requests over connections whose ClientHello +// matches a configured client profile. +// +// It manages HTTP/2 connections itself rather than delegating to +// http.Transport: Go only routes a connection to its HTTP/2 stack when the +// connection is a *crypto/tls.Conn, and a utls connection never is. Handing a +// utls connection to http.Transport would therefore speak HTTP/1.1 on a +// connection where the server already agreed to HTTP/2 via ALPN. +type RoundTripper struct { + opts Options + helloID utls.ClientHelloID + // rootCAs is resolved once at construction so a missing or malformed bundle + // is reported at setup instead of on every handshake. + rootCAs *x509.CertPool + + mu sync.Mutex + // h2Conns caches one HTTP/2 connection per host. HTTP/2 multiplexes, so a + // single connection carries concurrent requests. + h2Conns map[string]*http2.ClientConn + // dialing serializes connection setup per host so a burst of requests to a + // cold host opens one connection instead of one per request. + dialing map[string]*sync.Cond +} + +// New builds a RoundTripper for the given options. It returns an error for an +// unknown profile or an unparseable proxy URL so misconfiguration surfaces at +// setup instead of as a per-request failure. +func New(opts Options) (*RoundTripper, error) { + helloID, ok := ResolveProfile(opts.Profile) + if !ok { + return nil, fmt.Errorf("tlsfingerprint: unknown profile %q, want one of %s", + opts.Profile, strings.Join(AvailableProfiles(), ", ")) + } + if proxyURL := strings.TrimSpace(opts.ProxyURL); proxyURL != "" { + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("tlsfingerprint: parse proxy URL: %w", err) + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https", "socks5", "socks5h": + default: + return nil, fmt.Errorf("tlsfingerprint: unsupported proxy scheme %q", parsed.Scheme) + } + } + if opts.DialTimeout <= 0 { + opts.DialTimeout = defaultDialTimeout + } + + var rootCAs *x509.CertPool + if caPath := strings.TrimSpace(opts.CACertPath); caPath != "" { + pem, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("tlsfingerprint: read CA cert %s: %w", caPath, err) + } + rootCAs = x509.NewCertPool() + if !rootCAs.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("tlsfingerprint: parse CA cert %s", caPath) + } + } + + return &RoundTripper{ + opts: opts, + helloID: helloID, + rootCAs: rootCAs, + h2Conns: make(map[string]*http2.ClientConn), + dialing: make(map[string]*sync.Cond), + }, nil +} + +// RoundTrip implements http.RoundTripper. +func (t *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL == nil { + return nil, errors.New("tlsfingerprint: request has no URL") + } + if !strings.EqualFold(req.URL.Scheme, "https") { + // Plaintext requests carry no ClientHello, so there is nothing to + // fingerprint and no reason for this transport to handle them. + return nil, fmt.Errorf("tlsfingerprint: scheme %q is not supported, want https", req.URL.Scheme) + } + + host := req.URL.Hostname() + addr := canonicalAddr(req.URL) + + h2Conn, err := t.h2ConnFor(req.Context(), host, addr) + if err != nil { + return nil, err + } + if h2Conn != nil { + resp, errRT := h2Conn.RoundTrip(req) + if errRT != nil { + t.dropH2Conn(host, h2Conn) + return nil, errRT + } + return resp, nil + } + + // The server declined HTTP/2 for this profile's ALPN list. Fall back to a + // single-use HTTP/1.1 exchange; without Go's connection pool behind it there + // is no keep-alive here, which is acceptable because every provider this + // proxy targets negotiates HTTP/2. + return t.roundTripHTTP1(req, host, addr) +} + +// h2ConnFor returns a usable HTTP/2 connection for host, or nil when the +// handshake settled on HTTP/1.1. +func (t *RoundTripper) h2ConnFor(ctx context.Context, host, addr string) (*http2.ClientConn, error) { + t.mu.Lock() + for { + if conn, ok := t.h2Conns[host]; ok { + if conn.CanTakeNewRequest() { + t.mu.Unlock() + return conn, nil + } + delete(t.h2Conns, host) + } + cond, inFlight := t.dialing[host] + if !inFlight { + break + } + // Another goroutine is dialing this host; wait for it and re-check. + cond.Wait() + } + + cond := sync.NewCond(&t.mu) + t.dialing[host] = cond + t.mu.Unlock() + + conn, err := t.dialH2(ctx, host, addr) + + t.mu.Lock() + delete(t.dialing, host) + cond.Broadcast() + if err == nil && conn != nil { + t.h2Conns[host] = conn + } + t.mu.Unlock() + + return conn, err +} + +// dialH2 establishes a fingerprinted TLS connection and, when ALPN selected +// HTTP/2, wraps it in an HTTP/2 client connection. A nil connection with a nil +// error means the peer chose HTTP/1.1. +func (t *RoundTripper) dialH2(ctx context.Context, host, addr string) (*http2.ClientConn, error) { + tlsConn, err := t.dialTLS(ctx, host, addr) + if err != nil { + return nil, err + } + if tlsConn.ConnectionState().NegotiatedProtocol != http2.NextProtoTLS { + // Closed here because the HTTP/1.1 path re-dials: reusing this + // connection would require threading it back out through the cache, + // which is not worth it for a path the target providers never take. + _ = tlsConn.Close() + return nil, nil + } + h2Transport := &http2.Transport{} + h2Conn, err := h2Transport.NewClientConn(tlsConn) + if err != nil { + _ = tlsConn.Close() + return nil, err + } + return h2Conn, nil +} + +// dialTLS opens a TCP connection through the configured egress path and +// performs the fingerprinted TLS handshake on top of it. +func (t *RoundTripper) dialTLS(ctx context.Context, host, addr string) (*utls.UConn, error) { + rawConn, err := t.dialTCP(ctx, addr) + if err != nil { + return nil, err + } + + if deadline, ok := ctx.Deadline(); ok { + _ = rawConn.SetDeadline(deadline) + } else { + _ = rawConn.SetDeadline(time.Now().Add(t.opts.DialTimeout)) + } + + tlsConn := utls.UClient(rawConn, &utls.Config{ + ServerName: host, + InsecureSkipVerify: t.opts.InsecureSkipVerify, + RootCAs: t.rootCAs, + }, t.helloID) + if err = tlsConn.HandshakeContext(ctx); err != nil { + _ = rawConn.Close() + return nil, err + } + // Clear the handshake deadline: streaming responses must not inherit it. + _ = rawConn.SetDeadline(time.Time{}) + return tlsConn, nil +} + +// dialTCP establishes the underlying TCP connection, tunnelling through a proxy +// when one is configured. +func (t *RoundTripper) dialTCP(ctx context.Context, addr string) (net.Conn, error) { + proxyURL := strings.TrimSpace(t.opts.ProxyURL) + if proxyURL == "" { + return t.baseDialer().DialContext(ctx, t.network(), addr) + } + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("tlsfingerprint: parse proxy URL: %w", err) + } + switch strings.ToLower(parsed.Scheme) { + case "socks5", "socks5h": + return t.dialViaSOCKS5(ctx, parsed, addr) + case "http", "https": + return t.dialViaHTTPProxy(ctx, parsed, addr) + default: + return nil, fmt.Errorf("tlsfingerprint: unsupported proxy scheme %q", parsed.Scheme) + } +} + +func (t *RoundTripper) baseDialer() *net.Dialer { + return &net.Dialer{Timeout: t.opts.DialTimeout, KeepAlive: 30 * time.Second} +} + +func (t *RoundTripper) network() string { + if t.opts.PreferIPv4 { + return "tcp4" + } + return "tcp" +} + +func (t *RoundTripper) dialViaSOCKS5(ctx context.Context, proxyURL *url.URL, addr string) (net.Conn, error) { + var auth *proxy.Auth + if proxyURL.User != nil { + password, _ := proxyURL.User.Password() + auth = &proxy.Auth{User: proxyURL.User.Username(), Password: password} + } + dialer, err := proxy.SOCKS5(t.network(), proxyURL.Host, auth, t.baseDialer()) + if err != nil { + return nil, err + } + if contextDialer, ok := dialer.(proxy.ContextDialer); ok { + return contextDialer.DialContext(ctx, t.network(), addr) + } + return dialer.Dial(t.network(), addr) +} + +// dialViaHTTPProxy opens a CONNECT tunnel. http.Transport would normally handle +// this, but it only exposes the tunnelled connection to its own TLS stack, so +// the tunnel has to be established here for the utls handshake to run on top. +func (t *RoundTripper) dialViaHTTPProxy(ctx context.Context, proxyURL *url.URL, addr string) (net.Conn, error) { + proxyAddr := proxyURL.Host + if proxyURL.Port() == "" { + if strings.EqualFold(proxyURL.Scheme, "https") { + proxyAddr = net.JoinHostPort(proxyURL.Hostname(), "443") + } else { + proxyAddr = net.JoinHostPort(proxyURL.Hostname(), "80") + } + } + + conn, err := t.baseDialer().DialContext(ctx, t.network(), proxyAddr) + if err != nil { + return nil, err + } + + // An HTTPS proxy speaks TLS on the hop to the proxy itself. That handshake + // uses the standard library on purpose: the fingerprint that matters is the + // one the origin server sees, inside the tunnel. + if strings.EqualFold(proxyURL.Scheme, "https") { + proxyTLS := tls.Client(conn, &tls.Config{ServerName: proxyURL.Hostname()}) + if err = proxyTLS.HandshakeContext(ctx); err != nil { + _ = conn.Close() + return nil, err + } + conn = proxyTLS + } + + connectReq := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: make(http.Header), + } + if proxyURL.User != nil { + password, _ := proxyURL.User.Password() + connectReq.Header.Set("Proxy-Authorization", basicProxyAuth(proxyURL.User.Username(), password)) + } + + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } else { + _ = conn.SetDeadline(time.Now().Add(t.opts.DialTimeout)) + } + if err = connectReq.Write(conn); err != nil { + _ = conn.Close() + return nil, err + } + + // Buffered reads must stop at the end of the CONNECT response: anything the + // reader buffers past it belongs to the TLS handshake that follows. + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, connectReq) + if err != nil { + _ = conn.Close() + return nil, err + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _ = conn.Close() + return nil, fmt.Errorf("tlsfingerprint: proxy CONNECT to %s failed: %s", addr, resp.Status) + } + if reader.Buffered() > 0 { + _ = conn.Close() + return nil, errors.New("tlsfingerprint: proxy sent data before the tunnel was established") + } + _ = conn.SetDeadline(time.Time{}) + return conn, nil +} + +// roundTripHTTP1 serves one request over a dedicated fingerprinted connection. +func (t *RoundTripper) roundTripHTTP1(req *http.Request, host, addr string) (*http.Response, error) { + tlsConn, err := t.dialTLS(req.Context(), host, addr) + if err != nil { + return nil, err + } + if err = req.Write(tlsConn); err != nil { + _ = tlsConn.Close() + return nil, err + } + resp, err := http.ReadResponse(bufio.NewReader(tlsConn), req) + if err != nil { + _ = tlsConn.Close() + return nil, err + } + // The caller closes the body; closing the connection with it prevents the + // socket leaking, since this path keeps no pool. + resp.Body = &connClosingBody{ReadCloser: resp.Body, conn: tlsConn} + return resp, nil +} + +// connClosingBody ties a response body to the connection that produced it, so +// closing the body releases the socket on this pool-less path. +type connClosingBody struct { + io.ReadCloser + conn net.Conn +} + +func (b *connClosingBody) Close() error { + err := b.ReadCloser.Close() + if errConn := b.conn.Close(); err == nil { + err = errConn + } + return err +} + +func (t *RoundTripper) dropH2Conn(host string, conn *http2.ClientConn) { + t.mu.Lock() + if cached, ok := t.h2Conns[host]; ok && cached == conn { + delete(t.h2Conns, host) + } + t.mu.Unlock() +} + +// CloseIdleConnections implements the optional http.Transport behaviour used by +// http.Client.CloseIdleConnections. +func (t *RoundTripper) CloseIdleConnections() { + t.mu.Lock() + conns := make([]*http2.ClientConn, 0, len(t.h2Conns)) + for host, conn := range t.h2Conns { + conns = append(conns, conn) + delete(t.h2Conns, host) + } + t.mu.Unlock() + for _, conn := range conns { + _ = conn.Close() + } +} + +// canonicalAddr returns host:port, defaulting to the HTTPS port. +func canonicalAddr(u *url.URL) string { + host := u.Hostname() + port := u.Port() + if port == "" { + port = "443" + } + return net.JoinHostPort(host, port) +} + +func basicProxyAuth(username, password string) string { + req := &http.Request{Header: make(http.Header)} + req.SetBasicAuth(username, password) + return req.Header.Get("Authorization") +} diff --git a/internal/tlsfingerprint/roundtripper_test.go b/internal/tlsfingerprint/roundtripper_test.go new file mode 100644 index 000000000..42a6c6284 --- /dev/null +++ b/internal/tlsfingerprint/roundtripper_test.go @@ -0,0 +1,273 @@ +package tlsfingerprint + +import ( + "crypto/tls" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveProfile(t *testing.T) { + for _, name := range AvailableProfiles() { + if _, ok := ResolveProfile(name); !ok { + t.Fatalf("advertised profile %q does not resolve", name) + } + } + if _, ok := ResolveProfile(""); !ok { + t.Fatal("empty profile must resolve to the default") + } + if _, ok := ResolveProfile(" ChRoMe "); !ok { + t.Fatal("profile lookup must ignore case and surrounding space") + } + if _, ok := ResolveProfile("netscape"); ok { + t.Fatal("unknown profile must not resolve") + } + if !IsValidProfile("") || !IsValidProfile("firefox") || IsValidProfile("netscape") { + t.Fatal("IsValidProfile disagrees with ResolveProfile") + } +} + +func TestNewRejectsBadConfiguration(t *testing.T) { + if _, err := New(Options{Profile: "netscape"}); err == nil { + t.Fatal("an unknown profile must fail at construction, not per request") + } + if _, err := New(Options{ProxyURL: "ftp://proxy.invalid:21"}); err == nil { + t.Fatal("an unsupported proxy scheme must fail at construction") + } + if _, err := New(Options{ProxyURL: "://not a url"}); err == nil { + t.Fatal("an unparseable proxy URL must fail at construction") + } + if _, err := New(Options{Profile: "chrome", ProxyURL: "socks5://127.0.0.1:1080"}); err != nil { + t.Fatalf("valid options rejected: %v", err) + } +} + +// TestRoundTripHonoursCustomCA guards against the fingerprint transport quietly +// ignoring trust settings the standard transport applies: it does its own TLS, +// so a configured CA bundle has to be threaded through explicitly. +func TestRoundTripHonoursCustomCA(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + + caPath := filepath.Join(t.TempDir(), "ca.pem") + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + if err := os.WriteFile(caPath, pemBytes, 0o600); err != nil { + t.Fatalf("write CA bundle: %v", err) + } + + // Without the bundle the self-signed certificate must be rejected. + strict, err := New(Options{Profile: ProfileChrome}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer strict.CloseIdleConnections() + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := strict.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + t.Fatal("an untrusted certificate must fail verification") + } + + trusting, err := New(Options{Profile: ProfileChrome, CACertPath: caPath}) + if err != nil { + t.Fatalf("New with CA bundle: %v", err) + } + defer trusting.CloseIdleConnections() + req, err = http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err = trusting.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip with the configured CA: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() +} + +func TestNewRejectsUnusableCABundle(t *testing.T) { + if _, err := New(Options{Profile: ProfileChrome, CACertPath: filepath.Join(t.TempDir(), "missing.pem")}); err == nil { + t.Fatal("a missing CA bundle must fail at construction") + } + + junk := filepath.Join(t.TempDir(), "junk.pem") + if err := os.WriteFile(junk, []byte("not a certificate"), 0o600); err != nil { + t.Fatalf("write junk bundle: %v", err) + } + if _, err := New(Options{Profile: ProfileChrome, CACertPath: junk}); err == nil { + t.Fatal("an unparseable CA bundle must fail at construction") + } +} + +func TestRoundTripRejectsPlaintext(t *testing.T) { + rt, err := New(Options{Profile: ProfileChrome}) + if err != nil { + t.Fatalf("New: %v", err) + } + req := httptest.NewRequest(http.MethodGet, "http://example.invalid/", nil) + resp, err := rt.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + t.Fatal("plaintext requests carry no ClientHello and must be refused") + } +} + +// TestRoundTripAgainstTLSServer drives a real handshake so the ClientHello, +// ALPN negotiation and HTTP/2 framing are exercised end to end rather than +// mocked. +func TestRoundTripAgainstTLSServer(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Proto", r.Proto) + _, _ = io.WriteString(w, "ok") + })) + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + + rt, err := New(Options{Profile: ProfileChrome, InsecureSkipVerify: true}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if string(body) != "ok" { + t.Fatalf("body = %q, want %q", body, "ok") + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + if got := resp.Header.Get("X-Proto"); got != "HTTP/2.0" { + t.Fatalf("server saw %q, want the connection to negotiate HTTP/2", got) + } +} + +// TestRoundTripReusesHTTP2Connection verifies the per-host cache: a second +// request must multiplex over the connection the first one opened rather than +// repeating the handshake. +func TestRoundTripReusesHTTP2Connection(t *testing.T) { + var handshakes int + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + server.EnableHTTP2 = true + server.TLS = &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + handshakes++ + return nil, nil + }, + } + server.StartTLS() + defer server.Close() + + rt, err := New(Options{Profile: ProfileFirefox, InsecureSkipVerify: true}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer rt.CloseIdleConnections() + + for i := 0; i < 3; i++ { + req, errReq := http.NewRequest(http.MethodGet, server.URL, nil) + if errReq != nil { + t.Fatalf("NewRequest: %v", errReq) + } + resp, errRT := rt.RoundTrip(req) + if errRT != nil { + t.Fatalf("RoundTrip %d: %v", i, errRT) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + + if handshakes != 1 { + t.Fatalf("handshakes = %d, want the connection reused across requests", handshakes) + } +} + +// TestRoundTripSendsProfileSpecificClientHello confirms the transport actually +// changes the wire bytes: two profiles must not produce the same ClientHello, +// and neither should look like Go's default. +func TestRoundTripSendsProfileSpecificClientHello(t *testing.T) { + capture := func(profile string) *tls.ClientHelloInfo { + var captured *tls.ClientHelloInfo + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + server.TLS = &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + copied := *hello + captured = &copied + return nil, nil + }, + } + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + + rt, err := New(Options{Profile: profile, InsecureSkipVerify: true}) + if err != nil { + t.Fatalf("New(%s): %v", profile, err) + } + defer rt.CloseIdleConnections() + + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip(%s): %v", profile, err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if captured == nil { + t.Fatalf("no ClientHello captured for %s", profile) + } + return captured + } + + chrome := capture(ProfileChrome) + firefox := capture(ProfileFirefox) + + if joinUint16(chrome.CipherSuites) == joinUint16(firefox.CipherSuites) { + t.Fatal("chrome and firefox produced identical cipher suite lists") + } + for _, hello := range []*tls.ClientHelloInfo{chrome, firefox} { + if len(hello.SupportedProtos) == 0 || hello.SupportedProtos[0] != "h2" { + t.Fatalf("ALPN = %v, want h2 offered first", hello.SupportedProtos) + } + } +} + +func joinUint16(values []uint16) string { + parts := make([]string, 0, len(values)) + for _, v := range values { + parts = append(parts, string(rune(v))) + } + return strings.Join(parts, ",") +}