diff --git a/.env b/.env new file mode 100644 index 0000000..6ee3e47 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +UID= +GID= +USER= diff --git a/.gitignore b/.gitignore index 0892ad6..566248a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /clawshell.toml /tarpaulin-report.json /tarpaulin-report.html +.env +.idea/ # npm platform binaries (added during release, not checked in) npm/clawshell-*/bin/clawshell diff --git a/Cargo.lock b/Cargo.lock index 81c2b1e..b8b3605 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -28,9 +37,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" @@ -63,9 +72,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.101" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "assert-json-diff" @@ -92,6 +101,17 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -106,9 +126,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.0" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", "zeroize", @@ -116,9 +136,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.1" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" dependencies = [ "cc", "cmake", @@ -224,9 +244,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -252,6 +272,31 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clap" version = "4.6.0" @@ -288,17 +333,20 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clawshell" version = "0.1.1" dependencies = [ "assert_cmd", + "async-trait", "axum", + "base64", "bytes", + "chrono", "clap", "console 0.16.3", "futures-util", @@ -307,7 +355,9 @@ dependencies = [ "inquire", "insta", "nix", + "open", "predicates", + "rand 0.10.0", "regex", "reqwest", "rustls", @@ -319,11 +369,13 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "tokio-util", "toml", "tower", "tower-http", "tracing", "tracing-subscriber", + "urlencoding", "uuid", "vfs", "wiremock", @@ -340,9 +392,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" @@ -412,6 +464,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crossterm" version = "0.29.0" @@ -747,20 +808,21 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core 0.10.0", "wasip2", "wasip3", ] @@ -918,6 +980,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -1067,9 +1153,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" @@ -1081,6 +1167,25 @@ dependencies = [ "serde", ] +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1089,9 +1194,9 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jni" @@ -1127,9 +1232,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.85" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ "once_cell", "wasm-bindgen", @@ -1155,20 +1260,21 @@ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ "bitflags", "libc", - "redox_syscall 0.7.1", + "plain", + "redox_syscall 0.7.3", ] [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -1290,9 +1396,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -1300,6 +1406,17 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "open" +version = "5.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb73a7fa3799b198970490a51174027ba0d4ec504b03cd08caf513d40024bc" +dependencies = [ + "is-wsl", + "libc", + "pathdiff", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -1329,6 +1446,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1337,9 +1460,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -1347,6 +1470,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "potential_utf" version = "0.1.4" @@ -1444,7 +1573,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.2", "ring", "rustc-hash", "rustls", @@ -1485,6 +1614,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.2" @@ -1492,7 +1627,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.0", ] [[package]] @@ -1502,7 +1648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1514,6 +1660,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1525,9 +1677,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35985aa610addc02e24fc232012c86fd11f14111180f902b67e2d5331f8ebf2b" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" dependencies = [ "bitflags", ] @@ -1557,9 +1709,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" @@ -1587,6 +1739,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier", "serde", + "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", @@ -1633,9 +1786,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", @@ -1743,9 +1896,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -1758,9 +1911,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", "core-foundation", @@ -1771,9 +1924,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.16.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321c8673b092a9a42605034a9879d73cb79101ed5fd117bc9a597b89b4e9e61a" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -1867,7 +2020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1937,12 +2090,12 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1996,12 +2149,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.25.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2074,9 +2227,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -2106,9 +2259,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", @@ -2133,16 +2286,20 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "hashbrown 0.15.5", "pin-project-lite", + "slab", "tokio", ] [[package]] name = "toml" -version = "1.0.6+spec-1.1.0" +version = "1.0.7+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399b1124a3c9e16766831c6bba21e50192572cdd98706ea114f9502509686ffc" +checksum = "dd28d57d8a6f6e458bc0b8784f8fdcc4b99a437936056fa122cb234f18656a96" dependencies = [ "indexmap", "serde_core", @@ -2155,27 +2312,27 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.0+spec-1.1.0" +version = "1.0.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" +checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9" dependencies = [ "serde_core", ] [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.0.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.6+spec-1.1.0" +version = "1.0.7+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d" [[package]] name = "tower" @@ -2353,6 +2510,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -2371,7 +2534,7 @@ version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ - "getrandom 0.4.1", + "getrandom 0.4.2", "js-sys", "wasm-bindgen", ] @@ -2451,9 +2614,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ "cfg-if", "once_cell", @@ -2464,9 +2627,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.58" +version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ "cfg-if", "futures-util", @@ -2478,9 +2641,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2488,9 +2651,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ "bumpalo", "proc-macro2", @@ -2501,9 +2664,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] @@ -2557,9 +2720,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.85" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", @@ -2615,12 +2778,65 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -2854,9 +3070,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" [[package]] name = "wiremock" @@ -3000,18 +3216,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.39" +version = "0.8.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.39" +version = "0.8.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 806b1e4..7075453 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ description = "A security privileged process for the OpenClaw ecosystem." [dependencies] axum = "0.8.8" tokio = { version = "1.50", features = ["full"] } -reqwest = { version = "0.13.2", default-features = false, features = ["stream", "rustls", "form", "blocking"] } +reqwest = { version = "0.13.2", default-features = false, features = ["stream", "rustls", "form", "blocking", "json"] } rustls = { version = "0.23.37", default-features = false, features = ["ring", "std"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" @@ -34,6 +34,13 @@ semver = "1" rustls-native-certs = "0.8.3" sha2 = "0.10.9" uuid = { version = "1.22.0", features = ["v4"] } +async-trait = "0.1.89" +chrono = { version = "0.4.44", features = ["serde"] } +base64 = "0.22.1" +urlencoding = "2.1.3" +rand = "0.10.0" +open = "5.3.3" +tokio-util = { version = "0.7.18", features = ["full"] } [dev-dependencies] tokio = { version = "1.50", features = ["full", "test-util"] } diff --git a/README.md b/README.md index d6d0f1c..1915ea3 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,20 @@ ClawShell supports sender-based email filtering so each virtual key only sees ma - **Key Isolation**: IMAP credentials are stored in `/etc/clawshell/clawshell.toml`, readable only by the `clawshell` system user. OpenClaw holds only virtual keys. - **Provider Support**: Built-in Gmail and Outlook presets, with manual IMAP setup for other providers. -### 4. Seamless Integration +### 4. OAuth Authentication (Codex / ChatGPT) + +ClawShell supports OAuth-based authentication as an alternative to static API keys. + +- **Device Code Flow**: Log in via a one-time code — no browser required on the server. The onboard wizard prints a URL and code; authorize from any device. +- **Automatic Token Refresh**: Access tokens are refreshed transparently before they expire. +- **Request Translation**: Automatically translates OpenAI Chat Completions API requests to the ChatGPT Responses API format when using Codex OAuth. + +### 5. Seamless Integration - **Drop-in Sidecar**: Deploys alongside OpenClaw without requiring re-install — the `clawshell onboard` command automatically configures OpenClaw to point at ClawShell's address and forwards all requests upstream. - **No External Dependencies**: Uses Unix file system permissions to protect secrets. No IdP, Vault, or external key management service required. -### 5. Ultra Lightweight and Scalable +### 6. Ultra Lightweight and Scalable - Runs in under 10MB of memory. - Written in Rust with Tokio. @@ -173,6 +181,12 @@ sudo clawshell migrate-config By default ClawShell listens on `127.0.0.1:18790`. +You can override the bind address at runtime with environment variables: + +```bash +CLAWSHELL_SERVER_HOST=0.0.0.0 CLAWSHELL_SERVER_PORT=17890 clawshell start --foreground +``` + ### Customized Configuration ClawShell reads its config from `/etc/clawshell/clawshell.toml`. You can view or edit it with: @@ -236,6 +250,43 @@ imap_port = 993 # imap_host = "imap-mail.outlook.com" ``` +### OAuth Authentication (Codex / ChatGPT) + +Instead of a static API key, you can authenticate via OAuth using your ChatGPT / Codex account. + +#### Setup via onboard + +During `sudo clawshell onboard`, select **"Codex / ChatGPT (OAuth)"** as the provider. The wizard will start a device code flow: + +1. A URL and one-time code are printed to the terminal. +2. Open the URL on any device and enter the code. +3. Once authorized, tokens are saved automatically to `/etc/clawshell/oauth/`. + +No browser is required on the server. + +#### Manual configuration + +To configure OAuth manually in `clawshell.toml`: + +```toml +# OAuth-backed key — no real_key needed +[[keys]] +virtual_key = "vk-codex-001" +auth = "oauth" +oauth_provider = "codex" +provider = "openai" + +# OAuth provider definition +[[oauth_providers]] +provider = "codex" +# Optional overrides (defaults work for ChatGPT): +# client_id = "app_EMoamEEZ73f0CkXaXp7hrann" +# auth_url = "https://auth.openai.com/authorize" +# token_url = "https://auth.openai.com/oauth/token" +``` + +ClawShell handles token refresh automatically. When using Codex OAuth, requests to `/v1/chat/completions` are translated to the ChatGPT Responses API format and routed to `chatgpt.com/backend-api/codex`. + If `start`, `restart`, `stop`, `config --edit`, `onboard`, or `uninstall` reports that migration is required, run: ```bash diff --git a/src/app.rs b/src/app.rs index 45a4f1d..80a66a1 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,17 +1,18 @@ -use crate::config::{Config, Provider}; +use crate::config::{Config, KeyAuthMethod, Provider}; use crate::dlp::DlpScanner; use crate::email::{ EmailAccountCredentials, EmailGetMessageRequest, EmailListMessagesRequest, EmailMessageContent, EmailMessageMetadata, EmailPolicy, EmailService, EmailServiceError, ImapEmailService, normalize_sender_rule, }; -use crate::keys::{KeyManager, ResolvedKey}; +use crate::keys::{KeyManager, KeySource, ResolvedKey}; +use crate::oauth::OAuthRegistry; use crate::proxy::ProxyClient; use axum::Router; use axum::body::Body; use axum::extract::{DefaultBodyLimit, Path, Query, Request, State}; -use axum::http::StatusCode; +use axum::http::{HeaderMap, Method, StatusCode, Uri}; use axum::middleware::Next; use axum::response::{IntoResponse, Response}; use axum::routing::{any, get}; @@ -28,6 +29,7 @@ pub struct AppState { pub key_manager: Arc, pub dlp_scanner: Arc, pub proxy_client: Arc, + pub oauth_registry: Arc, pub email_enabled: bool, pub email_policy: Option, pub email_accounts: Arc>, @@ -35,7 +37,15 @@ pub struct AppState { } impl AppState { + #[allow(dead_code)] pub fn from_config(config: &Config) -> Result { + Self::from_config_with_registry(config, None) + } + + pub fn from_config_with_registry( + config: &Config, + oauth_registry: Option, + ) -> Result { let mut upstream_urls = BTreeMap::new(); upstream_urls.insert(Provider::Openai, config.upstream_url(Provider::Openai)); upstream_urls.insert( @@ -47,19 +57,29 @@ impl AppState { config.upstream_url(Provider::Anthropic), ); - let key_mappings = config - .key_map() - .iter() - .map(|(virtual_key, (real_key, provider))| { - ( - virtual_key.clone(), - ResolvedKey { - real_key: real_key.clone(), - provider: *provider, - }, - ) - }) - .collect(); + // Build key mappings for both static and OAuth keys + let mut key_mappings: BTreeMap = BTreeMap::new(); + + for key in &config.keys { + let source = match key.auth { + KeyAuthMethod::Static => KeySource::Static { + real_key: key.real_key.clone().unwrap_or_default(), + }, + KeyAuthMethod::OAuth => KeySource::OAuth { + provider_id: key.oauth_provider.clone().unwrap_or_default(), + }, + }; + key_mappings.insert( + key.virtual_key.clone(), + ResolvedKey { + source, + provider: key.provider, + }, + ); + } + + let oauth_registry = + oauth_registry.unwrap_or_else(|| OAuthRegistry::new(Default::default())); let email_policy = if config.email.enabled { config.email.mode.map(|mode| { @@ -112,6 +132,7 @@ impl AppState { upstream_urls, config.upstream.anthropic_version.clone(), )), + oauth_registry: Arc::new(oauth_registry), email_enabled: config.email.enabled, email_policy, email_accounts: Arc::new(email_accounts), @@ -493,7 +514,7 @@ async fn handle_request( ); error_response(StatusCode::UNAUTHORIZED, "Unknown API key") })?; - let real_key = resolved.real_key.clone(); + let source = resolved.source.clone(); let provider = resolved.provider; debug!( @@ -563,15 +584,36 @@ async fn handle_request( "Forwarding request to upstream" ); - let response = state - .proxy_client - .forward( + let response = match source { + KeySource::Static { real_key } => state + .proxy_client + .forward( + method.clone(), + &uri, + headers, + &real_key, + body_bytes, + provider, + ) + .await + .map_err(|e| { + error!( + method = %method, + path = %path, + virtual_key = %virtual_key, + error = %e, + "Proxy error" + ); + e.into_response() + })?, + KeySource::OAuth { provider_id } => forward_oauth_request( + &state, method.clone(), &uri, headers, - &real_key, body_bytes, provider, + &provider_id, ) .await .map_err(|e| { @@ -579,11 +621,13 @@ async fn handle_request( method = %method, path = %path, virtual_key = %virtual_key, + oauth_provider = %provider_id, error = %e, - "Proxy error" + "OAuth proxy error" ); - e.into_response() - })?; + error_response(StatusCode::BAD_GATEWAY, &format!("OAuth proxy error: {e}")) + })?, + }; // 5. DLP scan on response body (redact all PII before returning to client) let response = if state.dlp_scanner.scan_responses() { @@ -627,14 +671,16 @@ async fn handle_request( Response::from_parts(parts, Body::from(body)) } } else { - warn!( + debug!( method = %method, path = %path, virtual_key = %virtual_key, - "Streaming response (SSE) — DLP scanning is not supported for streaming responses; \ - PII in streamed content will not be redacted" + "Streaming response (SSE) — wrapping with DLP SSE scanner" ); - response + let (parts, body) = response.into_parts(); + let dlp_body = + crate::translate::wrap_body_with_dlp_sse_stream(body, state.dlp_scanner.clone()); + Response::from_parts(parts, dlp_body) } } else { trace!("Response DLP scanning disabled"); @@ -644,6 +690,276 @@ async fn handle_request( Ok(response) } +async fn forward_oauth_request( + state: &AppState, + method: Method, + uri: &Uri, + headers: HeaderMap, + body_bytes: Bytes, + provider: Provider, + oauth_provider_id: &str, +) -> Result { + // 1. Inject auth headers + let mut auth_headers = HeaderMap::new(); + state + .oauth_registry + .inject_auth(oauth_provider_id, &mut auth_headers) + .await + .map_err(|e| format!("OAuth auth injection failed: {e}"))?; + + // 2. Optionally transform the body + let body = match state + .oauth_registry + .prepare_request_body(oauth_provider_id, &body_bytes) + .await + .map_err(|e| format!("OAuth body preparation failed: {e}"))? + { + Some(transformed) => Bytes::from(transformed), + None => body_bytes.clone(), + }; + + // 2b. Check if path needs rewriting (e.g., /v1/chat/completions → /v1/responses) + let original_path = uri.path().to_string(); + let rewritten_path = state + .oauth_registry + .rewrite_request_path(oauth_provider_id, &original_path) + .map_err(|e| format!("OAuth path rewrite failed: {e}"))?; + let needs_translation = state + .oauth_registry + .needs_response_translation(oauth_provider_id, &original_path) + .map_err(|e| format!("OAuth translation check failed: {e}"))?; + let response_format = state + .oauth_registry + .response_format(oauth_provider_id, &original_path) + .map_err(|e| format!("OAuth response format check failed: {e}"))?; + // Check the transformed body for stream flag (fixups may force stream: true) + let stream_requested = serde_json::from_slice::(&body) + .ok() + .and_then(|v| v.get("stream")?.as_bool()) + .unwrap_or(false); + + let effective_uri = if let Some(ref new_path) = rewritten_path { + build_rewritten_uri(uri, new_path)? + } else { + uri.clone() + }; + + if rewritten_path.is_some() { + debug!( + oauth_provider = %oauth_provider_id, + original_path = %original_path, + effective_path = %effective_uri.path(), + "Rewrote request path for OAuth provider" + ); + } + + // 3. Optionally get upstream URL override + let upstream_url = state + .oauth_registry + .upstream_url(oauth_provider_id) + .await + .map_err(|e| format!("OAuth upstream URL resolution failed: {e}"))?; + + // 4. Forward the request + let response = state + .proxy_client + .forward_oauth( + method.clone(), + &effective_uri, + headers.clone(), + body.clone(), + provider, + auth_headers.clone(), + upstream_url.as_deref(), + ) + .await + .map_err(|e| format!("OAuth forward failed: {e}"))?; + + // 5. If we got a 401, refresh the token and retry once + if response.status() == StatusCode::UNAUTHORIZED { + info!( + oauth_provider = %oauth_provider_id, + effective_path = %effective_uri.path(), + "Got 401 from upstream, attempting token refresh and retry" + ); + if let Err(e) = state.oauth_registry.refresh(oauth_provider_id).await { + warn!( + oauth_provider = %oauth_provider_id, + error = %e, + "Token refresh failed after 401" + ); + return maybe_translate_response( + response, + needs_translation, + stream_requested, + response_format, + ) + .await; + } + + // Re-inject auth with refreshed token + let mut retry_auth_headers = HeaderMap::new(); + state + .oauth_registry + .inject_auth(oauth_provider_id, &mut retry_auth_headers) + .await + .map_err(|e| format!("OAuth retry auth injection failed: {e}"))?; + + // Optionally re-transform the body (tokens may have changed affecting body) + let retry_body = match state + .oauth_registry + .prepare_request_body(oauth_provider_id, &body_bytes) + .await + .map_err(|e| format!("OAuth retry body preparation failed: {e}"))? + { + Some(transformed) => Bytes::from(transformed), + None => body_bytes, + }; + + let retry_response = state + .proxy_client + .forward_oauth( + method, + &effective_uri, + headers, + retry_body, + provider, + retry_auth_headers, + upstream_url.as_deref(), + ) + .await + .map_err(|e| format!("OAuth retry forward failed: {e}"))?; + + if retry_response.status() == StatusCode::UNAUTHORIZED { + warn!( + oauth_provider = %oauth_provider_id, + effective_path = %effective_uri.path(), + "Retry after token refresh still returned 401" + ); + } + + return maybe_translate_response( + retry_response, + needs_translation, + stream_requested, + response_format, + ) + .await; + } + + // Log error response bodies for debugging upstream issues + if response.status().is_client_error() || response.status().is_server_error() { + let status = response.status(); + let (parts, body) = response.into_parts(); + let body_bytes_resp = body + .collect() + .await + .map(|b| b.to_bytes()) + .unwrap_or_default(); + if let Ok(body_str) = std::str::from_utf8(&body_bytes_resp) { + warn!( + oauth_provider = %oauth_provider_id, + effective_path = %effective_uri.path(), + status = %status, + response_body = %body_str, + "Upstream returned error" + ); + } + let response = Response::from_parts(parts, Body::from(body_bytes_resp)); + return maybe_translate_response( + response, + needs_translation, + stream_requested, + response_format, + ) + .await; + } + + maybe_translate_response( + response, + needs_translation, + stream_requested, + response_format, + ) + .await +} + +/// Optionally translate an upstream response back to chat/completions format. +async fn maybe_translate_response( + response: Response, + needs_translation: bool, + stream_requested: bool, + response_format: Option, +) -> Result { + // Use response_format if available; fall back to needs_translation for backwards compat + let format = match response_format { + Some(f) => f, + None if needs_translation => crate::oauth::ResponseFormat::ResponsesApi, + None => return Ok(response), + }; + + let is_streaming = stream_requested + || response + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .is_some_and(|ct| ct.contains("text/event-stream")); + + debug!(format = ?format, is_streaming, "maybe_translate_response: translating response"); + + if is_streaming { + let (parts, body) = response.into_parts(); + let translated_body = match format { + crate::oauth::ResponseFormat::ResponsesApi => { + debug!("Wrapping streaming response with ResponsesApi translator"); + crate::translate::wrap_body_with_translate_stream(body) + } + }; + return Ok(Response::from_parts(parts, translated_body)); + } + + // Non-streaming: only translate successful responses + let status = response.status(); + if !status.is_success() { + return Ok(response); + } + + let (mut parts, body) = response.into_parts(); + let body_bytes = body + .collect() + .await + .map_err(|e| format!("failed to read response body for translation: {e}"))? + .to_bytes(); + + match format { + crate::oauth::ResponseFormat::ResponsesApi => { + match crate::translate::responses_to_chat_completion(&body_bytes) { + Ok(translated) => { + parts.headers.remove("content-length"); + Ok(Response::from_parts(parts, Body::from(translated))) + } + Err(e) => { + warn!(error = %e, "Response translation failed, returning original"); + Ok(Response::from_parts(parts, Body::from(body_bytes))) + } + } + } + } +} + +/// Build a new URI with a rewritten path, preserving query string. +/// Incoming axum URIs are path-only (no scheme/authority), so we build path-only too. +fn build_rewritten_uri(original: &Uri, new_path: &str) -> Result { + let path_and_query = if let Some(query) = original.query() { + format!("{new_path}?{query}") + } else { + new_path.to_string() + }; + path_and_query + .parse::() + .map_err(|e| format!("failed to build rewritten URI: {e}")) +} + fn error_response(status: StatusCode, message: &str) -> Response { let body = serde_json::json!({ "error": message }); (status, axum::Json(body)).into_response() diff --git a/src/app/tests.rs b/src/app/tests.rs index f143ffc..2a9fa26 100644 --- a/src/app/tests.rs +++ b/src/app/tests.rs @@ -16,7 +16,8 @@ use crate::email::{ EmailAccountCredentials, EmailListMessagesResponse, EmailMessageContent, EmailMessageMetadata, EmailPolicy, EmailService, }; -use crate::keys::{KeyManager, ResolvedKey}; +use crate::keys::{KeyManager, KeySource, ResolvedKey}; +use crate::oauth::OAuthRegistry; use crate::proxy::ProxyClient; fn make_app(upstream_url: &str) -> axum::Router { @@ -24,14 +25,18 @@ fn make_app(upstream_url: &str) -> axum::Router { key_map.insert( "vk-test-1".to_string(), ResolvedKey { - real_key: "sk-real-1".to_string(), + source: KeySource::Static { + real_key: "sk-real-1".to_string(), + }, provider: Provider::Openai, }, ); key_map.insert( "vk-test-2".to_string(), ResolvedKey { - real_key: "sk-real-2".to_string(), + source: KeySource::Static { + real_key: "sk-real-2".to_string(), + }, provider: Provider::Openai, }, ); @@ -65,6 +70,7 @@ fn make_app(upstream_url: &str) -> axum::Router { upstream_urls, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: false, email_policy: None, email_accounts: Arc::new(BTreeMap::new()), @@ -79,14 +85,18 @@ fn make_app_with_anthropic(upstream_url: &str) -> axum::Router { key_map.insert( "vk-test-1".to_string(), ResolvedKey { - real_key: "sk-real-1".to_string(), + source: KeySource::Static { + real_key: "sk-real-1".to_string(), + }, provider: Provider::Openai, }, ); key_map.insert( "vk-ant-1".to_string(), ResolvedKey { - real_key: "sk-ant-real-1".to_string(), + source: KeySource::Static { + real_key: "sk-ant-real-1".to_string(), + }, provider: Provider::Anthropic, }, ); @@ -102,6 +112,7 @@ fn make_app_with_anthropic(upstream_url: &str) -> axum::Router { upstream_urls, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: false, email_policy: None, email_accounts: Arc::new(BTreeMap::new()), @@ -561,7 +572,10 @@ real_key = "sk-real-1" let config = Config::parse(toml_str).unwrap(); let state = AppState::from_config(&config).unwrap(); let resolved = state.key_manager.resolve("vk-1").unwrap(); - assert_eq!(resolved.real_key, "sk-real-1"); + match &resolved.source { + KeySource::Static { real_key } => assert_eq!(real_key, "sk-real-1"), + _ => panic!("expected Static key source"), + } assert_eq!(resolved.provider, Provider::Openai); assert!(state.key_manager.resolve("vk-unknown").is_none()); } @@ -587,10 +601,16 @@ provider = "anthropic" let config = Config::parse(toml_str).unwrap(); let state = AppState::from_config(&config).unwrap(); let oai = state.key_manager.resolve("vk-oai").unwrap(); - assert_eq!(oai.real_key, "sk-oai-key"); + match &oai.source { + KeySource::Static { real_key } => assert_eq!(real_key, "sk-oai-key"), + _ => panic!("expected Static key source"), + } assert_eq!(oai.provider, Provider::Openai); let ant = state.key_manager.resolve("vk-ant").unwrap(); - assert_eq!(ant.real_key, "sk-ant-key"); + match &ant.source { + KeySource::Static { real_key } => assert_eq!(real_key, "sk-ant-key"), + _ => panic!("expected Static key source"), + } assert_eq!(ant.provider, Provider::Anthropic); } @@ -664,7 +684,9 @@ async fn test_proxy_error_on_unreachable_upstream() { [( "vk-1".to_string(), ResolvedKey { - real_key: "sk-1".to_string(), + source: KeySource::Static { + real_key: "sk-1".to_string(), + }, provider: Provider::Openai, }, )] @@ -681,6 +703,7 @@ async fn test_proxy_error_on_unreachable_upstream() { }, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: false, email_policy: None, email_accounts: Arc::new(BTreeMap::new()), @@ -803,7 +826,9 @@ async fn test_anthropic_dlp_blocks_sensitive_data() { key_map.insert( "vk-ant-dlp".to_string(), ResolvedKey { - real_key: "sk-ant-key".to_string(), + source: KeySource::Static { + real_key: "sk-ant-key".to_string(), + }, provider: Provider::Anthropic, }, ); @@ -825,6 +850,7 @@ async fn test_anthropic_dlp_blocks_sensitive_data() { upstream_urls, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: false, email_policy: None, email_accounts: Arc::new(BTreeMap::new()), @@ -898,14 +924,18 @@ async fn test_openai_and_openrouter_keys_map_to_distinct_real_keys() { key_map.insert( "vk-openai".to_string(), ResolvedKey { - real_key: "sk-openai-real".to_string(), + source: KeySource::Static { + real_key: "sk-openai-real".to_string(), + }, provider: Provider::Openai, }, ); key_map.insert( "vk-openrouter".to_string(), ResolvedKey { - real_key: "sk-openrouter-real".to_string(), + source: KeySource::Static { + real_key: "sk-openrouter-real".to_string(), + }, provider: Provider::Openrouter, }, ); @@ -922,6 +952,7 @@ async fn test_openai_and_openrouter_keys_map_to_distinct_real_keys() { upstream_urls, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: false, email_policy: None, email_accounts: Arc::new(BTreeMap::new()), @@ -956,7 +987,9 @@ fn make_app_with_redact(upstream_url: &str) -> axum::Router { key_map.insert( "vk-test-1".to_string(), ResolvedKey { - real_key: "sk-real-1".to_string(), + source: KeySource::Static { + real_key: "sk-real-1".to_string(), + }, provider: Provider::Openai, }, ); @@ -990,6 +1023,7 @@ fn make_app_with_redact(upstream_url: &str) -> axum::Router { upstream_urls, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: false, email_policy: None, email_accounts: Arc::new(BTreeMap::new()), @@ -1175,7 +1209,9 @@ async fn test_response_dlp_disabled() { key_map.insert( "vk-test-1".to_string(), ResolvedKey { - real_key: "sk-real-1".to_string(), + source: KeySource::Static { + real_key: "sk-real-1".to_string(), + }, provider: Provider::Openai, }, ); @@ -1194,6 +1230,7 @@ async fn test_response_dlp_disabled() { upstream_urls, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: false, email_policy: None, email_accounts: Arc::new(BTreeMap::new()), @@ -1442,8 +1479,7 @@ async fn test_non_utf8_body_passes_through() { async fn test_streaming_response_with_dlp_enabled_passes_through() { let mock_server = MockServer::start().await; - // SSE response — should pass through when DLP scanning is enabled - // because streaming responses cannot be scanned (exercises lib.rs lines 261-268) + // SSE response with clean content — should pass through DLP scanning unchanged let sse_body = "data: {\"content\":\"hello world\"}\n\ndata: [DONE]\n\n"; Mock::given(method("POST")) .and(path("/v1/chat/completions")) @@ -1484,6 +1520,63 @@ async fn test_streaming_response_with_dlp_enabled_passes_through() { assert!(body_str.contains("[DONE]")); } +#[tokio::test] +async fn test_streaming_response_dlp_redacts_pii_in_sse() { + let mock_server = MockServer::start().await; + + // SSE response with PII in delta.content — DLP should redact it + let chunk = serde_json::json!({ + "id": "chatcmpl-1", + "object": "chat.completion.chunk", + "choices": [{ + "index": 0, + "delta": { "content": "Contact user@example.com for help" }, + "finish_reason": null, + }] + }); + let sse_body = format!( + "data: {}\n\ndata: [DONE]\n\n", + serde_json::to_string(&chunk).unwrap() + ); + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_body, "text/event-stream"), + ) + .mount(&mock_server) + .await; + + let app = make_app_with_redact(&mock_server.uri()); + let body = r#"{"model":"gpt-4","stream":true,"messages":[{"role":"user","content":"Hi"}]}"#; + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("authorization", "Bearer vk-test-1") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = resp.into_body().collect().await.unwrap().to_bytes(); + let body_str = std::str::from_utf8(&body).unwrap(); + assert!( + body_str.contains("[REDACTED:email]"), + "PII should be redacted in streaming SSE" + ); + assert!( + !body_str.contains("user@example.com"), + "Original email should be gone" + ); + assert!( + body_str.contains("[DONE]"), + "Stream should still end with [DONE]" + ); +} + fn make_email_app( policy: EmailPolicy, email_accounts: BTreeMap, @@ -1500,6 +1593,7 @@ fn make_email_app( upstream_urls, "2023-06-01".to_string(), )), + oauth_registry: Arc::new(OAuthRegistry::new(Default::default())), email_enabled: true, email_policy: Some(policy), email_accounts: Arc::new(email_accounts), diff --git a/src/config.rs b/src/config.rs index 6c619ca..024031b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,7 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +use std::env::VarError; use std::path::Path; #[derive(Debug, Default, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -37,6 +38,8 @@ pub struct Config { pub email: EmailConfig, #[serde(default = "default_log_level")] pub log_level: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub oauth_providers: Vec, } fn default_log_level() -> String { @@ -60,6 +63,9 @@ fn default_port() -> u16 { 18790 } +const SERVER_HOST_ENV: &str = "CLAWSHELL_SERVER_HOST"; +const SERVER_PORT_ENV: &str = "CLAWSHELL_SERVER_PORT"; + #[derive(Debug, Deserialize, Serialize, Clone)] #[serde(deny_unknown_fields)] pub struct UpstreamConfig { @@ -81,13 +87,33 @@ fn default_openai_base_url() -> String { "https://api.openai.com".to_string() } +/// How a key mapping authenticates: static API key or OAuth provider. +#[derive(Debug, Default, Deserialize, Serialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum KeyAuthMethod { + /// Static API key (the default, existing behavior). + #[default] + Static, + /// OAuth provider supplies the access token at runtime. + OAuth, +} + #[derive(Debug, Deserialize, Serialize, Clone)] #[serde(deny_unknown_fields)] pub struct KeyMapping { pub virtual_key: String, - pub real_key: String, + /// Required when auth = "static" (or omitted). Optional when auth = "oauth". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub real_key: Option, #[serde(default)] pub provider: Provider, + /// Authentication method for this key. Defaults to "static". + #[serde(default)] + pub auth: KeyAuthMethod, + /// Which OAuth provider supplies the token (e.g. "codex"). + /// Required when auth = "oauth". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oauth_provider: Option, } #[derive(Debug, Default, Deserialize, Serialize, Clone, Copy, PartialEq, Eq)] @@ -220,10 +246,54 @@ impl Config { Regex::new(&pattern.regex) .map_err(|e| format!("Invalid DLP regex for '{}': {}", pattern.name, e))?; } + self.validate_keys()?; self.validate_email()?; Ok(()) } + fn validate_keys(&self) -> Result<(), Box> { + for key in &self.keys { + match key.auth { + KeyAuthMethod::Static => { + if key.real_key.is_none() { + return Err(format!( + "key '{}': real_key is required when auth = \"static\"", + key.virtual_key + ) + .into()); + } + } + KeyAuthMethod::OAuth => { + if key + .oauth_provider + .as_ref() + .is_none_or(|p| p.trim().is_empty()) + { + return Err(format!( + "key '{}': oauth_provider is required when auth = \"oauth\"", + key.virtual_key + ) + .into()); + } + // Verify the referenced OAuth provider exists in config + let provider_id = key.oauth_provider.as_ref().unwrap(); + if !self + .oauth_providers + .iter() + .any(|p| p.provider == *provider_id) + { + return Err(format!( + "key '{}': oauth_provider '{}' not found in [[oauth_providers]]", + key.virtual_key, provider_id + ) + .into()); + } + } + } + } + Ok(()) + } + fn validate_email(&self) -> Result<(), Box> { let email = &self.email; @@ -315,10 +385,32 @@ impl Config { Ok(()) } + /// Returns a map of static key mappings: virtual_key → (real_key, provider). + /// OAuth-backed keys are excluded. + #[allow(dead_code)] pub fn key_map(&self) -> BTreeMap { self.keys .iter() - .map(|k| (k.virtual_key.clone(), (k.real_key.clone(), k.provider))) + .filter(|k| k.auth == KeyAuthMethod::Static) + .filter_map(|k| { + k.real_key + .clone() + .map(|rk| (k.virtual_key.clone(), (rk, k.provider))) + }) + .collect() + } + + /// Returns a map of OAuth key mappings: virtual_key → (oauth_provider_id, provider). + #[allow(dead_code)] + pub fn oauth_key_map(&self) -> BTreeMap { + self.keys + .iter() + .filter(|k| k.auth == KeyAuthMethod::OAuth) + .filter_map(|k| { + k.oauth_provider + .clone() + .map(|op| (k.virtual_key.clone(), (op, k.provider))) + }) .collect() } @@ -341,6 +433,60 @@ impl Config { pub fn listen_addr(&self) -> String { format!("{}:{}", self.server.host, self.server.port) } + + pub fn resolved_listen_addr(&self) -> Result> { + let host = resolve_server_host_override(&self.server.host)?; + let port = resolve_server_port_override(self.server.port)?; + Ok(format!("{host}:{port}")) + } +} + +fn resolve_server_host_override(default_host: &str) -> Result> { + resolve_server_host_override_from_var(default_host, std::env::var(SERVER_HOST_ENV)) +} + +fn resolve_server_host_override_from_var( + default_host: &str, + env_value: Result, +) -> Result> { + match env_value { + Ok(value) => { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Err(format!("{SERVER_HOST_ENV} cannot be empty").into()); + } + Ok(trimmed.to_string()) + } + Err(VarError::NotPresent) => Ok(default_host.to_string()), + Err(VarError::NotUnicode(_)) => { + Err(format!("{SERVER_HOST_ENV} must be valid UTF-8").into()) + } + } +} + +fn resolve_server_port_override(default_port: u16) -> Result> { + resolve_server_port_override_from_var(default_port, std::env::var(SERVER_PORT_ENV)) +} + +fn resolve_server_port_override_from_var( + default_port: u16, + env_value: Result, +) -> Result> { + match env_value { + Ok(value) => { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Err(format!("{SERVER_PORT_ENV} cannot be empty").into()); + } + trimmed.parse::().map_err(|_| { + format!("{SERVER_PORT_ENV} must be a valid port (0-65535), got '{trimmed}'").into() + }) + } + Err(VarError::NotPresent) => Ok(default_port), + Err(VarError::NotUnicode(_)) => { + Err(format!("{SERVER_PORT_ENV} must be valid UTF-8").into()) + } + } } pub(crate) fn validate_sender_rule(rule: &str) -> Result<(), String> { @@ -850,4 +996,64 @@ imap_port = 0 .contains("email.accounts[].imap_port must be greater than 0") ); } + + #[test] + fn test_resolved_listen_addr_uses_config_without_env() { + let cfg = r#" +[server] +host = "127.0.0.1" +port = 3000 + +[upstream] +openai_base_url = "https://api.openai.com" +"#; + let parsed = Config::parse(cfg).expect("config should parse"); + assert_eq!(parsed.listen_addr(), "127.0.0.1:3000"); + } + + #[test] + fn test_resolve_server_host_override_uses_default_when_unset() { + let host = resolve_server_host_override_from_var("127.0.0.1", Err(VarError::NotPresent)) + .expect("host should use default"); + assert_eq!(host, "127.0.0.1"); + } + + #[test] + fn test_resolve_server_host_override_accepts_env() { + let host = resolve_server_host_override_from_var("127.0.0.1", Ok("0.0.0.0".to_string())) + .expect("host override should be accepted"); + assert_eq!(host, "0.0.0.0"); + } + + #[test] + fn test_resolve_server_host_override_rejects_empty_env() { + let err = + resolve_server_host_override_from_var("127.0.0.1", Ok(" ".to_string())).unwrap_err(); + assert!( + err.to_string() + .contains("CLAWSHELL_SERVER_HOST cannot be empty") + ); + } + + #[test] + fn test_resolve_server_port_override_uses_default_when_unset() { + let port = resolve_server_port_override_from_var(3000, Err(VarError::NotPresent)).unwrap(); + assert_eq!(port, 3000); + } + + #[test] + fn test_resolve_server_port_override_accepts_env() { + let port = resolve_server_port_override_from_var(3000, Ok("17890".to_string())).unwrap(); + assert_eq!(port, 17890); + } + + #[test] + fn test_resolve_server_port_override_rejects_invalid_env() { + let err = + resolve_server_port_override_from_var(3000, Ok("not-a-port".to_string())).unwrap_err(); + assert!( + err.to_string() + .contains("CLAWSHELL_SERVER_PORT must be a valid port") + ); + } } diff --git a/src/keys.rs b/src/keys.rs index 1610cb7..b64beee 100644 --- a/src/keys.rs +++ b/src/keys.rs @@ -3,9 +3,15 @@ use crate::config::Provider; use std::collections::BTreeMap; use tracing::{debug, trace}; +#[derive(Debug, Clone)] +pub enum KeySource { + Static { real_key: String }, + OAuth { provider_id: String }, +} + #[derive(Debug, Clone)] pub struct ResolvedKey { - pub real_key: String, + pub source: KeySource, pub provider: Provider, } @@ -58,14 +64,16 @@ impl KeyManager { mod tests { use super::*; - fn make_map(entries: Vec<(&str, &str, Provider)>) -> BTreeMap { + fn make_static_map(entries: Vec<(&str, &str, Provider)>) -> BTreeMap { entries .into_iter() .map(|(vk, rk, p)| { ( vk.to_string(), ResolvedKey { - real_key: rk.to_string(), + source: KeySource::Static { + real_key: rk.to_string(), + }, provider: p, }, ) @@ -93,10 +101,13 @@ mod tests { #[test] fn test_resolve_existing_key() { - let map = make_map(vec![("vk-1", "sk-real-1", Provider::Openai)]); + let map = make_static_map(vec![("vk-1", "sk-real-1", Provider::Openai)]); let km = KeyManager::new(map); let resolved = km.resolve("vk-1").unwrap(); - assert_eq!(resolved.real_key, "sk-real-1"); + match &resolved.source { + KeySource::Static { real_key } => assert_eq!(real_key, "sk-real-1"), + KeySource::OAuth { .. } => panic!("expected Static"), + } assert_eq!(resolved.provider, Provider::Openai); } @@ -108,21 +119,51 @@ mod tests { #[test] fn test_multiple_virtual_to_same_real() { - let map = make_map(vec![ + let map = make_static_map(vec![ ("vk-1", "sk-shared", Provider::Openai), ("vk-2", "sk-shared", Provider::Openai), ]); let km = KeyManager::new(map); - assert_eq!(km.resolve("vk-1").unwrap().real_key, "sk-shared"); - assert_eq!(km.resolve("vk-2").unwrap().real_key, "sk-shared"); + match &km.resolve("vk-1").unwrap().source { + KeySource::Static { real_key } => assert_eq!(real_key, "sk-shared"), + _ => panic!("expected Static"), + } + match &km.resolve("vk-2").unwrap().source { + KeySource::Static { real_key } => assert_eq!(real_key, "sk-shared"), + _ => panic!("expected Static"), + } } #[test] fn test_resolve_anthropic_provider() { - let map = make_map(vec![("vk-ant", "sk-ant-key", Provider::Anthropic)]); + let map = make_static_map(vec![("vk-ant", "sk-ant-key", Provider::Anthropic)]); let km = KeyManager::new(map); let resolved = km.resolve("vk-ant").unwrap(); - assert_eq!(resolved.real_key, "sk-ant-key"); + match &resolved.source { + KeySource::Static { real_key } => assert_eq!(real_key, "sk-ant-key"), + _ => panic!("expected Static"), + } assert_eq!(resolved.provider, Provider::Anthropic); } + + #[test] + fn test_resolve_oauth_key() { + let mut map = BTreeMap::new(); + map.insert( + "vk-oauth".to_string(), + ResolvedKey { + source: KeySource::OAuth { + provider_id: "codex".to_string(), + }, + provider: Provider::Openai, + }, + ); + let km = KeyManager::new(map); + let resolved = km.resolve("vk-oauth").unwrap(); + match &resolved.source { + KeySource::OAuth { provider_id } => assert_eq!(provider_id, "codex"), + _ => panic!("expected OAuth"), + } + assert_eq!(resolved.provider, Provider::Openai); + } } diff --git a/src/main.rs b/src/main.rs index 5e716d6..b75cb7c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,11 +9,14 @@ mod dlp; mod email; mod keys; mod migration; +#[allow(dead_code)] +mod oauth; mod onboard; mod openclaw_cli; mod platform; mod process; mod proxy; +mod translate; mod tui; use clap::Parser; @@ -555,7 +558,10 @@ async fn cmd_start_inner(config_path: &str) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result> { + use crate::oauth::{OAuthRegistry, TokenStorage, codex::CodexProvider}; + use std::sync::Arc; + + let storage = TokenStorage::new(PathBuf::from("/etc/clawshell/oauth")); + let mut registry = OAuthRegistry::new(storage); + + for provider_config in &config.oauth_providers { + if !provider_config.enabled { + continue; + } + match provider_config.provider.as_str() { + "codex" => { + let provider = CodexProvider::from_config(provider_config); + registry.register(Arc::new(provider)); + } + other => { + return Err(format!("Unknown OAuth provider type: '{other}'").into()); + } + } + } + + // Load persisted tokens from disk + registry.load_tokens().await?; + + Ok(registry) +} + fn cmd_stop() -> Result<(), Box> { tui::print_banner("Stop"); ensure_default_config_migrated_if_present()?; @@ -1051,13 +1097,27 @@ fn cmd_onboard() -> Result<(), Box> { let toml_content = onboard::generate_clawshell_config(&ob_config); std::fs::write(&toml_config_path, &toml_content)?; - let config_json = serde_json::json!({ - "real_api_key": ob_config.real_api_key, - "virtual_api_key": ob_config.virtual_api_key, - "provider": ob_config.provider, - "model": ob_config.model, - "openclaw_config_path": ob_config.openclaw_config_path.to_string_lossy(), - }); + let config_json = match &ob_config.auth_method { + crate::onboard::OnboardAuthMethod::OAuth { provider_id } => { + serde_json::json!({ + "auth_method": "oauth", + "oauth_provider": provider_id, + "virtual_api_key": ob_config.virtual_api_key, + "provider": ob_config.provider, + "model": ob_config.model, + "openclaw_config_path": ob_config.openclaw_config_path.to_string_lossy(), + }) + } + crate::onboard::OnboardAuthMethod::StaticKey => { + serde_json::json!({ + "real_api_key": ob_config.real_api_key, + "virtual_api_key": ob_config.virtual_api_key, + "provider": ob_config.provider, + "model": ob_config.model, + "openclaw_config_path": ob_config.openclaw_config_path.to_string_lossy(), + }) + } + }; std::fs::write(&config_file, serde_json::to_string_pretty(&config_json)?)?; // Set permissions on config files diff --git a/src/oauth/codex.rs b/src/oauth/codex.rs new file mode 100644 index 0000000..4613b6b --- /dev/null +++ b/src/oauth/codex.rs @@ -0,0 +1,697 @@ +use super::{OAuthError, OAuthProvider, OAuthTokens}; +use async_trait::async_trait; +use axum::http::HeaderMap; +use axum::http::header::AUTHORIZATION; +use chrono::Utc; +use std::collections::BTreeMap; +use tracing::{debug, info}; + +const DEFAULT_CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; +const DEFAULT_AUTH_URL: &str = "https://auth.openai.com/authorize"; +const DEFAULT_TOKEN_URL: &str = "https://auth.openai.com/oauth/token"; +const DEFAULT_SCOPES: &[&str] = &["openid", "profile", "email", "offline_access"]; + +#[derive(Debug)] +pub struct CodexProvider { + client_id: String, + auth_url: String, + token_url: String, + scopes: Vec, + http_client: reqwest::Client, +} + +impl CodexProvider { + pub fn new( + client_id: Option<&str>, + auth_url: Option<&str>, + token_url: Option<&str>, + scopes: Option<&[String]>, + ) -> Self { + Self { + client_id: client_id.unwrap_or(DEFAULT_CLIENT_ID).to_string(), + auth_url: auth_url.unwrap_or(DEFAULT_AUTH_URL).to_string(), + token_url: token_url.unwrap_or(DEFAULT_TOKEN_URL).to_string(), + scopes: scopes + .map(|s| s.to_vec()) + .unwrap_or_else(|| DEFAULT_SCOPES.iter().map(|s| s.to_string()).collect()), + http_client: reqwest::Client::builder() + .user_agent(format!( + "ClawShell/{} (https://github.com/nicholasgasior/clawshell)", + env!("CARGO_PKG_VERSION") + )) + .build() + .expect("failed to build HTTP client"), + } + } + + pub fn from_config(config: &super::OAuthProviderConfig) -> Self { + Self::new( + config.client_id.as_deref(), + config.auth_url.as_deref(), + config.token_url.as_deref(), + config.scopes.as_deref(), + ) + } + + async fn exchange_code( + &self, + code: &str, + code_verifier: &str, + redirect_uri: &str, + ) -> Result { + let params = [ + ("grant_type", "authorization_code"), + ("client_id", &self.client_id), + ("code", code), + ("code_verifier", code_verifier), + ("redirect_uri", redirect_uri), + ]; + + let resp = self + .http_client + .post(&self.token_url) + .form(¶ms) + .send() + .await?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(OAuthError::LoginFailed(format!( + "token exchange failed ({status}): {body}" + ))); + } + + let json: serde_json::Value = resp.json().await?; + parse_token_response(&json) + } + + async fn exchange_refresh_token(&self, refresh_token: &str) -> Result { + let params = [ + ("grant_type", "refresh_token"), + ("client_id", &self.client_id), + ("refresh_token", refresh_token), + ]; + + let resp = self + .http_client + .post(&self.token_url) + .form(¶ms) + .send() + .await?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(OAuthError::RefreshFailed(format!( + "refresh failed ({status}): {body}" + ))); + } + + let json: serde_json::Value = resp.json().await?; + parse_token_response(&json) + } + + /// Poll OpenAI's custom device-auth token endpoint until user authorises. + /// Returns (authorization_code, code_verifier) on success. + async fn poll_device_auth( + &self, + device_auth_id: &str, + user_code: &str, + interval: u64, + ) -> Result<(String, String), OAuthError> { + let url = self.device_auth_base_url() + "/token"; + let max_wait = std::time::Duration::from_secs(15 * 60); + let start = std::time::Instant::now(); + + loop { + tokio::time::sleep(std::time::Duration::from_secs(interval)).await; + + if start.elapsed() > max_wait { + return Err(OAuthError::LoginFailed( + "device code polling timed out (15 min)".to_string(), + )); + } + + let body = serde_json::json!({ + "device_auth_id": device_auth_id, + "user_code": user_code, + }); + + let resp = self.http_client.post(&url).json(&body).send().await?; + + if resp.status().is_success() { + let json: serde_json::Value = resp.json().await?; + let auth_code = json + .get("authorization_code") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + OAuthError::LoginFailed( + "missing authorization_code in device-auth response".to_string(), + ) + })? + .to_string(); + let code_verifier = json + .get("code_verifier") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + OAuthError::LoginFailed( + "missing code_verifier in device-auth response".to_string(), + ) + })? + .to_string(); + return Ok((auth_code, code_verifier)); + } + + // 403 / 404 = authorization still pending + let status = resp.status(); + if status == reqwest::StatusCode::FORBIDDEN || status == reqwest::StatusCode::NOT_FOUND + { + debug!("Device code authorization pending ({status})"); + continue; + } + + let text = resp.text().await.unwrap_or_default(); + return Err(OAuthError::LoginFailed(format!( + "device-auth polling failed ({status}): {text}" + ))); + } + } + + /// Base URL for OpenAI's custom device-auth API, derived from `auth_url`. + fn device_auth_base_url(&self) -> String { + // auth_url is e.g. "https://auth.openai.com/authorize" + // We need "https://auth.openai.com/api/accounts/deviceauth" + let base = self + .auth_url + .trim_end_matches("/authorize") + .trim_end_matches('/'); + format!("{base}/api/accounts/deviceauth") + } +} + +fn parse_token_response(json: &serde_json::Value) -> Result { + let access_token = json + .get("access_token") + .and_then(|v| v.as_str()) + .ok_or_else(|| OAuthError::LoginFailed("missing access_token in response".to_string()))? + .to_string(); + + let refresh_token = json + .get("refresh_token") + .and_then(|v| v.as_str()) + .map(String::from); + + let id_token = json + .get("id_token") + .and_then(|v| v.as_str()) + .map(String::from); + + let expires_at = json + .get("expires_in") + .and_then(|v| v.as_i64()) + .map(|secs| Utc::now() + chrono::Duration::seconds(secs)); + + Ok(OAuthTokens { + access_token, + refresh_token, + id_token, + expires_at, + account_id: None, + extra: BTreeMap::new(), + }) +} + +fn generate_pkce() -> (String, String) { + use base64::Engine; + use sha2::{Digest, Sha256}; + + let verifier_bytes: [u8; 32] = rand::random(); + let verifier = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(verifier_bytes); + + let mut hasher = Sha256::new(); + hasher.update(verifier.as_bytes()); + let challenge = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(hasher.finalize()); + + (verifier, challenge) +} + +#[async_trait] +impl OAuthProvider for CodexProvider { + fn id(&self) -> &str { + "codex" + } + + fn display_name(&self) -> &str { + "Codex / ChatGPT (OAuth)" + } + + fn supports_device_code(&self) -> bool { + true + } + + async fn login_browser(&self, callback_port: u16) -> Result { + let (verifier, challenge) = generate_pkce(); + let redirect_uri = format!("http://localhost:{callback_port}/auth/callback"); + let state: String = uuid::Uuid::new_v4().to_string(); + + let auth_url = format!( + "{}?response_type=code&client_id={}&redirect_uri={}&scope={}&code_challenge={}&code_challenge_method=S256&state={}", + self.auth_url, + urlencoding::encode(&self.client_id), + urlencoding::encode(&redirect_uri), + urlencoding::encode(&self.scopes.join(" ")), + urlencoding::encode(&challenge), + urlencoding::encode(&state), + ); + + info!("Opening browser for Codex OAuth login"); + if let Err(e) = open::that(&auth_url) { + return Err(OAuthError::LoginFailed(format!( + "failed to open browser: {e}. Visit this URL manually: {auth_url}" + ))); + } + + // Start a temporary HTTP server to receive the callback + let (code, received_state) = wait_for_oauth_callback(callback_port) + .await + .map_err(|e| OAuthError::LoginFailed(format!("callback server failed: {e}")))?; + + if received_state != state { + return Err(OAuthError::LoginFailed( + "OAuth state mismatch — possible CSRF".to_string(), + )); + } + + self.exchange_code(&code, &verifier, &redirect_uri).await + } + + async fn login_headless(&self) -> Result { + // Step 1: Request a user code from OpenAI's device-auth endpoint + let usercode_url = self.device_auth_base_url() + "/usercode"; + let body = serde_json::json!({ "client_id": self.client_id }); + + let resp = self + .http_client + .post(&usercode_url) + .json(&body) + .send() + .await?; + + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + return Err(OAuthError::LoginFailed(format!( + "device code request failed ({status}): {text}" + ))); + } + + let json: serde_json::Value = resp.json().await?; + + let device_auth_id = json + .get("device_auth_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + OAuthError::LoginFailed("missing device_auth_id in response".to_string()) + })?; + + let user_code = json + .get("user_code") + .or_else(|| json.get("usercode")) + .and_then(|v| v.as_str()) + .ok_or_else(|| OAuthError::LoginFailed("missing user_code in response".to_string()))?; + + let interval = json + .get("interval") + .and_then(|v| { + v.as_u64() + .or_else(|| v.as_str().and_then(|s| s.parse().ok())) + }) + .unwrap_or(5); + + // Verification URL for the user + let base = self + .auth_url + .trim_end_matches("/authorize") + .trim_end_matches('/'); + let verification_url = format!("{base}/codex/device"); + + println!(); + println!(" Visit: {verification_url}"); + println!(" Enter code: {user_code}"); + println!(); + + // Step 2: Poll until user authorises, get authorization_code + code_verifier + let (auth_code, code_verifier) = self + .poll_device_auth(device_auth_id, user_code, interval) + .await?; + + // Step 3: Exchange authorization_code for tokens via the standard token endpoint + let redirect_uri = format!("{base}/deviceauth/callback"); + self.exchange_code(&auth_code, &code_verifier, &redirect_uri) + .await + } + + async fn refresh(&self, refresh_token: &str) -> Result { + self.exchange_refresh_token(refresh_token).await + } + + fn inject_auth(&self, headers: &mut HeaderMap, access_token: &str) -> Result<(), OAuthError> { + headers.insert(AUTHORIZATION, format!("Bearer {access_token}").parse()?); + // ChatGPT backend requires Accept header for SSE streaming + headers.insert( + axum::http::header::ACCEPT, + "text/event-stream".parse().unwrap(), + ); + Ok(()) + } + + fn prepare_request_body( + &self, + body: &[u8], + _tokens: &OAuthTokens, + ) -> Result>, OAuthError> { + // Only translate if the body is JSON with a "messages" field + let Ok(parsed) = serde_json::from_slice::(body) else { + return Ok(None); + }; + if parsed.get("messages").is_none() { + return Ok(None); + } + match crate::translate::chat_completions_to_responses(body) { + Ok(translated) => Ok(Some(fixup_for_chatgpt_backend(&translated))), + Err(e) => Err(OAuthError::LoginFailed(format!( + "request translation failed: {e}" + ))), + } + } + + fn upstream_url(&self, _tokens: &OAuthTokens) -> Option { + Some("https://chatgpt.com/backend-api/codex".to_string()) + } + + fn rewrite_request_path(&self, path: &str) -> Option { + if path == "/v1/chat/completions" { + Some("/responses".to_string()) + } else { + None + } + } + + fn needs_response_translation(&self, original_path: &str) -> bool { + original_path == "/v1/chat/completions" + } + + fn response_format(&self, original_path: &str) -> Option { + if original_path == "/v1/chat/completions" { + Some(super::ResponseFormat::ResponsesApi) + } else { + None + } + } +} + +/// Wait for an OAuth callback on a local HTTP server. +/// Returns (code, state). +async fn wait_for_oauth_callback( + port: u16, +) -> Result<(String, String), Box> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind(super::callback_bind_addr(port)).await?; + let (mut stream, _) = listener.accept().await?; + + let mut buf = vec![0u8; 4096]; + let n = stream.read(&mut buf).await?; + let request = String::from_utf8_lossy(&buf[..n]); + + // Parse the GET request for code and state query params + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or(""); + + let query = path.split('?').nth(1).unwrap_or(""); + let mut code = String::new(); + let mut state = String::new(); + + for param in query.split('&') { + if let Some((key, value)) = param.split_once('=') { + match key { + "code" => code = urlencoding::decode(value).unwrap_or_default().to_string(), + "state" => state = urlencoding::decode(value).unwrap_or_default().to_string(), + _ => {} + } + } + } + + let response = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n\ +

Login successful!

You can close this tab.

"; + stream.write_all(response.as_bytes()).await?; + stream.shutdown().await?; + + if code.is_empty() { + return Err("no authorization code in callback".into()); + } + + Ok((code, state)) +} + +/// Apply ChatGPT backend-specific fixups to the translated request body: +/// - Strip provider prefix from model (e.g. "openai/gpt-5.2-codex" → "gpt-5.2-codex") +/// - Set `store: false` (required by ChatGPT backend) +/// - Set `stream: true` (required by ChatGPT backend) +fn fixup_for_chatgpt_backend(body: &[u8]) -> Vec { + let Ok(mut parsed) = serde_json::from_slice::(body) else { + return body.to_vec(); + }; + if let Some(model) = parsed.get("model").and_then(|v| v.as_str()) { + if let Some(stripped) = model.strip_prefix("openai/") { + parsed["model"] = serde_json::Value::String(stripped.to_string()); + } + } + parsed["store"] = serde_json::Value::Bool(false); + parsed["stream"] = serde_json::Value::Bool(true); + // Codex backend does not support max_output_tokens + if let Some(obj) = parsed.as_object_mut() { + obj.remove("max_output_tokens"); + } + serde_json::to_vec(&parsed).unwrap_or_else(|_| body.to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_token_response() { + let json = serde_json::json!({ + "access_token": "eyJ...", + "refresh_token": "v1.MjQ...", + "id_token": "eyJhbG...", + "expires_in": 3600, + "token_type": "Bearer" + }); + + let tokens = parse_token_response(&json).unwrap(); + assert_eq!(tokens.access_token, "eyJ..."); + assert_eq!(tokens.refresh_token.as_deref(), Some("v1.MjQ...")); + assert_eq!(tokens.id_token.as_deref(), Some("eyJhbG...")); + assert!(tokens.expires_at.is_some()); + } + + #[test] + fn test_parse_token_response_missing_access_token() { + let json = serde_json::json!({ + "refresh_token": "v1.MjQ...", + }); + + let result = parse_token_response(&json); + assert!(result.is_err()); + } + + #[test] + fn test_parse_token_response_minimal() { + let json = serde_json::json!({ + "access_token": "minimal" + }); + + let tokens = parse_token_response(&json).unwrap(); + assert_eq!(tokens.access_token, "minimal"); + assert!(tokens.refresh_token.is_none()); + assert!(tokens.id_token.is_none()); + assert!(tokens.expires_at.is_none()); + } + + #[test] + fn test_generate_pkce() { + let (verifier, challenge) = generate_pkce(); + assert!(!verifier.is_empty()); + assert!(!challenge.is_empty()); + assert_ne!(verifier, challenge); + + // Verify challenge is S256 of verifier + use base64::Engine; + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(verifier.as_bytes()); + let expected = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(hasher.finalize()); + assert_eq!(challenge, expected); + } + + #[test] + fn test_codex_provider_defaults() { + let provider = CodexProvider::new(None, None, None, None); + assert_eq!(provider.id(), "codex"); + assert_eq!(provider.display_name(), "Codex / ChatGPT (OAuth)"); + assert!(provider.supports_device_code()); + assert!(!provider.supports_headless_url()); + assert_eq!(provider.client_id, DEFAULT_CLIENT_ID); + } + + #[test] + fn test_codex_provider_custom() { + let provider = CodexProvider::new( + Some("custom-client"), + Some("https://custom.auth/authorize"), + Some("https://custom.auth/token"), + Some(&["openid".to_string()]), + ); + assert_eq!(provider.client_id, "custom-client"); + assert_eq!(provider.auth_url, "https://custom.auth/authorize"); + assert_eq!(provider.token_url, "https://custom.auth/token"); + assert_eq!(provider.scopes, vec!["openid"]); + } + + #[test] + fn test_inject_auth() { + let provider = CodexProvider::new(None, None, None, None); + let mut headers = HeaderMap::new(); + provider.inject_auth(&mut headers, "test-token").unwrap(); + assert_eq!( + headers.get("authorization").unwrap().to_str().unwrap(), + "Bearer test-token" + ); + } + + #[test] + fn test_prepare_request_body_translates_chat() { + let provider = CodexProvider::new(None, None, None, None); + let tokens = OAuthTokens { + access_token: "t".to_string(), + refresh_token: None, + id_token: None, + expires_at: None, + account_id: None, + extra: BTreeMap::new(), + }; + let body = serde_json::json!({ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}] + }); + let result = provider + .prepare_request_body(body.to_string().as_bytes(), &tokens) + .unwrap(); + assert!(result.is_some()); + let parsed: serde_json::Value = serde_json::from_slice(&result.unwrap()).unwrap(); + assert!(parsed.get("input").is_some()); + assert!(parsed.get("messages").is_none()); + } + + #[test] + fn test_prepare_request_body_passthrough_non_chat() { + let provider = CodexProvider::new(None, None, None, None); + let tokens = OAuthTokens { + access_token: "t".to_string(), + refresh_token: None, + id_token: None, + expires_at: None, + account_id: None, + extra: BTreeMap::new(), + }; + // No "messages" field → passthrough + let body = serde_json::json!({"model": "gpt-4o", "input": "hello"}); + let result = provider + .prepare_request_body(body.to_string().as_bytes(), &tokens) + .unwrap(); + assert!(result.is_none()); + + // Non-JSON → passthrough + let result = provider.prepare_request_body(b"not json", &tokens).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_rewrite_path_chat_completions() { + let provider = CodexProvider::new(None, None, None, None); + assert_eq!( + provider.rewrite_request_path("/v1/chat/completions"), + Some("/responses".to_string()) + ); + } + + #[test] + fn test_rewrite_path_other() { + let provider = CodexProvider::new(None, None, None, None); + assert_eq!(provider.rewrite_request_path("/v1/models"), None); + assert_eq!(provider.rewrite_request_path("/v1/responses"), None); + assert_eq!(provider.rewrite_request_path("/responses"), None); + } + + #[test] + fn test_needs_translation_chat_completions() { + let provider = CodexProvider::new(None, None, None, None); + assert!(provider.needs_response_translation("/v1/chat/completions")); + } + + #[test] + fn test_needs_translation_other() { + let provider = CodexProvider::new(None, None, None, None); + assert!(!provider.needs_response_translation("/v1/models")); + assert!(!provider.needs_response_translation("/v1/responses")); + } + + #[test] + fn test_upstream_url_chatgpt() { + let provider = CodexProvider::new(None, None, None, None); + let tokens = OAuthTokens { + access_token: "t".to_string(), + refresh_token: None, + id_token: None, + expires_at: None, + account_id: None, + extra: BTreeMap::new(), + }; + assert_eq!( + provider.upstream_url(&tokens), + Some("https://chatgpt.com/backend-api/codex".to_string()) + ); + } + + #[test] + fn test_fixup_strips_model_prefix() { + let body = serde_json::to_vec(&serde_json::json!({ + "model": "openai/gpt-5.2-codex", + "input": [{"role": "user", "content": "hi"}] + })) + .unwrap(); + let result = fixup_for_chatgpt_backend(&body); + let parsed: serde_json::Value = serde_json::from_slice(&result).unwrap(); + assert_eq!(parsed["model"], "gpt-5.2-codex"); + assert_eq!(parsed["store"], false); + } + + #[test] + fn test_fixup_sets_store_false() { + let body = serde_json::to_vec(&serde_json::json!({ + "model": "gpt-4o-mini", + "input": [{"role": "user", "content": "hi"}] + })) + .unwrap(); + let result = fixup_for_chatgpt_backend(&body); + let parsed: serde_json::Value = serde_json::from_slice(&result).unwrap(); + assert_eq!(parsed["model"], "gpt-4o-mini"); + assert_eq!(parsed["store"], false); + } +} diff --git a/src/oauth/mod.rs b/src/oauth/mod.rs new file mode 100644 index 0000000..5d1e882 --- /dev/null +++ b/src/oauth/mod.rs @@ -0,0 +1,711 @@ +mod storage; + +pub mod codex; + +pub use storage::TokenStorage; + +use async_trait::async_trait; +use axum::http::HeaderMap; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt; +use std::sync::Arc; +use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +/// Error type for OAuth operations. +#[derive(Debug, thiserror::Error)] +pub enum OAuthError { + #[error("login failed: {0}")] + LoginFailed(String), + + #[error("token refresh failed: {0}")] + RefreshFailed(String), + + #[error("no tokens available for provider '{0}'")] + NoTokens(String), + + #[error("token expired for provider '{0}'")] + TokenExpired(String), + + #[error("provider not found: {0}")] + ProviderNotFound(String), + + #[error("header error: {0}")] + HeaderError(String), + + #[error("http error: {0}")] + HttpError(#[from] reqwest::Error), + + #[error("io error: {0}")] + IoError(#[from] std::io::Error), + + #[error("json error: {0}")] + JsonError(#[from] serde_json::Error), + + #[error("storage error: {0}")] + StorageError(String), +} + +impl From for OAuthError { + fn from(e: axum::http::header::InvalidHeaderValue) -> Self { + OAuthError::HeaderError(e.to_string()) + } +} + +/// Tokens obtained from an OAuth provider. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OAuthTokens { + pub access_token: String, + pub refresh_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub account_id: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extra: BTreeMap, +} + +impl OAuthTokens { + pub fn is_expired(&self) -> bool { + self.expires_at.is_some_and(|exp| exp <= Utc::now()) + } + + pub fn expires_in_secs(&self) -> Option { + self.expires_at.map(|exp| (exp - Utc::now()).num_seconds()) + } +} + +/// The core trait that each OAuth provider implements. +#[async_trait] +pub trait OAuthProvider: Send + Sync + fmt::Debug { + /// Unique identifier (e.g., "codex"). + fn id(&self) -> &str; + + /// Display name (e.g., "Codex (OpenAI)"). + fn display_name(&self) -> &str; + + /// Execute browser-based OAuth login flow. + async fn login_browser(&self, callback_port: u16) -> Result; + + /// Execute headless login flow (device code or copy/paste URL). + async fn login_headless(&self) -> Result; + + /// Refresh the access token using the refresh token. + async fn refresh(&self, refresh_token: &str) -> Result; + + /// Inject provider-specific auth headers into the request. + fn inject_auth(&self, headers: &mut HeaderMap, access_token: &str) -> Result<(), OAuthError>; + + /// Optionally transform the request body for provider-specific formats. + /// Returns None for pass-through (Codex). + fn prepare_request_body( + &self, + _body: &[u8], + _tokens: &OAuthTokens, + ) -> Result>, OAuthError> { + Ok(None) + } + + /// Resolve the upstream URL for this provider. + /// Returns None to use the configured [upstream] URL (Codex). + fn upstream_url(&self, _tokens: &OAuthTokens) -> Option { + None + } + + /// Whether this provider supports device code flow. + fn supports_device_code(&self) -> bool { + false + } + + /// Whether this provider supports headless copy/paste URL fallback. + fn supports_headless_url(&self) -> bool { + false + } + + /// Enrich tokens with provider-specific state if missing (e.g., project ID discovery). + /// Returns `Some(enriched)` if tokens were updated, `None` if no changes needed. + /// Called before `prepare_request_body` to ensure tokens are ready for use. + async fn enrich_tokens( + &self, + _tokens: &OAuthTokens, + ) -> Result, OAuthError> { + Ok(None) + } + + /// Optionally rewrite the request path (e.g., `/v1/chat/completions` → `/v1/responses`). + /// Returns `None` to use the original path unchanged. + fn rewrite_request_path(&self, _path: &str) -> Option { + None + } + + /// Whether responses from the upstream need to be translated back + /// to match the original request format. + fn needs_response_translation(&self, _original_path: &str) -> bool { + false + } + + /// What format the upstream response is in, for translation purposes. + /// Returns `None` if no translation is needed (passthrough). + fn response_format(&self, _original_path: &str) -> Option { + None + } +} + +/// The format of upstream API responses, used to select the correct translator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResponseFormat { + /// OpenAI Responses API → translate to chat.completion format + ResponsesApi, +} + +/// Configuration for an OAuth provider from TOML. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OAuthProviderConfig { + pub provider: String, + #[serde(default = "default_true")] + pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auth_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scopes: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub callback_port: Option, +} + +fn default_true() -> bool { + true +} + +const CALLBACK_BIND_HOST_ENV: &str = "CLAWSHELL_OAUTH_CALLBACK_HOST"; +const DEFAULT_CALLBACK_BIND_HOST: &str = "127.0.0.1"; + +pub(crate) fn callback_bind_addr(port: u16) -> String { + let host = std::env::var(CALLBACK_BIND_HOST_ENV) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| DEFAULT_CALLBACK_BIND_HOST.to_string()); + format!("{host}:{port}") +} + +/// Manages multiple OAuth providers, their tokens, and per-provider refresh tasks. +#[derive(Debug)] +pub struct OAuthRegistry { + providers: BTreeMap>, + tokens: Arc>>, + storage: TokenStorage, +} + +impl OAuthRegistry { + pub fn new(storage: TokenStorage) -> Self { + Self { + providers: BTreeMap::new(), + tokens: Arc::new(RwLock::new(BTreeMap::new())), + storage, + } + } + + pub fn register(&mut self, provider: Arc) { + let id = provider.id().to_string(); + debug!(provider = %id, "Registering OAuth provider"); + self.providers.insert(id, provider); + } + + /// Load persisted tokens from disk for all registered providers. + pub async fn load_tokens(&self) -> Result<(), OAuthError> { + let mut tokens = self.tokens.write().await; + for id in self.providers.keys() { + match self.storage.load(id) { + Ok(Some(t)) => { + info!(provider = %id, expired = t.is_expired(), "Loaded OAuth tokens from disk"); + tokens.insert(id.clone(), t); + } + Ok(None) => { + debug!(provider = %id, "No persisted tokens found"); + } + Err(e) => { + warn!(provider = %id, error = %e, "Failed to load persisted tokens"); + } + } + } + Ok(()) + } + + /// Get the current access token for a provider, refreshing if expired. + pub async fn current_access_token(&self, provider_id: &str) -> Result { + { + let tokens = self.tokens.read().await; + if let Some(t) = tokens.get(provider_id) { + if !t.is_expired() { + return Ok(t.access_token.clone()); + } + } + } + // Token is expired or missing — try refreshing + self.refresh(provider_id).await?; + let tokens = self.tokens.read().await; + tokens + .get(provider_id) + .map(|t| t.access_token.clone()) + .ok_or_else(|| OAuthError::NoTokens(provider_id.to_string())) + } + + /// Inject auth headers for the given provider. + pub async fn inject_auth( + &self, + provider_id: &str, + headers: &mut HeaderMap, + ) -> Result<(), OAuthError> { + let token = self.current_access_token(provider_id).await?; + let provider = self + .providers + .get(provider_id) + .ok_or_else(|| OAuthError::ProviderNotFound(provider_id.to_string()))?; + provider.inject_auth(headers, &token) + } + + /// Prepare the request body for the given provider. + /// Calls `enrich_tokens` first to ensure provider-specific state is populated. + pub async fn prepare_request_body( + &self, + provider_id: &str, + body: &[u8], + ) -> Result>, OAuthError> { + let provider = self + .providers + .get(provider_id) + .ok_or_else(|| OAuthError::ProviderNotFound(provider_id.to_string()))?; + + // Enrich tokens on-demand if the provider needs it (e.g., project_id discovery) + { + let tokens = self.tokens.read().await; + let t = tokens + .get(provider_id) + .ok_or_else(|| OAuthError::NoTokens(provider_id.to_string()))?; + if let Some(enriched) = provider.enrich_tokens(t).await? { + drop(tokens); + info!(provider = %provider_id, "Enriched OAuth tokens with provider-specific state"); + if let Err(e) = self.storage.save(provider_id, &enriched) { + warn!(provider = %provider_id, error = %e, "Failed to persist enriched tokens"); + } + self.tokens + .write() + .await + .insert(provider_id.to_string(), enriched); + } + } + + let tokens = self.tokens.read().await; + let t = tokens + .get(provider_id) + .ok_or_else(|| OAuthError::NoTokens(provider_id.to_string()))?; + provider.prepare_request_body(body, t) + } + + /// Resolve the upstream URL for the given provider. + pub async fn upstream_url(&self, provider_id: &str) -> Result, OAuthError> { + let provider = self + .providers + .get(provider_id) + .ok_or_else(|| OAuthError::ProviderNotFound(provider_id.to_string()))?; + let tokens = self.tokens.read().await; + let t = tokens + .get(provider_id) + .ok_or_else(|| OAuthError::NoTokens(provider_id.to_string()))?; + Ok(provider.upstream_url(t)) + } + + /// Refresh the access token for a specific provider. + pub async fn refresh(&self, provider_id: &str) -> Result<(), OAuthError> { + let provider = self + .providers + .get(provider_id) + .ok_or_else(|| OAuthError::ProviderNotFound(provider_id.to_string()))?; + + let refresh_token = { + let tokens = self.tokens.read().await; + tokens + .get(provider_id) + .and_then(|t| t.refresh_token.clone()) + .ok_or_else(|| { + OAuthError::RefreshFailed(format!( + "no refresh token for provider '{provider_id}'" + )) + })? + }; + + info!(provider = %provider_id, "Refreshing OAuth access token"); + let new_tokens = provider.refresh(&refresh_token).await?; + self.storage + .save(provider_id, &new_tokens) + .map_err(|e| OAuthError::StorageError(e.to_string()))?; + self.tokens + .write() + .await + .insert(provider_id.to_string(), new_tokens); + info!(provider = %provider_id, "OAuth token refreshed successfully"); + Ok(()) + } + + /// Store tokens after a successful login (called from onboard flow). + pub async fn store_tokens( + &self, + provider_id: &str, + tokens: OAuthTokens, + ) -> Result<(), OAuthError> { + self.storage + .save(provider_id, &tokens) + .map_err(|e| OAuthError::StorageError(e.to_string()))?; + self.tokens + .write() + .await + .insert(provider_id.to_string(), tokens); + Ok(()) + } + + /// Spawn background refresh tasks for all providers with tokens. + pub fn spawn_refresh_tasks(&self, cancel: CancellationToken) { + let tokens = Arc::clone(&self.tokens); + for (id, provider) in &self.providers { + let id = id.clone(); + let provider = Arc::clone(provider); + let tokens = Arc::clone(&tokens); + let storage = self.storage.clone(); + let cancel = cancel.clone(); + + tokio::spawn(async move { + loop { + let sleep_secs = { + let guard = tokens.read().await; + match guard.get(&id) { + Some(t) => { + let remaining = t.expires_in_secs().unwrap_or(3600); + // Refresh at 75% of TTL, minimum 60 seconds + (remaining * 3 / 4).max(60) + } + None => 3600, // no tokens yet, check hourly + } + }; + + debug!(provider = %id, sleep_secs, "OAuth refresh task sleeping"); + + tokio::select! { + _ = cancel.cancelled() => { + info!(provider = %id, "OAuth refresh task cancelled"); + return; + } + _ = tokio::time::sleep(std::time::Duration::from_secs(sleep_secs as u64)) => {} + } + + let refresh_token = { + let guard = tokens.read().await; + guard.get(&id).and_then(|t| t.refresh_token.clone()) + }; + + let Some(refresh_token) = refresh_token else { + debug!(provider = %id, "No refresh token available, skipping refresh"); + continue; + }; + + match provider.refresh(&refresh_token).await { + Ok(new_tokens) => { + if let Err(e) = storage.save(&id, &new_tokens) { + error!(provider = %id, error = %e, "Failed to persist refreshed tokens"); + } + tokens.write().await.insert(id.clone(), new_tokens); + info!(provider = %id, "Background token refresh successful"); + } + Err(e) => { + error!(provider = %id, error = %e, "Background token refresh failed"); + } + } + } + }); + } + } + + pub fn rewrite_request_path( + &self, + provider_id: &str, + path: &str, + ) -> Result, OAuthError> { + let provider = self + .providers + .get(provider_id) + .ok_or_else(|| OAuthError::ProviderNotFound(provider_id.to_string()))?; + Ok(provider.rewrite_request_path(path)) + } + + pub fn needs_response_translation( + &self, + provider_id: &str, + original_path: &str, + ) -> Result { + let provider = self + .providers + .get(provider_id) + .ok_or_else(|| OAuthError::ProviderNotFound(provider_id.to_string()))?; + Ok(provider.needs_response_translation(original_path)) + } + + pub fn response_format( + &self, + provider_id: &str, + original_path: &str, + ) -> Result, OAuthError> { + let provider = self + .providers + .get(provider_id) + .ok_or_else(|| OAuthError::ProviderNotFound(provider_id.to_string()))?; + Ok(provider.response_format(original_path)) + } + + pub fn has_provider(&self, id: &str) -> bool { + self.providers.contains_key(id) + } + + pub fn provider_ids(&self) -> Vec { + self.providers.keys().cloned().collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug)] + struct MockProvider { + id: String, + } + + #[async_trait] + impl OAuthProvider for MockProvider { + fn id(&self) -> &str { + &self.id + } + fn display_name(&self) -> &str { + "Mock Provider" + } + async fn login_browser(&self, _callback_port: u16) -> Result { + Ok(OAuthTokens { + access_token: "mock-access".to_string(), + refresh_token: Some("mock-refresh".to_string()), + id_token: None, + expires_at: Some(Utc::now() + chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }) + } + async fn login_headless(&self) -> Result { + self.login_browser(0).await + } + async fn refresh(&self, _refresh_token: &str) -> Result { + Ok(OAuthTokens { + access_token: "refreshed-access".to_string(), + refresh_token: Some("new-refresh".to_string()), + id_token: None, + expires_at: Some(Utc::now() + chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }) + } + fn inject_auth( + &self, + headers: &mut HeaderMap, + access_token: &str, + ) -> Result<(), OAuthError> { + headers.insert( + axum::http::header::AUTHORIZATION, + format!("Bearer {access_token}").parse()?, + ); + Ok(()) + } + } + + #[test] + fn test_tokens_not_expired() { + let tokens = OAuthTokens { + access_token: "test".to_string(), + refresh_token: None, + id_token: None, + expires_at: Some(Utc::now() + chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }; + assert!(!tokens.is_expired()); + } + + #[test] + fn test_tokens_expired() { + let tokens = OAuthTokens { + access_token: "test".to_string(), + refresh_token: None, + id_token: None, + expires_at: Some(Utc::now() - chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }; + assert!(tokens.is_expired()); + } + + #[test] + fn test_tokens_no_expiry() { + let tokens = OAuthTokens { + access_token: "test".to_string(), + refresh_token: None, + id_token: None, + expires_at: None, + account_id: None, + extra: BTreeMap::new(), + }; + assert!(!tokens.is_expired()); + assert!(tokens.expires_in_secs().is_none()); + } + + #[tokio::test] + async fn test_registry_register_and_access() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + let mut registry = OAuthRegistry::new(storage); + + let provider = Arc::new(MockProvider { + id: "mock".to_string(), + }); + registry.register(provider); + assert!(registry.has_provider("mock")); + assert!(!registry.has_provider("other")); + } + + #[tokio::test] + async fn test_registry_store_and_retrieve_tokens() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + let mut registry = OAuthRegistry::new(storage); + + let provider = Arc::new(MockProvider { + id: "mock".to_string(), + }); + registry.register(provider); + + let tokens = OAuthTokens { + access_token: "test-access".to_string(), + refresh_token: Some("test-refresh".to_string()), + id_token: None, + expires_at: Some(Utc::now() + chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }; + registry.store_tokens("mock", tokens).await.unwrap(); + + let token = registry.current_access_token("mock").await.unwrap(); + assert_eq!(token, "test-access"); + } + + #[tokio::test] + async fn test_registry_refresh_expired_token() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + let mut registry = OAuthRegistry::new(storage); + + let provider = Arc::new(MockProvider { + id: "mock".to_string(), + }); + registry.register(provider); + + // Store an expired token + let tokens = OAuthTokens { + access_token: "expired-access".to_string(), + refresh_token: Some("test-refresh".to_string()), + id_token: None, + expires_at: Some(Utc::now() - chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }; + registry.store_tokens("mock", tokens).await.unwrap(); + + // Should auto-refresh + let token = registry.current_access_token("mock").await.unwrap(); + assert_eq!(token, "refreshed-access"); + } + + #[tokio::test] + async fn test_registry_inject_auth() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + let mut registry = OAuthRegistry::new(storage); + + let provider = Arc::new(MockProvider { + id: "mock".to_string(), + }); + registry.register(provider); + + let tokens = OAuthTokens { + access_token: "inject-test".to_string(), + refresh_token: Some("r".to_string()), + id_token: None, + expires_at: Some(Utc::now() + chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }; + registry.store_tokens("mock", tokens).await.unwrap(); + + let mut headers = HeaderMap::new(); + registry.inject_auth("mock", &mut headers).await.unwrap(); + assert_eq!( + headers.get("authorization").unwrap().to_str().unwrap(), + "Bearer inject-test" + ); + } + + #[tokio::test] + async fn test_registry_provider_not_found() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + let registry = OAuthRegistry::new(storage); + + let result = registry.current_access_token("nonexistent").await; + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("provider not found") + ); + } + + #[tokio::test] + async fn test_registry_load_tokens_from_disk() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + + // Pre-persist tokens + let tokens = OAuthTokens { + access_token: "disk-token".to_string(), + refresh_token: Some("disk-refresh".to_string()), + id_token: None, + expires_at: Some(Utc::now() + chrono::Duration::hours(1)), + account_id: None, + extra: BTreeMap::new(), + }; + storage.save("mock", &tokens).unwrap(); + + let mut registry = OAuthRegistry::new(storage); + let provider = Arc::new(MockProvider { + id: "mock".to_string(), + }); + registry.register(provider); + registry.load_tokens().await.unwrap(); + + let token = registry.current_access_token("mock").await.unwrap(); + assert_eq!(token, "disk-token"); + } +} diff --git a/src/oauth/storage.rs b/src/oauth/storage.rs new file mode 100644 index 0000000..2e3a08c --- /dev/null +++ b/src/oauth/storage.rs @@ -0,0 +1,189 @@ +use super::OAuthTokens; +use std::path::PathBuf; +use tracing::debug; + +/// Per-provider token persistence under a directory (e.g. `/etc/clawshell/oauth/`). +#[derive(Debug, Clone)] +pub struct TokenStorage { + dir: PathBuf, +} + +impl Default for TokenStorage { + fn default() -> Self { + Self { + dir: PathBuf::from("/etc/clawshell/oauth"), + } + } +} + +impl TokenStorage { + pub fn new(dir: PathBuf) -> Self { + Self { dir } + } + + pub fn dir(&self) -> &PathBuf { + &self.dir + } + + fn token_path(&self, provider_id: &str) -> PathBuf { + self.dir.join(format!("{provider_id}.json")) + } + + /// Save tokens for a provider, creating the directory if needed. + pub fn save(&self, provider_id: &str, tokens: &OAuthTokens) -> Result<(), std::io::Error> { + std::fs::create_dir_all(&self.dir)?; + let path = self.token_path(provider_id); + let content = serde_json::to_string_pretty(tokens) + .map_err(|e| std::io::Error::other(format!("failed to serialize tokens: {e}")))?; + std::fs::write(&path, content)?; + + // Set file permissions to 0600 (owner read/write only) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))?; + } + + debug!(provider = %provider_id, path = %path.display(), "OAuth tokens saved"); + Ok(()) + } + + /// Load tokens for a provider, returning None if the file doesn't exist. + pub fn load(&self, provider_id: &str) -> Result, std::io::Error> { + let path = self.token_path(provider_id); + if !path.exists() { + return Ok(None); + } + let content = std::fs::read_to_string(&path)?; + let tokens: OAuthTokens = serde_json::from_str(&content) + .map_err(|e| std::io::Error::other(format!("failed to parse tokens: {e}")))?; + debug!(provider = %provider_id, path = %path.display(), "OAuth tokens loaded"); + Ok(Some(tokens)) + } + + /// Remove tokens for a provider. + pub fn remove(&self, provider_id: &str) -> Result<(), std::io::Error> { + let path = self.token_path(provider_id); + if path.exists() { + std::fs::remove_file(&path)?; + debug!(provider = %provider_id, path = %path.display(), "OAuth tokens removed"); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use std::collections::BTreeMap; + + fn test_tokens() -> OAuthTokens { + OAuthTokens { + access_token: "access-123".to_string(), + refresh_token: Some("refresh-456".to_string()), + id_token: None, + expires_at: Some(Utc::now() + chrono::Duration::hours(1)), + account_id: Some("user@test.com".to_string()), + extra: BTreeMap::new(), + } + } + + #[test] + fn test_save_and_load() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + + let tokens = test_tokens(); + storage.save("test-provider", &tokens).unwrap(); + + let loaded = storage.load("test-provider").unwrap().unwrap(); + assert_eq!(loaded.access_token, "access-123"); + assert_eq!(loaded.refresh_token.as_deref(), Some("refresh-456")); + assert_eq!(loaded.account_id.as_deref(), Some("user@test.com")); + } + + #[test] + fn test_load_nonexistent() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + + let loaded = storage.load("nonexistent").unwrap(); + assert!(loaded.is_none()); + } + + #[test] + fn test_remove() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + + let tokens = test_tokens(); + storage.save("removable", &tokens).unwrap(); + assert!(storage.load("removable").unwrap().is_some()); + + storage.remove("removable").unwrap(); + assert!(storage.load("removable").unwrap().is_none()); + } + + #[test] + fn test_remove_nonexistent() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + // Should not error + storage.remove("nonexistent").unwrap(); + } + + #[test] + fn test_creates_directory() { + let dir = tempfile::tempdir().unwrap(); + let nested = dir.path().join("a").join("b").join("c"); + let storage = TokenStorage::new(nested.clone()); + + let tokens = test_tokens(); + storage.save("test", &tokens).unwrap(); + assert!(nested.join("test.json").exists()); + } + + #[test] + fn test_tokens_with_extra_fields() { + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + + let mut tokens = test_tokens(); + tokens + .extra + .insert("project_id".to_string(), serde_json::json!("proj-abc-123")); + tokens + .extra + .insert("tier".to_string(), serde_json::json!("production")); + + storage.save("extra-fields", &tokens).unwrap(); + + let loaded = storage.load("extra-fields").unwrap().unwrap(); + assert_eq!( + loaded.extra.get("project_id").unwrap().as_str().unwrap(), + "proj-abc-123" + ); + assert_eq!( + loaded.extra.get("tier").unwrap().as_str().unwrap(), + "production" + ); + } + + #[cfg(unix)] + #[test] + fn test_file_permissions() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let storage = TokenStorage::new(dir.path().to_path_buf()); + + let tokens = test_tokens(); + storage.save("perms-test", &tokens).unwrap(); + + let path = dir.path().join("perms-test.json"); + let metadata = std::fs::metadata(path).unwrap(); + let mode = metadata.permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + } +} diff --git a/src/onboard/config_render.rs b/src/onboard/config_render.rs index 061b305..dfe30f8 100644 --- a/src/onboard/config_render.rs +++ b/src/onboard/config_render.rs @@ -1,4 +1,4 @@ -use super::types::{OnboardConfig, OnboardEmailMode}; +use super::types::{OnboardAuthMethod, OnboardConfig, OnboardEmailMode}; /// Return the default OpenClaw config path. pub fn default_openclaw_config_path() -> String { @@ -11,6 +11,47 @@ pub fn default_openclaw_config_path() -> String { /// Generate the ClawShell TOML configuration content with the given key mapping. pub fn generate_clawshell_config(config: &OnboardConfig) -> String { + let key_section = match &config.auth_method { + OnboardAuthMethod::OAuth { provider_id } => { + format!( + r#"[[keys]] +virtual_key = {virtual_key} +provider = {provider} +auth = "oauth" +oauth_provider = {oauth_provider} +"#, + virtual_key = toml_string(&config.virtual_api_key), + provider = toml_string(&config.provider), + oauth_provider = toml_string(provider_id), + ) + } + OnboardAuthMethod::StaticKey => { + format!( + r#"[[keys]] +virtual_key = {virtual_key} +real_key = {real_key} +provider = {provider} +"#, + virtual_key = toml_string(&config.virtual_api_key), + real_key = toml_string(&config.real_api_key), + provider = toml_string(&config.provider), + ) + } + }; + + let oauth_providers_section = match &config.auth_method { + OnboardAuthMethod::OAuth { provider_id } => { + format!( + r#" +[[oauth_providers]] +provider = {provider_id} +"#, + provider_id = toml_string(provider_id), + ) + } + OnboardAuthMethod::StaticKey => String::new(), + }; + let mut output = format!( r#"# ClawShell Configuration version = "{version}" @@ -25,11 +66,7 @@ openai_base_url = "https://api.openai.com" openrouter_base_url = "https://openrouter.ai/api" anthropic_base_url = "https://api.anthropic.com" -[[keys]] -virtual_key = {virtual_key} -real_key = {real_key} -provider = {provider} -[dlp] +{key_section}[dlp] scan_responses = true patterns = [ {{ name = "ssn", regex = '\\b\\d{{3}}-\\d{{2}}-\\d{{4}}\\b', action = "redact" }}, @@ -38,13 +75,12 @@ patterns = [ {{ name = "mastercard", regex = '\\b5[1-5][0-9]{{14}}\\b', action = "redact" }}, {{ name = "amex_card", regex = '\\b3[47][0-9]{{13}}\\b', action = "redact" }}, ] -"#, +{oauth_providers_section}"#, version = env!("CARGO_PKG_VERSION"), host = config.server_host, port = config.server_port, - virtual_key = toml_string(&config.virtual_api_key), - real_key = toml_string(&config.real_api_key), - provider = toml_string(&config.provider), + key_section = key_section, + oauth_providers_section = oauth_providers_section, ); if let Some(email) = &config.email { diff --git a/src/onboard/interactive.rs b/src/onboard/interactive.rs index e1afcfe..42823db 100644 --- a/src/onboard/interactive.rs +++ b/src/onboard/interactive.rs @@ -1,6 +1,6 @@ use super::config_render::default_openclaw_config_path; use super::credentials::detect_openclaw_api_key_for_provider; -use super::types::{OnboardConfig, OnboardEmailConfig, OnboardEmailMode}; +use super::types::{OnboardAuthMethod, OnboardConfig, OnboardEmailConfig, OnboardEmailMode}; use crate::email::{EmailAccountCredentials, ImapEmailService}; use crate::tui; @@ -69,6 +69,14 @@ fn load_existing_config_from_vfs(config_dir: &VfsPath) -> Option .get("openclaw_config_path") .and_then(|v| v.as_str()) .map(String::from); + existing.auth_method = json + .get("auth_method") + .and_then(|v| v.as_str()) + .map(String::from); + existing.oauth_provider = json + .get("oauth_provider") + .and_then(|v| v.as_str()) + .map(String::from); } // Read clawshell.toml for server host/port and optional Email settings @@ -259,6 +267,8 @@ struct ExistingConfig { openclaw_config_path: Option, server_host: Option, server_port: Option, + auth_method: Option, + oauth_provider: Option, email_enabled: Option, email_mode: Option, email_sender_rules: Vec, @@ -278,6 +288,8 @@ impl ExistingConfig { || self.openclaw_config_path.is_some() || self.server_host.is_some() || self.server_port.is_some() + || self.auth_method.is_some() + || self.oauth_provider.is_some() || self.email_enabled.is_some() || self.email_mode.is_some() || !self.email_sender_rules.is_empty() @@ -308,51 +320,56 @@ fn mask_secret(secret: &str) -> String { } } -/// Collect all onboarding information using the TUI (interactive terminal prompts). -/// If a previous configuration exists, its values are used as defaults. -pub fn collect_onboard_config_tui() -> Result> { - let existing = load_existing_config(); +/// Run the OAuth login flow for the given provider, persisting tokens. +fn run_oauth_login(provider_id: &str) -> Result<(), Box> { + use crate::oauth::codex::CodexProvider; + use crate::oauth::{OAuthProvider, TokenStorage}; - if existing.is_some() { - tui::print_success("Existing configuration detected — using as defaults."); - println!(); - } - - let existing = existing.unwrap_or_default(); + let provider: Box = match provider_id { + "codex" => Box::new(CodexProvider::new(None, None, None, None)), + other => return Err(format!("unknown OAuth provider: {other}").into()), + }; - tui::print_section("API Configuration"); + let storage = TokenStorage::default(); - // Provider selection — if existing, reorder so the existing choice is first - let provider_options = match existing.provider.as_deref() { - Some("anthropic") => vec!["Anthropic", "OpenAI", "OpenRouter"], - Some("openrouter") => vec!["OpenRouter", "OpenAI", "Anthropic"], - _ => vec!["OpenAI", "OpenRouter", "Anthropic"], + // Called from within #[tokio::main], so use block_in_place to avoid + // "Cannot start a runtime from within a runtime" panic. + let run_async = |fut: std::pin::Pin + Send>>| { + let handle = tokio::runtime::Handle::current(); + tokio::task::block_in_place(|| handle.block_on(fut)) }; - let provider_choice = tui::prompt_select("Select a model provider", provider_options)?; - let provider = match provider_choice { - "Anthropic" => "anthropic".to_string(), - "OpenRouter" => "openrouter".to_string(), - _ => "openai".to_string(), + + let tokens = if provider.supports_device_code() { + tui::print_info("Flow", "device code (no browser required)"); + run_async(Box::pin(provider.login_headless()))? + } else if provider.supports_headless_url() { + tui::print_info("Flow", "headless (copy URL, paste code)"); + run_async(Box::pin(provider.login_headless()))? + } else { + tui::print_info("Flow", "browser login"); + tui::print_warning("A browser window will open for you to authorize access."); + run_async(Box::pin(provider.login_browser(8400)))? }; - // Model name — use existing model or provider-specific default - let default_model = existing - .model - .as_deref() - .unwrap_or(match provider.as_str() { - "anthropic" => "claude-sonnet-4-5-20250929", - "openai" => "gpt-5.2-chat-latest", - "openrouter" => "openrouter/auto", - _ => unreachable!(), - }); - let model = tui::prompt_text("Enter the model name", Some(default_model))?; + storage.save(provider_id, &tokens)?; + tui::print_success("OAuth login successful — tokens saved."); + if let Some(acct) = tokens.account_id.as_deref() { + tui::print_info("Account", acct); + } - // Real API key — if ClawShell already has one, use it; otherwise try detecting from OpenClaw + Ok(()) +} + +/// Collect a static API key from the user (original flow). +fn collect_static_api_key( + provider: &str, + existing: &ExistingConfig, +) -> Result> { let is_first_onboard = existing.real_api_key.is_none(); let effective_existing_key = if !is_first_onboard { existing.real_api_key.clone() } else { - let key = detect_openclaw_api_key_for_provider(&provider); + let key = detect_openclaw_api_key_for_provider(provider); if key.is_some() { tui::print_warning( "An API key was detected from your OpenClaw config. \ @@ -364,15 +381,12 @@ pub fn collect_onboard_config_tui() -> Result Result Result> { + let existing = load_existing_config(); + + if existing.is_some() { + tui::print_success("Existing configuration detected — using as defaults."); + println!(); + } + + let existing = existing.unwrap_or_default(); + + tui::print_section("API Configuration"); + + // Provider selection + const MENU_OPENAI: &str = "OpenAI"; + const MENU_OPENROUTER: &str = "OpenRouter"; + const MENU_ANTHROPIC: &str = "Anthropic"; + const MENU_CODEX: &str = "Codex / ChatGPT (OAuth)"; + let all_options = [MENU_OPENAI, MENU_OPENROUTER, MENU_ANTHROPIC, MENU_CODEX]; + + // Reorder so the existing choice appears first + let preferred = match ( + existing.auth_method.as_deref(), + existing.oauth_provider.as_deref(), + existing.provider.as_deref(), + ) { + (Some("oauth"), Some("codex"), _) | (Some("oauth"), _, _) => Some(MENU_CODEX), + (_, _, Some("anthropic")) => Some(MENU_ANTHROPIC), + (_, _, Some("openrouter")) => Some(MENU_OPENROUTER), + (_, _, Some("openai")) => Some(MENU_OPENAI), + _ => None, + }; + let provider_options: Vec<&str> = if let Some(first) = preferred { + std::iter::once(first) + .chain(all_options.iter().copied().filter(|o| *o != first)) + .collect() + } else { + all_options.to_vec() + }; + + let provider_choice = tui::prompt_select("Select a model provider", provider_options)?; + + let (provider, auth_method) = match provider_choice { + MENU_ANTHROPIC => ("anthropic".to_string(), OnboardAuthMethod::StaticKey), + MENU_OPENROUTER => ("openrouter".to_string(), OnboardAuthMethod::StaticKey), + MENU_CODEX => ( + "openai".to_string(), + OnboardAuthMethod::OAuth { + provider_id: "codex".to_string(), + }, + ), + _ => ("openai".to_string(), OnboardAuthMethod::StaticKey), + }; + + // Model name — use existing model or provider/auth-specific default + let default_model = existing.model.as_deref().unwrap_or(match provider_choice { + MENU_ANTHROPIC => "claude-sonnet-4-5-20250929", + MENU_OPENROUTER => "openrouter/auto", + MENU_CODEX => "gpt-5.2-chat-latest", + _ => "gpt-5.2-chat-latest", // OpenAI default + }); + let model = tui::prompt_text("Enter the model name", Some(default_model))?; + + let real_api_key = match &auth_method { + OnboardAuthMethod::OAuth { provider_id } => { + // OAuth flow — run device code or browser login + tui::print_section("OAuth Login"); + tui::print_info("OAuth provider", provider_id); + + run_oauth_login(provider_id)?; + + // No static API key needed for OAuth + String::new() + } + OnboardAuthMethod::StaticKey => { + // Static key flow — same as before + collect_static_api_key(&provider, &existing)? + } + }; + // Virtual API key let fallback_virtual_key = format!("{{clawshell-virtual-key-{}}}", provider); let default_virtual = existing @@ -705,6 +803,7 @@ pub fn collect_onboard_config_tui() -> Result OnboardConfig { OnboardConfig { provider: "openai".to_string(), model: "gpt-5.2".to_string(), + auth_method: super::types::OnboardAuthMethod::StaticKey, real_api_key: "sk-real-key-123".to_string(), virtual_api_key: "{clawshell-virtual-key-openai}".to_string(), openclaw_config_path: PathBuf::from("/tmp/test-openclaw.json"), diff --git a/src/onboard/types.rs b/src/onboard/types.rs index bb18fa4..6c3933d 100644 --- a/src/onboard/types.rs +++ b/src/onboard/types.rs @@ -41,11 +41,26 @@ pub struct OpenclawFileRemovalPreview { pub removals: Vec, } +/// Authentication method chosen during onboarding. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub enum OnboardAuthMethod { + /// Static API key (the traditional approach). + #[default] + StaticKey, + /// OAuth provider supplies access tokens at runtime. + OAuth { + /// Provider identifier, e.g. "codex". + provider_id: String, + }, +} + /// Collected onboarding configuration from user prompts. #[derive(Debug, Clone)] pub struct OnboardConfig { pub provider: String, pub model: String, + pub auth_method: OnboardAuthMethod, + /// Set for `StaticKey`; empty for `OAuth`. pub real_api_key: String, pub virtual_api_key: String, pub openclaw_config_path: PathBuf, diff --git a/src/openclaw_cli.rs b/src/openclaw_cli.rs index 24bc57a..349157f 100644 --- a/src/openclaw_cli.rs +++ b/src/openclaw_cli.rs @@ -692,6 +692,7 @@ mod tests { onboard::OnboardConfig { provider: "openai".to_string(), model: "gpt-5".to_string(), + auth_method: onboard::OnboardAuthMethod::StaticKey, real_api_key: "real_key".to_string(), virtual_api_key: "virtual_key".to_string(), openclaw_config_path: PathBuf::from("/home/user/.openclaw/openclaw.json"), diff --git a/src/proxy.rs b/src/proxy.rs index f958a4f..4e9b77b 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -62,22 +62,7 @@ impl ProxyClient { "Preparing upstream request" ); - let mut req_headers = HeaderMap::new(); - for (name, value) in &headers { - let name_str = name.as_str().to_lowercase(); - // Skip hop-by-hop headers and the original auth header - if name_str == "host" - || name_str == "authorization" - || name_str == "connection" - || name_str == "content-length" - || name_str == "transfer-encoding" - || name_str == "x-api-key" - { - trace!(header = %name_str, "Skipping hop-by-hop/auth header"); - continue; - } - req_headers.insert(name.clone(), value.clone()); - } + let mut req_headers = filter_hop_by_hop_headers(&headers); trace!( forwarded_header_count = req_headers.len(), @@ -107,6 +92,74 @@ impl ProxyClient { } } + self.send_upstream(method, &upstream_url, req_headers, body) + .await + } + + /// Forward a request using OAuth-injected auth headers and optional overrides. + #[allow(clippy::too_many_arguments)] + pub async fn forward_oauth( + &self, + method: Method, + uri: &Uri, + original_headers: HeaderMap, + body: Bytes, + provider: Provider, + auth_headers: HeaderMap, + upstream_url_override: Option<&str>, + ) -> Result { + let upstream_url = if let Some(base) = upstream_url_override { + format!( + "{}{}", + base, + uri.path_and_query() + .map(|pq| pq.as_str()) + .unwrap_or(uri.path()) + ) + } else { + let base_url = self.upstream_urls.get(&provider).ok_or_else(|| { + ProxyError::Internal(format!("No upstream URL for provider {:?}", provider)) + })?; + format!( + "{}{}", + base_url, + uri.path_and_query() + .map(|pq| pq.as_str()) + .unwrap_or(uri.path()) + ) + }; + + debug!( + %upstream_url, + %method, + provider = ?provider, + body_size = body.len(), + "Preparing OAuth upstream request" + ); + + let mut req_headers = filter_hop_by_hop_headers(&original_headers); + + // Apply OAuth auth headers (these may include Authorization, x-goog-api-client, etc.) + for (name, value) in &auth_headers { + req_headers.insert(name.clone(), value.clone()); + } + + trace!( + forwarded_header_count = req_headers.len(), + "Filtered request headers (OAuth)" + ); + + self.send_upstream(method, &upstream_url, req_headers, body) + .await + } + + async fn send_upstream( + &self, + method: Method, + upstream_url: &str, + req_headers: HeaderMap, + body: Bytes, + ) -> Result { let reqwest_method = match method { Method::GET => reqwest::Method::GET, Method::POST => reqwest::Method::POST, @@ -124,7 +177,7 @@ impl ProxyClient { let upstream_resp = self .client - .request(reqwest_method, &upstream_url) + .request(reqwest_method, upstream_url) .headers(req_headers) .body(body) .send() @@ -137,7 +190,6 @@ impl ProxyClient { debug!( upstream_status = %status, - provider = ?provider, "Received upstream response" ); @@ -164,15 +216,9 @@ impl ProxyClient { let byte_stream = upstream_resp.bytes_stream().map_err(IoError::other); let body = Body::from_stream(byte_stream); - // Rebind the `status` var to clarify the type for human developer: - // it is guaranteed to be `StatusCode` due to the `.unwrap_or` in its assignment above. let status: StatusCode = status; let mut response = Response::builder().status(status); - // INVARIANT: the `status` variable is guaranteed to be `StatusCode`, - // so this `.unwrap` should never panic. *response.headers_mut().unwrap() = resp_headers; - // INVARIANT: the builder should always succeed since we just added a valid status code and headers, - // so this `.unwrap` should never panic. Ok(response.body(body).unwrap()) } else { // Buffer the full response @@ -186,18 +232,34 @@ impl ProxyClient { "Buffered upstream response body" ); - // Rebind the `status` var to clarify the type for human developer: - // it is guaranteed to be `StatusCode` due to the `.unwrap_or` in its assignment above. let status: StatusCode = status; let mut response = Response::builder().status(status); - // INVARIANT: the builder should always succeed since we just added a valid status code and headers, - // so this `.unwrap` should never panic. *response.headers_mut().unwrap() = resp_headers; Ok(response.body(Body::from(resp_body)).unwrap()) } } } +fn filter_hop_by_hop_headers(headers: &HeaderMap) -> HeaderMap { + let mut filtered = HeaderMap::new(); + for (name, value) in headers { + let name_str = name.as_str().to_lowercase(); + // Skip hop-by-hop headers and the original auth header + if name_str == "host" + || name_str == "authorization" + || name_str == "connection" + || name_str == "content-length" + || name_str == "transfer-encoding" + || name_str == "x-api-key" + { + trace!(header = %name_str, "Skipping hop-by-hop/auth header"); + continue; + } + filtered.insert(name.clone(), value.clone()); + } + filtered +} + #[derive(Debug)] pub enum ProxyError { Upstream(String), @@ -308,4 +370,19 @@ mod tests { let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); assert!(json["error"].as_str().unwrap().contains("TRACE")); } + + #[test] + fn test_filter_hop_by_hop_headers() { + let mut headers = HeaderMap::new(); + headers.insert("authorization", "Bearer vk-test".parse().unwrap()); + headers.insert("content-type", "application/json".parse().unwrap()); + headers.insert("host", "localhost".parse().unwrap()); + headers.insert("x-custom", "custom-value".parse().unwrap()); + + let filtered = filter_hop_by_hop_headers(&headers); + assert!(filtered.get("authorization").is_none()); + assert!(filtered.get("host").is_none()); + assert!(filtered.get("content-type").is_some()); + assert!(filtered.get("x-custom").is_some()); + } } diff --git a/src/translate.rs b/src/translate.rs new file mode 100644 index 0000000..5763615 --- /dev/null +++ b/src/translate.rs @@ -0,0 +1,995 @@ +use crate::dlp::DlpScanner; +use axum::body::Body; +use bytes::{Bytes, BytesMut}; +use futures_util::Stream; +use serde_json::Value; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use tracing::warn; + +#[derive(Debug, thiserror::Error)] +pub enum TranslateError { + #[error("json error: {0}")] + Json(#[from] serde_json::Error), + + #[error("missing field: {0}")] + MissingField(&'static str), +} + +/// Fields that are compatible between chat/completions and responses API. +const PASSTHROUGH_FIELDS: &[&str] = &["model", "stream", "temperature", "top_p", "stop"]; + +/// Fields that must be stripped from chat/completions requests (not supported by responses API). +const STRIP_FIELDS: &[&str] = &[ + "frequency_penalty", + "presence_penalty", + "logprobs", + "top_logprobs", + "logit_bias", + "n", + "response_format", + "seed", + "service_tier", + "user", +]; + +/// Translate a `/v1/chat/completions` request body to `/v1/responses` format. +pub fn chat_completions_to_responses(body: &[u8]) -> Result, TranslateError> { + let req: Value = serde_json::from_slice(body)?; + let obj = req + .as_object() + .ok_or(TranslateError::MissingField("root object"))?; + + let messages = obj + .get("messages") + .and_then(Value::as_array) + .ok_or(TranslateError::MissingField("messages"))?; + + let mut result = serde_json::Map::new(); + + // Separate system messages → instructions, rest → input + let mut system_parts: Vec<&str> = Vec::new(); + let mut input: Vec = Vec::new(); + + for msg in messages { + let role = msg.get("role").and_then(Value::as_str).unwrap_or(""); + if role == "system" { + if let Some(content) = msg.get("content").and_then(Value::as_str) { + system_parts.push(content); + } + } else { + input.push(convert_message_content(msg.clone())); + } + } + + // Codex responses API requires `instructions` even when empty + result.insert( + "instructions".to_string(), + Value::String(system_parts.join("\n")), + ); + result.insert("input".to_string(), Value::Array(input)); + + // Rename max_tokens → max_output_tokens + if let Some(max_tokens) = obj.get("max_tokens") { + result.insert("max_output_tokens".to_string(), max_tokens.clone()); + } + + // Pass through compatible fields + for &field in PASSTHROUGH_FIELDS { + if let Some(value) = obj.get(field) { + result.insert(field.to_string(), value.clone()); + } + } + + // Strip incompatible fields — they are simply not copied over. + // (No action needed since we build a new object.) + let _ = STRIP_FIELDS; // acknowledge the constant is used by design + + Ok(serde_json::to_vec(&Value::Object(result))?) +} + +/// Convert a chat/completions message to a Responses API input item. +/// - Adds `type: "message"` (required by Responses API) +/// - For user messages: converts content `type: "text"` → `type: "input_text"` +/// - For assistant messages: converts content `type: "text"` → `type: "output_text"` +/// - Converts content `type: "image_url"` → `type: "input_image"` +/// - String content is left as-is (the Responses API accepts string content directly). +fn convert_message_content(mut msg: Value) -> Value { + let role = msg.get("role").and_then(Value::as_str).unwrap_or(""); + let is_assistant = role == "assistant"; + + // Responses API requires "type": "message" on each input item + if let Some(obj) = msg.as_object_mut() { + if !obj.contains_key("type") { + obj.insert("type".to_string(), Value::String("message".to_string())); + } + } + + let Some(content) = msg.get_mut("content") else { + return msg; + }; + let Some(parts) = content.as_array_mut() else { + // String content — no conversion needed + return msg; + }; + for part in parts.iter_mut() { + let Some(obj) = part.as_object_mut() else { + continue; + }; + match obj.get("type").and_then(Value::as_str) { + Some("text") => { + let text_type = if is_assistant { + "output_text" + } else { + "input_text" + }; + obj.insert("type".to_string(), Value::String(text_type.to_string())); + } + Some("image_url") => { + obj.insert("type".to_string(), Value::String("input_image".to_string())); + } + _ => {} + } + } + msg +} + +/// Translate a `/v1/responses` response body to `/v1/chat/completions` format. +pub fn responses_to_chat_completion(body: &[u8]) -> Result, TranslateError> { + let resp: Value = serde_json::from_slice(body)?; + let obj = resp + .as_object() + .ok_or(TranslateError::MissingField("root object"))?; + + let id = obj + .get("id") + .and_then(Value::as_str) + .unwrap_or("chatcmpl-translate"); + let model = obj + .get("model") + .and_then(Value::as_str) + .unwrap_or("unknown"); + + // Extract text content from output[].content[].text where type == "output_text" + let mut content_parts: Vec<&str> = Vec::new(); + if let Some(output) = obj.get("output").and_then(Value::as_array) { + for item in output { + if item.get("type").and_then(Value::as_str) == Some("message") { + if let Some(content) = item.get("content").and_then(Value::as_array) { + for part in content { + if part.get("type").and_then(Value::as_str) == Some("output_text") { + if let Some(text) = part.get("text").and_then(Value::as_str) { + content_parts.push(text); + } + } + } + } + } + } + } + let content = content_parts.join(""); + + // Map status → finish_reason + let finish_reason = match obj.get("status").and_then(Value::as_str) { + Some("completed") | None => "stop", + Some("incomplete") => "length", + Some("failed") => "stop", + Some(_) => "stop", + }; + + // Map usage + let usage = if let Some(u) = obj.get("usage") { + serde_json::json!({ + "prompt_tokens": u.get("input_tokens").and_then(Value::as_u64).unwrap_or(0), + "completion_tokens": u.get("output_tokens").and_then(Value::as_u64).unwrap_or(0), + "total_tokens": + u.get("input_tokens").and_then(Value::as_u64).unwrap_or(0) + + u.get("output_tokens").and_then(Value::as_u64).unwrap_or(0) + }) + } else { + serde_json::json!({ "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }) + }; + + let result = serde_json::json!({ + "id": id, + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": content, + }, + "finish_reason": finish_reason, + }], + "usage": usage, + }); + + Ok(serde_json::to_vec(&result)?) +} + +/// Translate a single SSE line from Responses API format to chat.completion.chunk format. +/// +/// Returns `Some(line(s))` for events that map to chat completions output, +/// or `None` for events that should be suppressed. +/// +/// `response_id` and `model` are captured from early events and reused in later chunks. +pub fn translate_sse_line( + line: &str, + response_id: &mut Option, + model: &mut Option, +) -> Option { + // Pass through [DONE] + if line.starts_with("data: [DONE]") { + return Some(line.to_string()); + } + + // Only process data: lines with JSON + let json_str = line.strip_prefix("data: ")?; + + let event: Value = serde_json::from_str(json_str).ok()?; + let event_type = event.get("type").and_then(Value::as_str)?; + + match event_type { + "response.created" | "response.in_progress" => { + // Capture response ID and model from these early events + if let Some(resp) = event.get("response") { + if let Some(id) = resp.get("id").and_then(Value::as_str) { + *response_id = Some(id.to_string()); + } + if let Some(m) = resp.get("model").and_then(Value::as_str) { + *model = Some(m.to_string()); + } + } + None // suppress + } + + "response.output_text.delta" => { + let delta = event.get("delta").and_then(Value::as_str).unwrap_or(""); + let id = response_id.as_deref().unwrap_or("chatcmpl-translate"); + let m = model.as_deref().unwrap_or("unknown"); + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": m, + "choices": [{ + "index": 0, + "delta": { "content": delta }, + "finish_reason": null, + }] + }); + Some(format!( + "data: {}", + serde_json::to_string(&chunk).unwrap_or_default() + )) + } + + "response.completed" => { + let id = response_id.as_deref().unwrap_or("chatcmpl-translate"); + let m = model.as_deref().unwrap_or("unknown"); + let final_chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": m, + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "stop", + }] + }); + Some(format!( + "data: {}\n\ndata: [DONE]", + serde_json::to_string(&final_chunk).unwrap_or_default() + )) + } + + "response.failed" => { + let id = response_id.as_deref().unwrap_or("chatcmpl-translate"); + let m = model.as_deref().unwrap_or("unknown"); + let final_chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": m, + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "stop", + }] + }); + Some(format!( + "data: {}\n\ndata: [DONE]", + serde_json::to_string(&final_chunk).unwrap_or_default() + )) + } + + "response.incomplete" => { + let id = response_id.as_deref().unwrap_or("chatcmpl-translate"); + let m = model.as_deref().unwrap_or("unknown"); + let final_chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": m, + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "length", + }] + }); + Some(format!( + "data: {}\n\ndata: [DONE]", + serde_json::to_string(&final_chunk).unwrap_or_default() + )) + } + + // Suppress all structural/metadata events + "response.output_text.done" + | "response.content_part.added" + | "response.content_part.done" + | "response.output_item.added" + | "response.output_item.done" => None, + + // Suppress any other unknown events + _ => None, + } +} + +/// A stream adapter that wraps an axum Body and translates Responses API SSE events +/// to chat.completion.chunk format. +pub struct TranslateStream { + inner: Pin> + Send>>, + buffer: BytesMut, + response_id: Option, + model: Option, + output_buffer: Vec, +} + +impl std::fmt::Debug for TranslateStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TranslateStream") + .field("buffer_len", &self.buffer.len()) + .field("response_id", &self.response_id) + .field("model", &self.model) + .finish() + } +} + +impl TranslateStream { + pub fn new(body: Body) -> Self { + use futures_util::StreamExt; + use http_body_util::BodyStream; + + let stream = BodyStream::new(body).filter_map(|result| async move { + match result { + Ok(frame) => frame.into_data().ok().map(Ok), + Err(e) => Some(Err(e)), + } + }); + + Self { + inner: Box::pin(stream), + buffer: BytesMut::new(), + response_id: None, + model: None, + output_buffer: Vec::new(), + } + } + + fn process_buffered_lines(&mut self) { + loop { + let Some(pos) = self.buffer.iter().position(|&b| b == b'\n') else { + break; + }; + + let line_bytes = self.buffer.split_to(pos + 1); + let line = String::from_utf8_lossy(&line_bytes).trim().to_string(); + + if line.is_empty() { + self.output_buffer.extend_from_slice(b"\n"); + continue; + } + + let rid = &mut self.response_id; + let mdl = &mut self.model; + if let Some(translated) = translate_sse_line(&line, rid, mdl) { + self.output_buffer.extend_from_slice(translated.as_bytes()); + self.output_buffer.extend_from_slice(b"\n\n"); + } + } + } +} + +impl Stream for TranslateStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + // First, drain any pending output + if !this.output_buffer.is_empty() { + let data = std::mem::take(&mut this.output_buffer); + return Poll::Ready(Some(Ok(Bytes::from(data)))); + } + + // Poll the inner stream for more data + match this.inner.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(chunk))) => { + this.buffer.extend_from_slice(&chunk); + this.process_buffered_lines(); + // Loop to check if we produced output + } + Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), + Poll::Ready(None) => { + // Stream ended — process any remaining buffer + if !this.buffer.is_empty() { + let remaining = std::mem::take(&mut this.buffer); + let line = String::from_utf8_lossy(&remaining).trim().to_string(); + if !line.is_empty() { + if let Some(translated) = + translate_sse_line(&line, &mut this.response_id, &mut this.model) + { + return Poll::Ready(Some(Ok(Bytes::from(format!( + "{translated}\n\n" + ))))); + } + } + } + return Poll::Ready(None); + } + Poll::Pending => return Poll::Pending, + } + } + } +} + +/// Wrap a Body in a TranslateStream and return a new Body. +pub fn wrap_body_with_translate_stream(body: Body) -> Body { + Body::from_stream(TranslateStream::new(body)) +} + +// --------------------------------------------------------------------------- +// DLP scanning for SSE streams +// --------------------------------------------------------------------------- + +/// Apply DLP redaction to a single SSE `data:` line. +/// +/// Parses the JSON, extracts `choices[0].delta.content`, runs redaction on it, +/// and patches the JSON back if any PII was found. Returns the (possibly +/// modified) line. +/// +/// Lines that are not `data:` JSON or don't contain delta content are returned +/// unchanged. +pub fn redact_sse_data_line(line: &str, scanner: &DlpScanner) -> String { + // Only process data: lines with JSON + let Some(json_str) = line.strip_prefix("data: ") else { + return line.to_string(); + }; + + // Don't touch [DONE] + if json_str.starts_with("[DONE]") { + return line.to_string(); + } + + let Ok(mut event) = serde_json::from_str::(json_str) else { + return line.to_string(); + }; + + // Extract delta.content from choices[0] + let Some(content) = event + .get_mut("choices") + .and_then(Value::as_array_mut) + .and_then(|choices| choices.first_mut()) + .and_then(|choice| choice.get_mut("delta")) + .and_then(|delta| delta.get_mut("content")) + else { + return line.to_string(); + }; + + let Some(text) = content.as_str() else { + return line.to_string(); + }; + + let (redacted, redacted_names) = scanner.redact_all(text.as_bytes()); + if redacted_names.is_empty() { + return line.to_string(); + } + + warn!( + redacted_patterns = ?redacted_names, + "PII redacted from streaming SSE chunk" + ); + + let redacted_str = String::from_utf8_lossy(&redacted); + *content = Value::String(redacted_str.into_owned()); + format!( + "data: {}", + serde_json::to_string(&event).unwrap_or_else(|_| json_str.to_string()) + ) +} + +/// Stream adapter that applies DLP redaction to SSE data lines. +pub struct DlpSseStream { + inner: Pin> + Send>>, + buffer: BytesMut, + scanner: Arc, + output_buffer: Vec, +} + +impl DlpSseStream { + pub fn new(body: Body, scanner: Arc) -> Self { + use futures_util::StreamExt; + use http_body_util::BodyStream; + + let stream = BodyStream::new(body).filter_map(|result| async move { + match result { + Ok(frame) => frame.into_data().ok().map(Ok), + Err(e) => Some(Err(e)), + } + }); + + Self { + inner: Box::pin(stream), + buffer: BytesMut::new(), + scanner, + output_buffer: Vec::new(), + } + } + + fn process_buffered_lines(&mut self) { + loop { + let Some(pos) = self.buffer.iter().position(|&b| b == b'\n') else { + break; + }; + + let line_bytes = self.buffer.split_to(pos + 1); + let line = String::from_utf8_lossy(&line_bytes).trim().to_string(); + + if line.is_empty() { + self.output_buffer.extend_from_slice(b"\n"); + continue; + } + + let redacted = redact_sse_data_line(&line, &self.scanner); + self.output_buffer.extend_from_slice(redacted.as_bytes()); + self.output_buffer.extend_from_slice(b"\n"); + } + } +} + +impl Stream for DlpSseStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + if !this.output_buffer.is_empty() { + let data = std::mem::take(&mut this.output_buffer); + return Poll::Ready(Some(Ok(Bytes::from(data)))); + } + + match this.inner.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(chunk))) => { + this.buffer.extend_from_slice(&chunk); + this.process_buffered_lines(); + } + Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), + Poll::Ready(None) => { + if !this.buffer.is_empty() { + let remaining = std::mem::take(&mut this.buffer); + let line = String::from_utf8_lossy(&remaining).trim().to_string(); + if !line.is_empty() { + let redacted = redact_sse_data_line(&line, &this.scanner); + return Poll::Ready(Some(Ok(Bytes::from(format!("{redacted}\n"))))); + } + } + return Poll::Ready(None); + } + Poll::Pending => return Poll::Pending, + } + } + } +} + +/// Wrap a Body in a DlpSseStream for streaming DLP redaction. +pub fn wrap_body_with_dlp_sse_stream(body: Body, scanner: Arc) -> Body { + Body::from_stream(DlpSseStream::new(body, scanner)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_chat_to_responses_basic() { + let body = serde_json::json!({ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "say hi"} + ] + }); + let result = chat_completions_to_responses(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["model"], "gpt-4o-mini"); + assert_eq!( + parsed["instructions"], "", + "instructions should be empty when no system messages" + ); + let input = parsed["input"].as_array().unwrap(); + assert_eq!(input.len(), 1); + assert_eq!(input[0]["type"], "message"); + assert_eq!(input[0]["role"], "user"); + assert_eq!(input[0]["content"], "say hi"); + assert!(parsed.get("messages").is_none()); + } + + #[test] + fn test_chat_to_responses_with_system() { + let body = serde_json::json!({ + "model": "gpt-4o", + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "hello"} + ] + }); + let result = chat_completions_to_responses(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["instructions"], "You are helpful."); + let input = parsed["input"].as_array().unwrap(); + assert_eq!(input.len(), 1); + assert_eq!(input[0]["role"], "user"); + } + + #[test] + fn test_chat_to_responses_multiple_system() { + let body = serde_json::json!({ + "model": "gpt-4o", + "messages": [ + {"role": "system", "content": "Be concise."}, + {"role": "system", "content": "Use markdown."}, + {"role": "user", "content": "hello"} + ] + }); + let result = chat_completions_to_responses(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["instructions"], "Be concise.\nUse markdown."); + } + + #[test] + fn test_chat_to_responses_max_tokens() { + let body = serde_json::json!({ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 100 + }); + let result = chat_completions_to_responses(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["max_output_tokens"], 100); + assert!(parsed.get("max_tokens").is_none()); + } + + #[test] + fn test_chat_to_responses_strips_unsupported() { + let body = serde_json::json!({ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": 0.5, + "presence_penalty": 0.5, + "logprobs": true, + "top_logprobs": 5, + "logit_bias": {"123": 1}, + "n": 2, + "response_format": {"type": "json_object"}, + "seed": 42, + "service_tier": "default", + "user": "user-123" + }); + let result = chat_completions_to_responses(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + for field in STRIP_FIELDS { + assert!( + parsed.get(*field).is_none(), + "field '{}' should be stripped", + field + ); + } + } + + #[test] + fn test_chat_to_responses_passthrough() { + let body = serde_json::json!({ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "stream": true, + "temperature": 0.7, + "top_p": 0.9, + "stop": ["\n"] + }); + let result = chat_completions_to_responses(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["model"], "gpt-4o-mini"); + assert_eq!(parsed["stream"], true); + assert_eq!(parsed["temperature"], 0.7); + assert_eq!(parsed["top_p"], 0.9); + assert_eq!(parsed["stop"], serde_json::json!(["\n"])); + } + + #[test] + fn test_responses_to_chat_completion_basic() { + let body = serde_json::json!({ + "id": "resp_abc123", + "model": "gpt-4o-mini", + "status": "completed", + "output": [{ + "type": "message", + "content": [{ + "type": "output_text", + "text": "Hello!" + }] + }], + "usage": { + "input_tokens": 10, + "output_tokens": 5 + } + }); + let result = responses_to_chat_completion(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["id"], "resp_abc123"); + assert_eq!(parsed["object"], "chat.completion"); + assert_eq!(parsed["model"], "gpt-4o-mini"); + let choice = &parsed["choices"][0]; + assert_eq!(choice["message"]["role"], "assistant"); + assert_eq!(choice["message"]["content"], "Hello!"); + assert_eq!(choice["finish_reason"], "stop"); + } + + #[test] + fn test_responses_to_chat_completion_usage() { + let body = serde_json::json!({ + "id": "resp_abc", + "model": "gpt-4o", + "status": "completed", + "output": [{ + "type": "message", + "content": [{"type": "output_text", "text": "hi"}] + }], + "usage": { + "input_tokens": 50, + "output_tokens": 25 + } + }); + let result = responses_to_chat_completion(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["usage"]["prompt_tokens"], 50); + assert_eq!(parsed["usage"]["completion_tokens"], 25); + assert_eq!(parsed["usage"]["total_tokens"], 75); + } + + #[test] + fn test_responses_to_chat_completion_incomplete() { + let body = serde_json::json!({ + "id": "resp_inc", + "model": "gpt-4o", + "status": "incomplete", + "output": [{ + "type": "message", + "content": [{"type": "output_text", "text": "partial"}] + }], + "usage": { "input_tokens": 10, "output_tokens": 5 } + }); + let result = responses_to_chat_completion(body.to_string().as_bytes()).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + + assert_eq!(parsed["choices"][0]["finish_reason"], "length"); + } + + #[test] + fn test_sse_delta() { + let event = serde_json::json!({ + "type": "response.output_text.delta", + "delta": "Hello" + }); + let line = format!("data: {}", event); + let mut response_id = Some("resp_123".to_string()); + let mut model = Some("gpt-4o-mini".to_string()); + let result = translate_sse_line(&line, &mut response_id, &mut model).unwrap(); + + assert!(result.starts_with("data: ")); + let json_str = result.strip_prefix("data: ").unwrap(); + let parsed: Value = serde_json::from_str(json_str).unwrap(); + + assert_eq!(parsed["object"], "chat.completion.chunk"); + assert_eq!(parsed["id"], "resp_123"); + assert_eq!(parsed["model"], "gpt-4o-mini"); + assert_eq!(parsed["choices"][0]["delta"]["content"], "Hello"); + assert!(parsed["choices"][0]["finish_reason"].is_null()); + } + + #[test] + fn test_sse_completed() { + let event = serde_json::json!({ + "type": "response.completed", + "response": {"id": "resp_456", "status": "completed"} + }); + let line = format!("data: {}", event); + let mut response_id = Some("resp_456".to_string()); + let mut model = Some("gpt-4o".to_string()); + let result = translate_sse_line(&line, &mut response_id, &mut model).unwrap(); + + // Should contain a final chunk with finish_reason: "stop" and then [DONE] + assert!(result.contains("\"finish_reason\":\"stop\"")); + assert!(result.contains("data: [DONE]")); + } + + #[test] + fn test_sse_meta_suppressed() { + let mut response_id = None; + let mut model = None; + + let created = serde_json::json!({ + "type": "response.created", + "response": {"id": "resp_789", "model": "gpt-4o"} + }); + let result = + translate_sse_line(&format!("data: {}", created), &mut response_id, &mut model); + assert!(result.is_none()); + assert_eq!(response_id.as_deref(), Some("resp_789")); + assert_eq!(model.as_deref(), Some("gpt-4o")); + + let in_progress = serde_json::json!({ + "type": "response.in_progress", + "response": {"id": "resp_789"} + }); + let result = translate_sse_line( + &format!("data: {}", in_progress), + &mut response_id, + &mut model, + ); + assert!(result.is_none()); + + // Structural events should also be suppressed + let content_part = serde_json::json!({"type": "response.content_part.added"}); + let result = translate_sse_line( + &format!("data: {}", content_part), + &mut response_id, + &mut model, + ); + assert!(result.is_none()); + } + + #[test] + fn test_sse_done_passthrough() { + let mut response_id = None; + let mut model = None; + let result = translate_sse_line("data: [DONE]", &mut response_id, &mut model); + assert_eq!(result, Some("data: [DONE]".to_string())); + } + + #[test] + fn test_chat_to_responses_multipart_content_types() { + let body = serde_json::to_vec(&serde_json::json!({ + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/img.png"}} + ] + } + ] + })) + .unwrap(); + + let result = chat_completions_to_responses(&body).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + assert_eq!(parsed["input"][0]["type"], "message"); + let content = parsed["input"][0]["content"].as_array().unwrap(); + assert_eq!(content[0]["type"], "input_text"); + assert_eq!(content[0]["text"], "What is in this image?"); + assert_eq!(content[1]["type"], "input_image"); + } + + #[test] + fn test_chat_to_responses_string_content_unchanged() { + let body = serde_json::to_vec(&serde_json::json!({ + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "hello"} + ] + })) + .unwrap(); + + let result = chat_completions_to_responses(&body).unwrap(); + let parsed: Value = serde_json::from_slice(&result).unwrap(); + assert_eq!(parsed["input"][0]["type"], "message"); + assert_eq!(parsed["input"][0]["role"], "user"); + assert_eq!(parsed["input"][0]["content"], "hello"); + } + + // ----------------------------------------------------------------------- + // DLP SSE redaction tests + // ----------------------------------------------------------------------- + + fn test_dlp_scanner() -> DlpScanner { + use crate::config::{DlpAction, DlpPattern}; + DlpScanner::new( + &[ + DlpPattern { + name: "email".to_string(), + regex: r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b".to_string(), + action: DlpAction::Redact, + }, + DlpPattern { + name: "ssn".to_string(), + regex: r"\b\d{3}-\d{2}-\d{4}\b".to_string(), + action: DlpAction::Block, + }, + ], + true, + ) + .unwrap() + } + + #[test] + fn test_redact_sse_data_line_with_pii() { + let scanner = test_dlp_scanner(); + let line = r#"data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Contact user@example.com for info"},"finish_reason":null}]}"#; + let result = redact_sse_data_line(line, &scanner); + assert!( + result.starts_with("data: "), + "Should still be an SSE data line" + ); + assert!( + result.contains("[REDACTED:email]"), + "Email should be redacted" + ); + assert!( + !result.contains("user@example.com"), + "Original email should be gone" + ); + } + + #[test] + fn test_redact_sse_data_line_clean() { + let scanner = test_dlp_scanner(); + let line = r#"data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Hello world"},"finish_reason":null}]}"#; + let result = redact_sse_data_line(line, &scanner); + assert_eq!(result, line, "Clean content should pass through unchanged"); + } + + #[test] + fn test_redact_sse_data_line_done() { + let scanner = test_dlp_scanner(); + let result = redact_sse_data_line("data: [DONE]", &scanner); + assert_eq!(result, "data: [DONE]"); + } + + #[test] + fn test_redact_sse_data_line_no_delta_content() { + let scanner = test_dlp_scanner(); + let line = r#"data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#; + let result = redact_sse_data_line(line, &scanner); + assert_eq!( + result, line, + "Lines without delta.content pass through unchanged" + ); + } + + #[test] + fn test_redact_sse_data_line_non_data_line() { + let scanner = test_dlp_scanner(); + let result = redact_sse_data_line("event: message", &scanner); + assert_eq!( + result, "event: message", + "Non-data lines pass through unchanged" + ); + } +} diff --git a/tests/snapshots/config_fixtures__all_fields.snap b/tests/snapshots/config_fixtures__all_fields.snap index ea04598..23c1fca 100644 --- a/tests/snapshots/config_fixtures__all_fields.snap +++ b/tests/snapshots/config_fixtures__all_fields.snap @@ -13,9 +13,11 @@ keys: - virtual_key: vk-1 real_key: sk-real-1 provider: openai + auth: static - virtual_key: vk-2 real_key: sk-real-2 provider: anthropic + auth: static dlp: patterns: - name: ssn diff --git a/tests/snapshots/config_fixtures__empty_keys.snap b/tests/snapshots/config_fixtures__empty_keys.snap index df56020..2a114e0 100644 --- a/tests/snapshots/config_fixtures__empty_keys.snap +++ b/tests/snapshots/config_fixtures__empty_keys.snap @@ -13,6 +13,7 @@ keys: - virtual_key: "" real_key: "" provider: openai + auth: static dlp: patterns: [] scan_responses: true diff --git a/tests/snapshots/config_fixtures__key_missing_real_key.snap b/tests/snapshots/config_fixtures__key_missing_real_key.snap index 29f035d..5aaa8e8 100644 --- a/tests/snapshots/config_fixtures__key_missing_real_key.snap +++ b/tests/snapshots/config_fixtures__key_missing_real_key.snap @@ -2,8 +2,4 @@ source: tests/config_fixtures.rs expression: err.to_string() --- -TOML parse error at line 4, column 1 - | -4 | [[keys]] - | ^^^^^^^^ -missing field `real_key` +key 'vk-1': real_key is required when auth = "static"