From 4227ce543c642cf3be89107547e0ce72d4f8fdbc Mon Sep 17 00:00:00 2001 From: AptS-2077 Date: Mon, 6 Jul 2026 16:15:58 +0800 Subject: [PATCH 1/5] feat: add SFTP storage backend --- Cargo.lock | 1156 +++++++++++++++-- Cargo.toml | 2 + .../components/admin/StoragePolicyDialog.tsx | 12 +- .../policyPresentation.test.ts | 1 + .../admin-policies-page/policyPresentation.ts | 1 + .../StoragePolicyCreateWizard.tsx | 5 +- .../StoragePolicyEditForm.tsx | 5 +- .../StoragePolicyObjectStorageFields.tsx | 37 +- .../connectionNormalization.ts | 88 +- .../descriptorPredicates.ts | 11 + .../src/i18n/locales/en/admin/policies.json | 8 + .../src/i18n/locales/zh/admin/policies.json | 8 + .../pages/admin/AdminPoliciesPage.test.tsx | 42 + .../src/pages/admin/AdminPoliciesPage.tsx | 10 +- frontend-panel/src/services/api.generated.ts | 2 +- src/storage/connectors/mod.rs | 42 +- src/storage/connectors/sftp.rs | 159 +++ src/storage/connectors/tests.rs | 121 +- src/storage/connectors/upload.rs | 9 +- src/storage/drivers/mod.rs | 1 + src/storage/drivers/sftp.rs | 839 ++++++++++++ src/storage/registry.rs | 5 + src/types/storage_policy.rs | 4 + 23 files changed, 2410 insertions(+), 158 deletions(-) create mode 100644 src/storage/connectors/sftp.rs create mode 100644 src/storage/drivers/sftp.rs diff --git a/Cargo.lock b/Cargo.lock index 34400a227..749a35c9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,7 +264,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common 0.1.7", - "generic-array", + "generic-array 0.14.7", +] + +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.2", ] [[package]] @@ -274,31 +284,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] +[[package]] +name = "aes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.0", + "zeroize", +] + [[package]] name = "aes-gcm" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", + "aead 0.5.2", + "aes 0.8.4", + "cipher 0.4.4", + "ctr 0.9.2", + "ghash 0.5.1", "subtle", ] +[[package]] +name = "aes-gcm" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" +dependencies = [ + "aead 0.6.1", + "aes 0.9.1", + "cipher 0.5.2", + "ctr 0.10.1", + "ghash 0.6.0", + "subtle", + "zeroize", +] + [[package]] name = "aes-kw" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69fa2b352dcefb5f7f3a5fb840e02665d311d878955380515e4fd50095dd3d8c" dependencies = [ - "aes", + "aes 0.8.4", ] [[package]] @@ -482,9 +519,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", - "blake2", + "blake2 0.10.6", "cpufeatures 0.2.17", - "password-hash", + "password-hash 0.5.0", +] + +[[package]] +name = "argon2" +version = "0.6.0-rc.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +dependencies = [ + "base64ct", + "blake2 0.11.0-rc.6", + "cpufeatures 0.3.0", + "password-hash 0.6.1", ] [[package]] @@ -710,9 +759,9 @@ dependencies = [ "actix-multipart", "actix-web", "actix-ws", - "aes-gcm", + "aes-gcm 0.10.3", "api-docs-macros", - "argon2", + "argon2 0.5.3", "async-stream", "async-trait", "aws-credential-types", @@ -763,7 +812,9 @@ dependencies = [ "redis", "reqwest 0.13.4", "ring", - "rsa", + "rsa 0.9.10", + "russh", + "russh-sftp", "rust-embed", "sea-orm", "secrecy", @@ -991,13 +1042,13 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac 0.13.0", "http 0.2.12", "http 1.4.2", - "p256", + "p256 0.13.2", "percent-encoding", "sha2 0.11.0", "subtle", @@ -1368,6 +1419,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.21.7" @@ -1396,6 +1453,17 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bcrypt-pbkdf" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144e573728da132683b9488acd528274c790e07fc06ff81ee29f9d8f8b1041e0" +dependencies = [ + "blowfish", + "pbkdf2 0.13.0", + "sha2 0.11.0", +] + [[package]] name = "bigdecimal" version = "0.4.10" @@ -1452,13 +1520,22 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.11.0-rc.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1468,6 +1545,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", + "zeroize", ] [[package]] @@ -1476,7 +1554,26 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array", + "generic-array 0.14.7", +] + +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blowfish" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ce3946557b35e71d1bbe07ec385073ce9eda05043f95de134eb578fcf1a298" +dependencies = [ + "byteorder", + "cipher 0.5.2", ] [[package]] @@ -1696,7 +1793,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher 0.5.2", ] [[package]] @@ -1749,8 +1855,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", + "cipher 0.5.2", "cpufeatures 0.3.0", "rand_core 0.10.1", + "zeroize", ] [[package]] @@ -1811,7 +1919,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.2", + "zeroize", ] [[package]] @@ -1994,6 +2114,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2134,19 +2260,36 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "serdect", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "typenum", ] @@ -2157,6 +2300,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ + "getrandom 0.4.3", "hybrid-array", "rand_core 0.10.1", ] @@ -2167,40 +2311,40 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "842583c7b5969afdd00c31fd4ffe23f084d62ea3bc07a2132a2a3c4340ab1d09" dependencies = [ - "aes", - "aes-gcm", + "aes 0.8.4", + "aes-gcm 0.10.3", "aes-kw", - "argon2", + "argon2 0.5.3", "base64 0.22.1", - "cbc", - "cipher", + "cbc 0.1.2", + "cipher 0.4.4", "const-oid 0.9.6", "crypto-common 0.1.7", "crypto-common 0.2.2", - "der", + "der 0.7.10", "digest 0.11.3", - "ecdsa", - "elliptic-curve", - "generic-array", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "generic-array 0.14.7", "hex", "hkdf 0.12.4", "hmac 0.12.1", "hmac 0.13.0", "hybrid-array", "kbkdf", - "p256", - "p384", - "p521", - "pbkdf2", - "pkcs8", + "p256 0.13.2", + "p384 0.13.1", + "p521 0.13.3", + "pbkdf2 0.12.2", + "pkcs8 0.10.2", "rand 0.8.6", - "rsa", - "sec1", + "rsa 0.9.10", + "sec1 0.7.3", "serde", "sha1 0.10.6", "sha2 0.10.9", "sha2 0.11.0", - "spki", + "spki 0.7.3", "subtle", "tracing", "uuid", @@ -2208,13 +2352,32 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-primes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" +dependencies = [ + "crypto-bigint 0.7.5", + "rand_core 0.10.1", +] + [[package]] name = "ctr" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher 0.5.2", ] [[package]] @@ -2224,6 +2387,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" dependencies = [ "cmov", + "subtle", ] [[package]] @@ -2236,7 +2400,23 @@ dependencies = [ "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", - "fiat-crypto", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c906a87e53a36ff795d72e06e8162a83c5436e3ea89e942a9cb9fc083f0a384f" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", "rustc_version", "subtle", "zeroize", @@ -2342,6 +2522,17 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "delegate" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "der" version = "0.7.10" @@ -2351,7 +2542,18 @@ dependencies = [ "const-oid 0.9.6", "der_derive", "flagset", - "pem-rfc7468", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468 1.0.0", "zeroize", ] @@ -2423,6 +2625,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "des" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +dependencies = [ + "cipher 0.5.2", +] + [[package]] name = "diff" version = "0.1.13" @@ -2499,12 +2710,27 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", +] + +[[package]] +name = "ecdsa" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" +dependencies = [ + "der 0.8.0", + "digest 0.11.3", + "elliptic-curve 0.14.1", + "rfc6979 0.6.0", + "signature 3.0.0", + "spki 0.8.0", + "zeroize", ] [[package]] @@ -2513,8 +2739,18 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8 0.11.0", + "signature 3.0.0", ] [[package]] @@ -2523,14 +2759,30 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ - "curve25519-dalek", - "ed25519", + "curve25519-dalek 4.1.3", + "ed25519 2.2.3", "serde", "sha2 0.10.9", "subtle", "zeroize", ] +[[package]] +name = "ed25519-dalek" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1685663e23882cd8517dcbcb1c23a6ebff4433c22dfb681d760219b62cd1b849" +dependencies = [ + "curve25519-dalek 5.0.0-rc.1", + "ed25519 3.0.0", + "rand_core 0.10.1", + "serde", + "sha2 0.11.0", + "signature 3.0.0", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.16.0" @@ -2546,17 +2798,39 @@ version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest 0.10.7", - "ff", - "generic-array", - "group", + "ff 0.13.1", + "generic-array 0.14.7", + "group 0.13.0", "hkdf 0.12.4", - "pem-rfc7468", - "pkcs8", + "pem-rfc7468 0.7.0", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" +dependencies = [ + "base16ct 1.0.0", + "crypto-bigint 0.7.5", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff 0.14.0", + "group 0.14.0", + "hkdf 0.13.0", + "hybrid-array", + "pem-rfc7468 1.0.0", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1 0.8.1", "subtle", "zeroize", ] @@ -2586,6 +2860,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2689,12 +2975,28 @@ dependencies = [ "subtle", ] +[[package]] +name = "ff" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "fiat-crypto" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "filetime" version = "0.2.29" @@ -2899,6 +3201,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic-array" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e55f16dcf0e9c00efbe2e655ffe45fc98e7066b52bc92f8a79e64060a79351" +dependencies = [ + "generic-array 0.14.7", + "rustversion", + "typenum", +] + [[package]] name = "geo-types" version = "0.7.19" @@ -2944,9 +3257,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -2956,7 +3271,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ "opaque-debug", - "polyval", + "polyval 0.6.2", +] + +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval 0.7.1", ] [[package]] @@ -2975,6 +3299,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482ce8a491a501da4cd806bd190275363d674f2845005c6ddbd5d3e1dd54495d" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "governor" version = "0.10.4" @@ -3004,11 +3340,22 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "h2" version = "0.3.27" @@ -3124,6 +3471,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-literal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" + [[package]] name = "hkdf" version = "0.12.4" @@ -3238,11 +3591,13 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ + "ctutils", "serde", + "subtle", "typenum", "zeroize", ] @@ -3562,8 +3917,30 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", - "generic-array", + "block-padding 0.3.3", + "generic-array 0.14.7", +] + +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding 0.4.2", + "hybrid-array", +] + +[[package]] +name = "internal-russh-num-bigint" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8" +dependencies = [ + "num-integer", + "num-traits", + "rand 0.10.1", + "rand_core 0.10.1", ] [[package]] @@ -3704,7 +4081,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature", + "signature 2.2.0", "simple_asn1", "zeroize", ] @@ -3718,6 +4095,26 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + +[[package]] +name = "kem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01737161ba802849cfd486b5bd209d38ba4943494c249a8126005170c7621edd" +dependencies = [ + "crypto-common 0.2.2", + "rand_core 0.10.1", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -3937,7 +4334,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" dependencies = [ - "nix", + "nix 0.29.0", "serde", "winapi", ] @@ -3967,6 +4364,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + [[package]] name = "memchr" version = "2.8.2" @@ -4036,6 +4439,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ml-kem" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e15f3e5b957493873e396a66914e83e616b6afe335cdef7efe5c6e1216aba66" +dependencies = [ + "hybrid-array", + "kem", + "module-lattice", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sha3", +] + +[[package]] +name = "module-lattice" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" +dependencies = [ + "ctutils", + "hybrid-array", + "num-traits", +] + [[package]] name = "moka" version = "0.12.15" @@ -4079,6 +4507,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -4353,16 +4793,16 @@ dependencies = [ "base64 0.21.7", "chrono", "dyn-clone", - "ed25519-dalek", + "ed25519-dalek 2.2.0", "hmac 0.12.1", "http 1.4.2", "itertools 0.10.5", "log", "oauth2", - "p256", - "p384", + "p256 0.13.2", + "p384 0.13.1", "rand 0.8.6", - "rsa", + "rsa 0.9.10", "serde", "serde-value", "serde_json", @@ -4435,38 +4875,99 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", "sha2 0.10.9", ] +[[package]] +name = "p256" +version = "0.14.0-rc.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bb40a5099e2c38a09dd29321a7a7f045f165a54317679c7cdfb0cbaf8f6b1e" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.1", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.0", +] + [[package]] name = "p384" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", "sha2 0.10.9", ] +[[package]] +name = "p384" +version = "0.14.0-rc.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "492f329d7eb11d22dadc988626b9ea1f503b4ab043a8b1e4e2cc4ae45dabdd70" +dependencies = [ + "ecdsa 0.17.0", + "elliptic-curve 0.14.1", + "fiat-crypto 0.3.0", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.0", +] + [[package]] name = "p521" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" dependencies = [ - "base16ct", - "ecdsa", - "elliptic-curve", - "primeorder", + "base16ct 0.2.0", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", "rand_core 0.6.4", "sha2 0.10.9", ] +[[package]] +name = "p521" +version = "0.14.0-rc.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff42e4ace5424e3b6d7cb82514be89866b85af87015f80341e37dcc21a66ce6e" +dependencies = [ + "base16ct 1.0.0", + "ecdsa 0.17.0", + "elliptic-curve 0.14.1", + "primefield", + "primeorder 0.14.0", + "sha2 0.11.0", +] + +[[package]] +name = "pageant" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5" +dependencies = [ + "base16ct 1.0.0", + "byteorder", + "bytes", + "delegate", + "futures", + "log", + "rand 0.10.1", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "windows", + "windows-strings", +] + [[package]] name = "parking" version = "2.2.1" @@ -4532,6 +5033,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "password-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +dependencies = [ + "phc", +] + [[package]] name = "paste" version = "1.0.15" @@ -4554,21 +5064,40 @@ dependencies = [ "hmac 0.12.1", ] +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", + "hmac 0.13.0", +] + [[package]] name = "pem" version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.22.1", - "serde_core", + "base64 0.22.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", ] [[package]] name = "pem-rfc7468" -version = "0.7.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" dependencies = [ "base64ct", ] @@ -4588,6 +5117,16 @@ dependencies = [ "serde", ] +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", +] + [[package]] name = "phf" version = "0.11.3" @@ -4662,9 +5201,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs1" +version = "0.8.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e" +dependencies = [ + "der 0.8.0", + "spki 0.8.0", ] [[package]] @@ -4673,13 +5222,30 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ - "aes", - "cbc", - "der", - "pbkdf2", - "scrypt", + "aes 0.8.4", + "cbc 0.1.2", + "der 0.7.10", + "pbkdf2 0.12.2", + "scrypt 0.11.0", "sha2 0.10.9", - "spki", + "spki 0.7.3", +] + +[[package]] +name = "pkcs5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" +dependencies = [ + "aes 0.9.1", + "aes-gcm 0.11.0", + "cbc 0.2.1", + "der 0.8.0", + "pbkdf2 0.13.0", + "rand_core 0.10.1", + "scrypt 0.12.0", + "sha2 0.11.0", + "spki 0.8.0", ] [[package]] @@ -4688,10 +5254,22 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "pkcs5", + "der 0.7.10", + "pkcs5 0.7.1", "rand_core 0.6.4", - "spki", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.0", + "pkcs5 0.8.1", + "rand_core 0.10.1", + "spki 0.8.0", ] [[package]] @@ -4751,6 +5329,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "poly1305" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00baa632505d05512f48a963e16051c54fda9a95cc9acea1a4e3c90991c4a2e" +dependencies = [ + "cpufeatures 0.3.0", + "universal-hash 0.6.1", + "zeroize", +] + [[package]] name = "polyval" version = "0.6.2" @@ -4760,7 +5349,18 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", +] + +[[package]] +name = "polyval" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfc63250416fea14f5749b90725916a6c903f599d51cb635aa7a52bfd03eede" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash 0.6.1", ] [[package]] @@ -4803,13 +5403,40 @@ dependencies = [ "yansi", ] +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint 0.7.5", + "crypto-common 0.2.2", + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "primeorder" version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" +dependencies = [ + "elliptic-curve 0.14.1", + "once_cell", + "primefield", + "serdect", + "wnaf", ] [[package]] @@ -5366,6 +5993,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "rfc6979" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" +dependencies = [ + "crypto-bigint 0.7.5", + "hmac 0.13.0", +] + [[package]] name = "ring" version = "0.17.14" @@ -5429,16 +6066,151 @@ dependencies = [ "num-bigint-dig", "num-integer", "num-traits", - "pkcs1", - "pkcs8", + "pkcs1 0.7.5", + "pkcs8 0.10.2", "rand_core 0.6.4", "sha2 0.10.9", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "zeroize", +] + +[[package]] +name = "rsa" +version = "0.10.0-rc.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" +dependencies = [ + "const-oid 0.10.2", + "crypto-bigint 0.7.5", + "crypto-primes", + "digest 0.11.3", + "pkcs1 0.8.0-rc.4", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sha2 0.11.0", + "signature 3.0.0", + "spki 0.8.0", + "zeroize", +] + +[[package]] +name = "russh" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20d2039c3e50abb1ec3bdadea27098679da12a33befa9ab4e62adad9a13ebcb2" +dependencies = [ + "aes 0.9.1", + "aws-lc-rs", + "bitflags 2.13.0", + "block-padding 0.4.2", + "byteorder", + "bytes", + "cbc 0.2.1", + "cipher 0.5.2", + "crypto-bigint 0.7.5", + "ctr 0.10.1", + "curve25519-dalek 5.0.0-rc.1", + "data-encoding", + "delegate", + "der 0.8.0", + "digest 0.11.3", + "ecdsa 0.17.0", + "ed25519-dalek 3.0.0-rc.1", + "elliptic-curve 0.14.1", + "enum_dispatch", + "flate2", + "futures", + "generic-array 1.4.3", + "getrandom 0.4.3", + "ghash 0.6.0", + "hex-literal", + "hmac 0.13.0", + "inout 0.2.2", + "internal-russh-num-bigint", + "keccak", + "log", + "md5", + "ml-kem", + "module-lattice", + "num-bigint", + "p256 0.14.0-rc.15", + "p384 0.14.0-rc.15", + "p521 0.14.0-rc.15", + "pageant", + "pbkdf2 0.13.0", + "pkcs1 0.8.0-rc.4", + "pkcs5 0.8.1", + "pkcs8 0.11.0", + "polyval 0.7.1", + "rand 0.10.1", + "rand_core 0.10.1", + "rsa 0.10.0-rc.18", + "russh-cryptovec", + "russh-util", + "salsa20 0.11.0", + "scrypt 0.12.0", + "sec1 0.8.1", + "sha1 0.11.0", + "sha2 0.11.0", + "sha3", + "signature 3.0.0", + "spki 0.8.0", + "ssh-encoding", + "ssh-key", "subtle", + "thiserror 2.0.18", + "tokio", + "typenum", + "universal-hash 0.6.1", "zeroize", ] +[[package]] +name = "russh-cryptovec" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438" +dependencies = [ + "log", + "nix 0.31.3", + "ssh-encoding", + "windows-sys 0.61.2", +] + +[[package]] +name = "russh-sftp" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed8949eca4163c18a8f59ff96d32cf61e9c13b9735e21ef32b3907f4aafa1a9" +dependencies = [ + "bitflags 2.13.0", + "bytes", + "chrono", + "dashmap", + "gloo-timers", + "log", + "serde", + "serde_bytes", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "wasm-bindgen-futures", +] + +[[package]] +name = "russh-util" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668424a5dde0bcb45b55ba7de8476b93831b4aa2fa6947e145f3b053e22c60b6" +dependencies = [ + "chrono", + "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "rust-embed" version = "8.11.0" @@ -5622,7 +6394,17 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "salsa20" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" +dependencies = [ + "cfg-if", + "cipher 0.5.2", ] [[package]] @@ -5679,11 +6461,23 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ - "pbkdf2", - "salsa20", + "pbkdf2 0.12.2", + "salsa20 0.10.2", "sha2 0.10.9", ] +[[package]] +name = "scrypt" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87af57419b594aa23fa95f09f0e06d80d84ba01c26148c43844cad6ff4485f0" +dependencies = [ + "cfg-if", + "pbkdf2 0.13.0", + "salsa20 0.11.0", + "sha2 0.11.0", +] + [[package]] name = "sea-bae" version = "0.2.1" @@ -5872,10 +6666,24 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", + "base16ct 0.2.0", + "der 0.7.10", + "generic-array 0.14.7", + "pkcs8 0.10.2", + "subtle", + "zeroize", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct 1.0.0", + "ctutils", + "der 0.8.0", + "hybrid-array", "subtle", "zeroize", ] @@ -6082,6 +6890,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct 1.0.0", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -6132,6 +6950,16 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -6167,6 +6995,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.9" @@ -6273,7 +7111,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.0", ] [[package]] @@ -6383,7 +7231,7 @@ dependencies = [ "either", "futures-core", "futures-util", - "generic-array", + "generic-array 0.14.7", "log", "percent-encoding", "rust_decimal", @@ -6463,6 +7311,65 @@ dependencies = [ "uuid", ] +[[package]] +name = "ssh-cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597" +dependencies = [ + "aead 0.6.1", + "aes 0.9.1", + "aes-gcm 0.11.0", + "chacha20", + "cipher 0.5.2", + "ctutils", + "des", + "poly1305", + "ssh-encoding", + "zeroize", +] + +[[package]] +name = "ssh-encoding" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b54d0ed0498daf3f78d82e00e28c8eec9d75a067c4cfbcc7a0f7d0f4077749e" +dependencies = [ + "base64ct", + "bytes", + "crypto-bigint 0.7.5", + "ctutils", + "digest 0.11.3", + "pem-rfc7468 1.0.0", + "zeroize", +] + +[[package]] +name = "ssh-key" +version = "0.7.0-rc.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a32fae177b74a22aa9c5b01bf7e68b33545be32d9e381e248058d2adc15ce3" +dependencies = [ + "argon2 0.6.0-rc.8", + "bcrypt-pbkdf", + "ctutils", + "ed25519-dalek 3.0.0-rc.1", + "hex", + "hmac 0.13.0", + "p256 0.14.0-rc.15", + "p384 0.14.0-rc.15", + "p521 0.14.0-rc.15", + "rand_core 0.10.1", + "rsa 0.10.0-rc.18", + "sec1 0.8.1", + "sha1 0.11.0", + "sha2 0.11.0", + "signature 3.0.0", + "ssh-cipher", + "ssh-encoding", + "zeroize", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -7294,6 +8201,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -8050,6 +8967,17 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wnaf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" +dependencies = [ + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", +] + [[package]] name = "writeable" version = "0.6.3" @@ -8072,10 +9000,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ "const-oid 0.9.6", - "der", + "der 0.7.10", "sha1 0.10.6", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "tls_codec", ] diff --git a/Cargo.toml b/Cargo.toml index edc7899c1..3ba61d8aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,6 +129,8 @@ serde_repr = "0.1.20" secrecy = "0.10" sha1 = "0.11" sha2 = "0.11" +russh = "0.62.1" +russh-sftp = "2.3.0" tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "fs", "io-util"] } tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } tokio-util = { version = "0.7", features = ["io", "io-util"] } diff --git a/frontend-panel/src/components/admin/StoragePolicyDialog.tsx b/frontend-panel/src/components/admin/StoragePolicyDialog.tsx index 87a6671d4..d9cb11470 100644 --- a/frontend-panel/src/components/admin/StoragePolicyDialog.tsx +++ b/frontend-panel/src/components/admin/StoragePolicyDialog.tsx @@ -12,6 +12,7 @@ import { supportsOneDrivePolicyOptions, supportsRemoteNodeBinding, supportsSavedConnectionTest, + supportsStaticSecretConnection, supportsStorageNativeProcessing, } from "@/components/admin/storage-policy-dialog/descriptorPredicates"; import type { PolicyFormData } from "@/components/admin/storage-policy-dialog/formTypes"; @@ -192,6 +193,9 @@ function useStoragePolicyDialogContent({ const canUseObjectStorageConnection = supportsObjectStorageConnection( storageDriverDescriptor, ); + const canUseStaticSecretConnection = supportsStaticSecretConnection( + storageDriverDescriptor, + ); const canUseRemoteNodeBinding = supportsRemoteNodeBinding( storageDriverDescriptor, ); @@ -286,7 +290,7 @@ function useStoragePolicyDialogContent({ ? t("onedrive_client_secret_required") : null; const createEndpointError = - canUseObjectStorageConnection && !form.endpoint.trim() + canUseStaticSecretConnection && !form.endpoint.trim() ? isCreateMode ? createStep === 1 && createStepTouched ? t("policy_wizard_endpoint_required") @@ -410,13 +414,15 @@ function useStoragePolicyDialogContent({ }, ] : []), - ...(canUseObjectStorageConnection + ...(canUseStaticSecretConnection ? [ { label: t("endpoint"), value: form.endpoint || t("policy_wizard_default_endpoint"), }, - { label: t("bucket"), value: form.bucket || "—" }, + ...(canUseObjectStorageConnection + ? [{ label: t("bucket"), value: form.bucket || "—" }] + : []), ...(canUseObjectStorageTransferStrategy ? [ { diff --git a/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.test.ts b/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.test.ts index 27fec5d41..de7c8aab0 100644 --- a/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.test.ts +++ b/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.test.ts @@ -5,6 +5,7 @@ describe("policyPresentation", () => { it("returns distinct badge classes for every storage driver", () => { expect(getPolicyDriverBadgeClass("local")).toContain("text-emerald-600"); expect(getPolicyDriverBadgeClass("s3")).toContain("text-blue-600"); + expect(getPolicyDriverBadgeClass("sftp")).toContain("text-violet-700"); expect(getPolicyDriverBadgeClass("tencent_cos")).toContain("text-cyan-700"); expect(getPolicyDriverBadgeClass("azure_blob")).toContain("text-sky-700"); expect(getPolicyDriverBadgeClass("remote")).toContain("text-amber-600"); diff --git a/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.ts b/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.ts index f365f6bf3..af121806f 100644 --- a/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.ts +++ b/frontend-panel/src/components/admin/admin-policies-page/policyPresentation.ts @@ -11,6 +11,7 @@ const POLICY_DRIVER_BADGE_CLASSES = { remote: "border-amber-500/60 bg-amber-500/10 text-amber-600 dark:text-amber-300", s3: "border-blue-500/60 bg-blue-500/10 text-blue-600 dark:text-blue-300", + sftp: "border-violet-500/60 bg-violet-500/10 text-violet-700 dark:text-violet-300", tencent_cos: "border-cyan-500/60 bg-cyan-500/10 text-cyan-700 dark:text-cyan-300", } satisfies Record; diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyCreateWizard.tsx b/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyCreateWizard.tsx index 2b5cadd63..f93104713 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyCreateWizard.tsx +++ b/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyCreateWizard.tsx @@ -41,6 +41,7 @@ import { supportsObjectStorageTransferStrategy, supportsOneDrivePolicyOptions, supportsRemoteNodeBinding, + supportsStaticSecretConnection, supportsStorageNativeProcessing, } from "./descriptorPredicates"; import type { PolicyFormData } from "./formTypes"; @@ -428,7 +429,7 @@ function ConnectionStep({ remoteStorageTargetsLoading, t, }: ConnectionStepProps) { - const canUseObjectStorageConnection = supportsObjectStorageConnection( + const canUseStaticSecretConnection = supportsStaticSecretConnection( storageDriverDescriptor, ); const canUseRemoteNodeBinding = supportsRemoteNodeBinding( @@ -459,7 +460,7 @@ function ConnectionStep({ t={t} onFieldChange={onFieldChange} /> - {canUseObjectStorageConnection ? ( + {canUseStaticSecretConnection ? ( - {canUseObjectStorageConnection ? ( + {canUseStaticSecretConnection ? (
@@ -65,22 +66,26 @@ export function ObjectStorageConnectionFields({

) : null} -
- - onFieldChange("bucket", e.target.value)} - aria-invalid={showCreateValidation && bucketError ? true : undefined} - className={ADMIN_CONTROL_HEIGHT_CLASS} - required - /> - {showCreateValidation && bucketError ? ( -

{bucketError}

- ) : null} -
+ {hasBucketField ? ( +
+ + onFieldChange("bucket", e.target.value)} + aria-invalid={ + showCreateValidation && bucketError ? true : undefined + } + className={ADMIN_CONTROL_HEIGHT_CLASS} + required={bucketField.required} + /> + {showCreateValidation && bucketError ? ( +

{bucketError}

+ ) : null} +
+ ) : null} {showPathStyleField ? ( field.scope === "connection" && field.name === "endpoint", )?.invalid_protocol_message_key ?? "s3_endpoint_protocol_required_error"; + const allowedProtocols = allowedEndpointProtocols(descriptor); + if (!hasEndpointUrlScheme(trimmedEndpoint)) { + return allowedProtocols.includes("sftp:") + ? null + : t(endpointProtocolMessage); + } + let endpointUrl: URL; try { endpointUrl = new URL(trimmedEndpoint); @@ -264,7 +289,7 @@ export function getEndpointValidationMessage( return t(endpointProtocolMessage); } - if (endpointUrl.protocol !== "http:" && endpointUrl.protocol !== "https:") { + if (!allowedProtocols.includes(endpointUrl.protocol)) { return t(endpointProtocolMessage); } @@ -281,6 +306,16 @@ function shouldUseObjectStorageConnection( : hasObjectStorageConnectionFields(form, editingPolicy); } +function shouldUseStaticSecretConnection( + form: PolicyFormData, + descriptor?: StorageConnectorDescriptor | null, + editingPolicy?: StoragePolicy | null, +) { + return descriptor + ? supportsStaticSecretConnection(descriptor) + : hasStaticSecretConnectionFields(form, editingPolicy); +} + function shouldUseMicrosoftGraphConfig( form: PolicyFormData, descriptor?: StorageConnectorDescriptor | null, @@ -313,10 +348,41 @@ function hasObjectStorageConnectionFields( ); } +function hasStaticSecretConnectionFields( + form: PolicyFormData, + editingPolicy?: StoragePolicy | null, +) { + return Boolean( + hasObjectStorageConnectionFields(form, editingPolicy) || + hasText(form.endpoint) || + hasText(editingPolicy?.endpoint), + ); +} + function hasEndpointUrlScheme(endpoint?: string | null) { return /^[a-z][a-z0-9+.-]*:/i.test(endpoint?.trim() ?? ""); } +function allowedEndpointProtocols( + descriptor?: StorageConnectorDescriptor | null, +) { + return descriptor?.driver_type === "sftp" ? ["sftp:"] : ["http:", "https:"]; +} + +function shouldTrimConnectionField( + descriptor: StorageConnectorDescriptor | null | undefined, + fieldName: string, +) { + return ( + descriptor?.fields.some( + (field) => + field.scope === "connection" && + field.name === fieldName && + field.trim_on_blur === true, + ) ?? false + ); +} + function hasExplicitMicrosoftGraphFields(form: PolicyFormData) { const microsoftGraph = microsoftGraphCredentials(form); const tenant = form.onedrive_tenant?.trim(); diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/descriptorPredicates.ts b/frontend-panel/src/components/admin/storage-policy-dialog/descriptorPredicates.ts index 3c5af6585..9cf7c5b22 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/descriptorPredicates.ts +++ b/frontend-panel/src/components/admin/storage-policy-dialog/descriptorPredicates.ts @@ -46,6 +46,17 @@ export function supportsObjectStorageConnection( ); } +export function supportsStaticSecretConnection( + descriptor: StorageConnectorDescriptor | null | undefined, +) { + return ( + descriptor?.credential_mode === "static_secret" && + descriptorHasConnectionField(descriptor, "endpoint") && + descriptorHasConnectionField(descriptor, "access_key") && + descriptorHasConnectionField(descriptor, "secret_key") + ); +} + export function supportsRemoteNodeBinding( descriptor: StorageConnectorDescriptor | null | undefined, ) { diff --git a/frontend-panel/src/i18n/locales/en/admin/policies.json b/frontend-panel/src/i18n/locales/en/admin/policies.json index cabc2a2d8..69c30a3f5 100644 --- a/frontend-panel/src/i18n/locales/en/admin/policies.json +++ b/frontend-panel/src/i18n/locales/en/admin/policies.json @@ -88,9 +88,11 @@ "s3_endpoint_hint": "Enter the S3-compatible API endpoint and keep the bucket in the bucket field. Providers differ on path-style support, so test the connection before saving.", "cos_endpoint_hint": "For Tencent COS, use the bucket domain such as https://.cos..myqcloud.com; enter the bucket separately, for example .", "azure_blob_endpoint_hint": "Enter the Azure Blob service endpoint, for example https://.blob.core.windows.net. Use the bucket field for the container name.", + "sftp_endpoint_hint": "Enter an SFTP server endpoint such as sftp://example.com:22. Put the remote root directory in base path.", "policy_wizard_endpoint_required": "Endpoint is required.", "s3_endpoint_protocol_required_error": "S3 endpoint must include http:// or https://.", "azure_blob_endpoint_protocol_required_error": "Azure Blob endpoint must include http:// or https://.", + "sftp_endpoint_protocol_required_error": "SFTP endpoint must use sftp:// or omit the scheme.", "download_strategy_relay_stream": "Server Relay Download", "download_strategy_relay_stream_desc": "AsterDrive fetches the object from the storage backend and streams it back to the browser. Use this when you need the app node to fully control response headers, same-origin delivery, or downstream network policy.", "download_strategy_presigned": "Presigned Redirect", @@ -160,6 +162,7 @@ "driver_type_local": "Local", "driver_type_remote": "Remote", "driver_type_s3": "S3", + "driver_type_sftp": "SFTP", "driver_type_tencent_cos": "Tencent COS", "driver_type_azure_blob": "Azure Blob", "driver_type_onedrive": "OneDrive", @@ -177,6 +180,8 @@ "policy_wizard_step_connection_title": "Configure Connection", "policy_wizard_step_connection_desc": "Review the connection settings required by this storage backend.", "policy_wizard_step_object_storage_connection_desc": "Set the object-storage endpoint, bucket, and credentials.", + "policy_wizard_step_sftp_title": "Configure SFTP", + "policy_wizard_step_sftp_desc": "Set the SFTP endpoint, SSH username, password, and remote root path.", "policy_wizard_step_tencent_cos_connection_desc": "Set the Tencent COS bucket domain, bucket, and credentials.", "policy_wizard_step_azure_blob_connection_desc": "Set the Azure Blob endpoint, container name, storage account name, and key.", "policy_wizard_step_onedrive_title": "Configure Microsoft Graph", @@ -192,6 +197,7 @@ "policy_wizard_local_storage_desc": "Store files on the AsterDrive server filesystem. Simple setup and suitable for single-node deployments.", "policy_wizard_remote_storage_desc": "Store files on another AsterDrive node over the internal remote storage protocol. Good for tiered or federated deployments.", "policy_wizard_s3_storage_desc": "Store files in an S3-compatible object bucket such as Amazon S3, MinIO, or RustFS.", + "policy_wizard_sftp_storage_desc": "Store files on an SFTP server through server-side streaming. Browsers never connect to SFTP directly.", "policy_wizard_tencent_cos_storage_desc": "Store files in Tencent Cloud COS and enable storage-native document preview through COS CI.", "policy_wizard_azure_blob_storage_desc": "Store files in an Azure Blob container with native SAS URLs and Block Blob multipart uploads.", "policy_wizard_onedrive_storage_desc": "Store files in OneDrive, SharePoint document libraries, or Microsoft 365 group drives through Microsoft Graph.", @@ -224,6 +230,7 @@ "policy_wizard_onedrive_helper": "Enter the Microsoft application client ID while creating the policy. After saving, the dialog switches to editing so you can authorize Microsoft Graph directly. Choose Global for microsoft.com tenants and China for 21Vianet tenants.", "policy_wizard_remote_helper": "The selected remote node handles network transport. This policy controls the remote path prefix, upload mode, and size limits.", "policy_wizard_local_helper": "Use a relative or absolute path. Leave it empty to fall back to the application's default data directory.", + "policy_wizard_sftp_helper": "AsterDrive streams uploads and downloads through the app server, then reads and writes files over SFTP using the configured SSH credentials.", "policy_wizard_remote_rules_helper": "Remote storage does not use per-policy endpoint or credentials. Bind a remote node, choose pure stream relay or presigned direct upload, then configure upload limits and default-policy behavior.", "policy_wizard_onedrive_rules_helper": "OneDrive upload behavior is managed by Microsoft Graph upload sessions. Configure size limits and default-policy behavior here; OAuth authorization happens after the policy is saved.", "policy_wizard_summary_title": "Policy Summary", @@ -245,6 +252,7 @@ "policy_edit_context_title": "Current Policy", "policy_edit_context_local_desc": "Local policies write directly to the server filesystem. Adjust paths and upload rules below.", "policy_edit_context_object_storage_desc": "Test object storage connections before saving. Blank secret fields keep the current credentials.", + "policy_edit_context_sftp_desc": "SFTP policies stream through the app server and write to the configured remote root. Blank password fields keep the current credential.", "policy_edit_context_azure_blob_desc": "Azure Blob policies use the storage account key to issue short-lived SAS URLs. Test the connection before saving; blank secret fields keep the current credentials.", "policy_edit_context_remote_desc": "Remote policies transfer through the bound node. Adjust paths, node binding, and upload rules below.", "policy_edit_context_onedrive_desc": "OneDrive policies use saved Microsoft Graph OAuth credentials. Save target changes before starting authorization.", diff --git a/frontend-panel/src/i18n/locales/zh/admin/policies.json b/frontend-panel/src/i18n/locales/zh/admin/policies.json index c216fe6ad..e0ba6bf07 100644 --- a/frontend-panel/src/i18n/locales/zh/admin/policies.json +++ b/frontend-panel/src/i18n/locales/zh/admin/policies.json @@ -88,9 +88,11 @@ "s3_endpoint_hint": "填写兼容 S3 的 API endpoint,bucket 请单独填写。不同厂商对 path-style 的要求不同,保存前请先测试连接。", "cos_endpoint_hint": "腾讯云 COS 请填写 bucket 域名,例如 https://.cos..myqcloud.com;bucket 请单独填写,例如 。", "azure_blob_endpoint_hint": "填写 Azure Blob 服务 endpoint,例如 https://.blob.core.windows.net;bucket 字段填写容器名称。", + "sftp_endpoint_hint": "填写 SFTP 服务器 endpoint,例如 sftp://example.com:22;远程根目录请填写在基础路径里。", "policy_wizard_endpoint_required": "必须填写端点。", "s3_endpoint_protocol_required_error": "S3 endpoint 必须包含 http:// 或 https://。", "azure_blob_endpoint_protocol_required_error": "Azure Blob endpoint 必须包含 http:// 或 https://。", + "sftp_endpoint_protocol_required_error": "SFTP endpoint 必须使用 sftp://,或省略协议。", "download_strategy_relay_stream": "服务端中继下载", "download_strategy_relay_stream_desc": "AsterDrive 先从存储后端拉取对象,再把内容流式回传给浏览器。适合需要由应用节点完全控制响应头、同源下载行为或下游网络策略的场景。", "download_strategy_presigned": "Presigned 重定向", @@ -160,6 +162,7 @@ "driver_type_local": "本机", "driver_type_remote": "远程节点", "driver_type_s3": "S3", + "driver_type_sftp": "SFTP", "driver_type_tencent_cos": "腾讯云 COS", "driver_type_azure_blob": "Azure Blob", "driver_type_onedrive": "OneDrive", @@ -177,6 +180,8 @@ "policy_wizard_step_connection_title": "配置连接", "policy_wizard_step_connection_desc": "检查当前存储后端需要的连接配置。", "policy_wizard_step_object_storage_connection_desc": "填写对象存储 endpoint、bucket 和访问凭证。", + "policy_wizard_step_sftp_title": "配置 SFTP", + "policy_wizard_step_sftp_desc": "填写 SFTP endpoint、SSH 用户名、密码和远程根目录。", "policy_wizard_step_tencent_cos_connection_desc": "填写腾讯云 COS bucket 域名、bucket 和访问凭证。", "policy_wizard_step_azure_blob_connection_desc": "填写 Azure Blob endpoint、容器名称、存储账户名和密钥。", "policy_wizard_step_onedrive_title": "配置 Microsoft Graph", @@ -192,6 +197,7 @@ "policy_wizard_local_storage_desc": "文件直接落在 AsterDrive 所在服务器的文件系统上,配置简单,适合单机部署。", "policy_wizard_remote_storage_desc": "通过内部远程存储协议把文件写入另一台 AsterDrive 节点,适合分层或多节点部署。", "policy_wizard_s3_storage_desc": "文件存入兼容 S3 的对象存储,例如 Amazon S3、MinIO 或 RustFS。", + "policy_wizard_sftp_storage_desc": "通过服务端流式中继把文件存入 SFTP 服务器,浏览器不会直接连接 SFTP。", "policy_wizard_tencent_cos_storage_desc": "文件存入腾讯云 COS,并可使用 COS 数据万象进行原生文档预览。", "policy_wizard_azure_blob_storage_desc": "文件存入 Azure Blob 容器,使用原生 SAS URL 和 Block Blob 分块上传。", "policy_wizard_onedrive_storage_desc": "通过 Microsoft Graph 把文件存入 OneDrive、SharePoint 文档库或 Microsoft 365 组 drive。", @@ -224,6 +230,7 @@ "policy_wizard_onedrive_helper": "创建时填写 Microsoft 应用 Client ID;保存后会直接进入编辑态授权 Microsoft Graph。microsoft.com 租户选国际版,世纪互联租户选中国版。", "policy_wizard_remote_helper": "实际网络传输由远程节点负责,这个策略控制远端路径前缀、上传方式和大小限制。", "policy_wizard_local_helper": "支持相对路径或绝对路径。留空时会回退到应用默认的数据目录。", + "policy_wizard_sftp_helper": "AsterDrive 会通过应用服务器流式中继上传和下载,再使用配置的 SSH 凭据通过 SFTP 读写文件。", "policy_wizard_remote_rules_helper": "远程存储不使用策略级 endpoint 或凭证;这里只需绑定远程节点,并选择纯流式转发还是 Presigned 直传,再配置上传限制和默认策略。", "policy_wizard_onedrive_rules_helper": "OneDrive 上传行为由 Microsoft Graph upload session 管理。这里配置大小限制和默认策略;OAuth 授权会在策略保存后进行。", "policy_wizard_summary_title": "策略摘要", @@ -245,6 +252,7 @@ "policy_edit_context_title": "当前策略", "policy_edit_context_local_desc": "本机策略直接写入服务器文件系统;路径和上传规则在下方调整。", "policy_edit_context_object_storage_desc": "对象存储策略保存前建议测试连接;留空密钥字段会保留现有凭证。", + "policy_edit_context_sftp_desc": "SFTP 策略通过应用服务器流式中继,并写入配置的远程根目录;留空密码字段会保留现有凭证。", "policy_edit_context_azure_blob_desc": "Azure Blob 策略使用存储账户密钥签发短期 SAS URL;保存前建议测试连接,留空密钥字段会保留现有凭证。", "policy_edit_context_remote_desc": "远程策略由绑定节点负责传输;这里调整路径、节点和上传规则。", "policy_edit_context_onedrive_desc": "OneDrive 策略使用已保存的 Microsoft Graph OAuth 凭据。开始授权前请先保存目标配置。", diff --git a/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx b/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx index f55d97140..f378f73aa 100644 --- a/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx +++ b/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx @@ -66,6 +66,8 @@ const mockTranslate = vi.hoisted( return "Local"; case "driver_type_s3": return "S3"; + case "driver_type_sftp": + return "SFTP"; case "driver_type_tencent_cos": return "Tencent COS"; case "driver_type_azure_blob": @@ -854,6 +856,19 @@ function storageConnectorUi(driverType: string) { icon_src: "/static/storage/onedrive.svg", label_key: "driver_type_onedrive", }; + case "sftp": + return { + base_path_empty_display: "core:root", + base_path_placeholder: "/srv/asterdrive", + config_step_description_key: "policy_wizard_step_sftp_desc", + config_step_title_key: "policy_wizard_step_sftp_title", + description_key: "policy_wizard_sftp_storage_desc", + edit_context_key: "policy_edit_context_sftp_desc", + helper_key: "policy_wizard_sftp_helper", + icon_name: "ServerCog", + icon_src: null, + label_key: "driver_type_sftp", + }; default: return { ...sharedObjectStorageUi, @@ -996,6 +1011,30 @@ function createStorageDriverDescriptors() { presigned_upload: true, }, }), + createStorageDriverDescriptor("sftp", { + capabilities: { + ...createStorageDriverDescriptor("sftp").capabilities, + capacity: false, + list: false, + }, + credential_mode: "static_secret", + fields: [ + fieldDescriptor("endpoint", "connection", "text", { + help_key: "sftp_endpoint_hint", + invalid_protocol_message_key: "sftp_endpoint_protocol_required_error", + placeholder: "sftp://example.com:22", + required: true, + trim_on_blur: true, + }), + fieldDescriptor("access_key", "connection", "text", { + required: true, + }), + fieldDescriptor("secret_key", "connection", "secret", { + required: true, + }), + fieldDescriptor("base_path", "connection", "text"), + ], + }), createStorageDriverDescriptor("tencent_cos", { actions: [ { @@ -1132,6 +1171,7 @@ function openCreateWizard( | "local" | "remote" | "s3" + | "sftp" | "tencent_cos" | "azure_blob" | "one_drive" = "local", @@ -1143,6 +1183,8 @@ function openCreateWizard( fireEvent.click(screen.getByRole("button", { name: /^Remote\b/ })); } else if (driver === "s3") { fireEvent.click(screen.getByRole("button", { name: /^S3\b/ })); + } else if (driver === "sftp") { + fireEvent.click(screen.getByRole("button", { name: /^SFTP\b/ })); } else if (driver === "tencent_cos") { fireEvent.click(screen.getByRole("button", { name: /Tencent COS/ })); } else if (driver === "azure_blob") { diff --git a/frontend-panel/src/pages/admin/AdminPoliciesPage.tsx b/frontend-panel/src/pages/admin/AdminPoliciesPage.tsx index 6c78f3d01..0e7e173d5 100644 --- a/frontend-panel/src/pages/admin/AdminPoliciesPage.tsx +++ b/frontend-panel/src/pages/admin/AdminPoliciesPage.tsx @@ -24,6 +24,7 @@ import { supportsOneDrivePolicyOptions, supportsRemoteNodeBinding, supportsSavedConnectionTest, + supportsStaticSecretConnection, supportsStorageCredentialLifecycle, supportsStorageNativeProcessing, supportsStoragePolicyAction, @@ -1069,10 +1070,13 @@ function useAdminPoliciesPageContent() { ); const nextSupportsStorageNativeProcessing = supportsStorageNativeProcessing(nextDriverDescriptor); - if (supportsObjectStorageConnection(nextDriverDescriptor)) { + if (supportsStaticSecretConnection(nextDriverDescriptor)) { return { ...prevWithoutS3PathStyle, driver_type: driverType, + bucket: supportsObjectStorageConnection(nextDriverDescriptor) + ? prev.bucket + : "", remote_node_id: "", remote_storage_target_key: "", storage_native_processing_enabled: nextSupportsStorageNativeProcessing @@ -1237,7 +1241,7 @@ function useAdminPoliciesPageContent() { } if ( - supportsObjectStorageConnection(descriptor) || + supportsStaticSecretConnection(descriptor) || supportsRemoteNodeBinding(descriptor) ) { setValidatedConnectionKey( @@ -1641,7 +1645,7 @@ function useAdminPoliciesPageContent() { } if ( - supportsObjectStorageConnection(currentStorageDriverDescriptor) && + supportsStaticSecretConnection(currentStorageDriverDescriptor) && !form.endpoint.trim() ) { return; diff --git a/frontend-panel/src/services/api.generated.ts b/frontend-panel/src/services/api.generated.ts index 6ef1066bf..902211739 100644 --- a/frontend-panel/src/services/api.generated.ts +++ b/frontend-panel/src/services/api.generated.ts @@ -5244,7 +5244,7 @@ export interface components { * @description 存储驱动类型 * @enum {string} */ - DriverType: "local" | "s3" | "azure_blob" | "tencent_cos" | "remote" | "one_drive"; + DriverType: "local" | "s3" | "sftp" | "azure_blob" | "tencent_cos" | "remote" | "one_drive"; /** @description Check a storage policy migration plan without creating a task. */ DryRunStoragePolicyMigrationReq: { delete_source_after_success?: boolean; diff --git a/src/storage/connectors/mod.rs b/src/storage/connectors/mod.rs index 4af86cbf1..a3724bcad 100644 --- a/src/storage/connectors/mod.rs +++ b/src/storage/connectors/mod.rs @@ -16,6 +16,7 @@ mod models; mod onedrive; mod remote; mod s3; +mod sftp; mod tencent_cos; mod upload; @@ -35,7 +36,8 @@ use crate::storage::connector_descriptor::{ StorageConnectorDescriptorProvider, StoragePolicyExecutableAction, }; use crate::storage::drivers::{ - azure_blob::AzureBlobDriver, local::LocalDriver, s3::S3Driver, tencent_cos::TencentCosDriver, + azure_blob::AzureBlobDriver, local::LocalDriver, s3::S3Driver, sftp::SftpDriver, + tencent_cos::TencentCosDriver, }; use crate::types::{DriverType, StorageCredentialKind, StorageCredentialProvider}; @@ -57,6 +59,7 @@ pub(crate) use models::{ use onedrive::OneDriveConnector; use remote::RemoteConnector; use s3::S3Connector; +use sftp::SftpConnector; use tencent_cos::TencentCosConnector; pub use upload::{StorageConnectorChunkedCompletion, StorageConnectorUploadTransport}; @@ -317,6 +320,7 @@ struct StorageConnectorRegistration { enum BuiltinStorageConnector { Local, S3, + Sftp, AzureBlob, TencentCos, Remote, @@ -328,6 +332,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::storage_connector_descriptor(), Self::S3 => S3Connector::storage_connector_descriptor(), + Self::Sftp => SftpConnector::storage_connector_descriptor(), Self::AzureBlob => AzureBlobConnector::storage_connector_descriptor(), Self::TencentCos => TencentCosConnector::storage_connector_descriptor(), Self::Remote => RemoteConnector::storage_connector_descriptor(), @@ -345,6 +350,9 @@ impl BuiltinStorageConnector { common::normalize_policy_connection_for::(db, input).await } Self::S3 => common::normalize_policy_connection_for::(db, input).await, + Self::Sftp => { + common::normalize_policy_connection_for::(db, input).await + } Self::AzureBlob => { common::normalize_policy_connection_for::(db, input).await } @@ -370,6 +378,7 @@ impl BuiltinStorageConnector { LocalConnector::prepare_connection_for_storage(input, application_config) } Self::S3 => S3Connector::prepare_connection_for_storage(input, application_config), + Self::Sftp => SftpConnector::prepare_connection_for_storage(input, application_config), Self::AzureBlob => { AzureBlobConnector::prepare_connection_for_storage(input, application_config) } @@ -396,6 +405,7 @@ impl BuiltinStorageConnector { LocalConnector::validate_policy_options(db, remote_node_id, options).await } Self::S3 => S3Connector::validate_policy_options(db, remote_node_id, options).await, + Self::Sftp => SftpConnector::validate_policy_options(db, remote_node_id, options).await, Self::AzureBlob => { AzureBlobConnector::validate_policy_options(db, remote_node_id, options).await } @@ -440,6 +450,16 @@ impl BuiltinStorageConnector { ) .await } + Self::Sftp => { + SftpConnector::persist_application_config( + db, + encryption_key, + policy_id, + options, + application_config, + ) + .await + } Self::AzureBlob => { AzureBlobConnector::persist_application_config( db, @@ -491,6 +511,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::test_draft_connection(state, input).await, Self::S3 => S3Connector::test_draft_connection(state, input).await, + Self::Sftp => SftpConnector::test_draft_connection(state, input).await, Self::AzureBlob => AzureBlobConnector::test_draft_connection(state, input).await, Self::TencentCos => TencentCosConnector::test_draft_connection(state, input).await, Self::Remote => RemoteConnector::test_draft_connection(state, input).await, @@ -506,6 +527,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::test_saved_connection(state, policy).await, Self::S3 => S3Connector::test_saved_connection(state, policy).await, + Self::Sftp => SftpConnector::test_saved_connection(state, policy).await, Self::AzureBlob => AzureBlobConnector::test_saved_connection(state, policy).await, Self::TencentCos => TencentCosConnector::test_saved_connection(state, policy).await, Self::Remote => RemoteConnector::test_saved_connection(state, policy).await, @@ -522,6 +544,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::execute_saved_action(state, policy, action).await, Self::S3 => S3Connector::execute_saved_action(state, policy, action).await, + Self::Sftp => SftpConnector::execute_saved_action(state, policy, action).await, Self::AzureBlob => { AzureBlobConnector::execute_saved_action(state, policy, action).await } @@ -541,6 +564,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::execute_draft_action(state, input).await, Self::S3 => S3Connector::execute_draft_action(state, input).await, + Self::Sftp => SftpConnector::execute_draft_action(state, input).await, Self::AzureBlob => AzureBlobConnector::execute_draft_action(state, input).await, Self::TencentCos => TencentCosConnector::execute_draft_action(state, input).await, Self::Remote => RemoteConnector::execute_draft_action(state, input).await, @@ -552,6 +576,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::upload_transport(policy), Self::S3 => S3Connector::upload_transport(policy), + Self::Sftp => SftpConnector::upload_transport(policy), Self::AzureBlob => AzureBlobConnector::upload_transport(policy), Self::TencentCos => TencentCosConnector::upload_transport(policy), Self::Remote => RemoteConnector::upload_transport(policy), @@ -563,6 +588,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::presigned_download_enabled(policy), Self::S3 => S3Connector::presigned_download_enabled(policy), + Self::Sftp => SftpConnector::presigned_download_enabled(policy), Self::AzureBlob => AzureBlobConnector::presigned_download_enabled(policy), Self::TencentCos => TencentCosConnector::presigned_download_enabled(policy), Self::Remote => RemoteConnector::presigned_download_enabled(policy), @@ -574,6 +600,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::runtime_credential_requirement(), Self::S3 => S3Connector::runtime_credential_requirement(), + Self::Sftp => SftpConnector::runtime_credential_requirement(), Self::AzureBlob => AzureBlobConnector::runtime_credential_requirement(), Self::TencentCos => TencentCosConnector::runtime_credential_requirement(), Self::Remote => RemoteConnector::runtime_credential_requirement(), @@ -593,6 +620,9 @@ impl BuiltinStorageConnector { LocalConnector::load_runtime_credential(db, config, policy, credential).await } Self::S3 => S3Connector::load_runtime_credential(db, config, policy, credential).await, + Self::Sftp => { + SftpConnector::load_runtime_credential(db, config, policy, credential).await + } Self::AzureBlob => { AzureBlobConnector::load_runtime_credential(db, config, policy, credential).await } @@ -616,6 +646,7 @@ impl BuiltinStorageConnector { match self { Self::Local => LocalConnector::build_authorized_driver(policy, credential), Self::S3 => S3Connector::build_authorized_driver(policy, credential), + Self::Sftp => SftpConnector::build_authorized_driver(policy, credential), Self::AzureBlob => AzureBlobConnector::build_authorized_driver(policy, credential), Self::TencentCos => TencentCosConnector::build_authorized_driver(policy, credential), Self::Remote => RemoteConnector::build_authorized_driver(policy, credential), @@ -635,6 +666,7 @@ impl BuiltinStorageConnector { LocalConnector::validate_credential(db, config, policy, credential).await } Self::S3 => S3Connector::validate_credential(db, config, policy, credential).await, + Self::Sftp => SftpConnector::validate_credential(db, config, policy, credential).await, Self::AzureBlob => { AzureBlobConnector::validate_credential(db, config, policy, credential).await } @@ -658,7 +690,7 @@ impl BuiltinStorageConnector { match self { Self::Remote => RemoteConnector::cleanup_snapshot_for_policy(state, policy).await, Self::OneDrive => OneDriveConnector::cleanup_snapshot_for_policy(state, policy).await, - Self::Local | Self::S3 | Self::AzureBlob | Self::TencentCos => Ok(None), + Self::Local | Self::S3 | Self::Sftp | Self::AzureBlob | Self::TencentCos => Ok(None), } } @@ -671,6 +703,7 @@ impl BuiltinStorageConnector { match self { Self::Local => Ok(Arc::new(LocalDriver::new(policy)?)), Self::S3 => Ok(Arc::new(S3Driver::new(policy)?)), + Self::Sftp => Ok(Arc::new(SftpDriver::new(policy)?)), Self::AzureBlob => Ok(Arc::new(AzureBlobDriver::new(policy)?)), Self::TencentCos => Ok(Arc::new(TencentCosDriver::new(policy)?)), Self::Remote => RemoteConnector::build_cleanup_driver(state, policy, snapshots).await, @@ -705,6 +738,11 @@ static CONNECTOR_REGISTRATIONS: &[StorageConnectorRegistration] = &[ connector: BuiltinStorageConnector::S3, cleanup_snapshot_required: false, }, + StorageConnectorRegistration { + driver_type: DriverType::Sftp, + connector: BuiltinStorageConnector::Sftp, + cleanup_snapshot_required: false, + }, StorageConnectorRegistration { driver_type: DriverType::AzureBlob, connector: BuiltinStorageConnector::AzureBlob, diff --git a/src/storage/connectors/sftp.rs b/src/storage/connectors/sftp.rs new file mode 100644 index 000000000..bbddddfd1 --- /dev/null +++ b/src/storage/connectors/sftp.rs @@ -0,0 +1,159 @@ +use async_trait::async_trait; + +use crate::entities::storage_policy; +use crate::errors::Result; +use crate::runtime::RemoteProtocolRuntimeState; +use crate::storage::StorageDriver; +use crate::storage::connector_descriptor::{ + StorageConnectorCapabilities, StorageConnectorCredentialMode, StorageConnectorDescriptor, + StorageConnectorDescriptorProvider, StorageConnectorFieldDisplayInput, + StorageConnectorFieldKind, StorageConnectorFieldScope, StorageConnectorUiDescriptorInput, + StorageConnectorUploadWorkflows, draft_connection_test_action_descriptor, + saved_connection_test_action_descriptor, server_relay_simple_upload_capabilities, + storage_connector_field, storage_connector_field_with_display, storage_connector_ui_descriptor, +}; +use crate::storage::drivers::sftp::SftpDriver; +use crate::types::DriverType; + +use super::common::{ensure_onedrive_options_absent, validate_static_secret_credentials}; +use super::{StorageConnector, StorageConnectorConnectionInput, StorageConnectorUploadTransport}; + +pub struct SftpConnector; + +impl StorageConnectorDescriptorProvider for SftpConnector { + fn storage_connector_descriptor() -> StorageConnectorDescriptor { + StorageConnectorDescriptor { + driver_type: DriverType::Sftp, + enabled: true, + label: "SFTP".to_string(), + description: "SSH File Transfer Protocol storage policy".to_string(), + ui: storage_connector_ui_descriptor(StorageConnectorUiDescriptorInput { + label_key: "driver_type_sftp", + description_key: "policy_wizard_sftp_storage_desc", + icon_src: None, + icon_name: Some("ServerCog"), + helper_key: "policy_wizard_sftp_helper", + config_step_title_key: "policy_wizard_step_sftp_title", + config_step_description_key: "policy_wizard_step_sftp_desc", + edit_context_key: "policy_edit_context_sftp_desc", + base_path_empty_display: "core:root", + base_path_placeholder: "/srv/asterdrive", + }), + credential_mode: StorageConnectorCredentialMode::StaticSecret, + requires_authorization: false, + authorization_provider: None, + capabilities: StorageConnectorCapabilities { + efficient_range: true, + capacity: false, + list: false, + presigned_download: false, + storage_native_thumbnail: false, + storage_native_media_metadata: false, + remote_node_binding: false, + object_storage_transfer_strategy: false, + }, + upload_workflows: StorageConnectorUploadWorkflows { + simple_upload: true, + simple_upload_capabilities: server_relay_simple_upload_capabilities(None), + stream_upload: true, + object_multipart_upload: false, + object_multipart_upload_capabilities: None, + provider_resumable_upload: false, + presigned_upload: false, + frontend_direct_provider_resumable_upload: false, + provider_resumable_upload_capabilities: None, + }, + fields: vec![ + storage_connector_field_with_display(StorageConnectorFieldDisplayInput { + name: "endpoint", + scope: StorageConnectorFieldScope::Connection, + kind: StorageConnectorFieldKind::Text, + required: true, + secret: false, + label_key: "endpoint", + placeholder: Some("sftp://example.com:22"), + help_key: Some("sftp_endpoint_hint"), + required_message_key: None, + invalid_protocol_message_key: Some("sftp_endpoint_protocol_required_error"), + trim_on_blur: true, + visible_when_driver_types: Vec::new(), + }), + storage_connector_field( + "access_key", + StorageConnectorFieldScope::Connection, + StorageConnectorFieldKind::Text, + true, + false, + ), + storage_connector_field( + "secret_key", + StorageConnectorFieldScope::Connection, + StorageConnectorFieldKind::Secret, + true, + true, + ), + storage_connector_field( + "base_path", + StorageConnectorFieldScope::Connection, + StorageConnectorFieldKind::Text, + false, + false, + ), + ], + actions: vec![ + draft_connection_test_action_descriptor(), + saved_connection_test_action_descriptor(false), + ], + driver_recommendations: Vec::new(), + related_issues: vec![125], + } + } +} + +#[async_trait(?Send)] +impl StorageConnector for SftpConnector { + fn driver_type() -> DriverType { + DriverType::Sftp + } + + fn normalize_connection_fields(endpoint: &str, bucket: &str) -> Result<(String, String)> { + let _ = bucket; + Ok((SftpDriver::normalize_endpoint(endpoint)?, String::new())) + } + + fn validate_connection_credentials(input: &StorageConnectorConnectionInput) -> Result<()> { + validate_static_secret_credentials(input, "SFTP")?; + SftpDriver::validate_connection_parts( + &input.endpoint, + &input.access_key, + &input.secret_key, + &input.base_path, + ) + } + + fn supports_saved_draft_credentials() -> bool { + true + } + + async fn validate_policy_options( + db: &C, + remote_node_id: Option, + options: &crate::types::StoragePolicyOptions, + ) -> Result<()> { + let _ = (db, remote_node_id); + ensure_onedrive_options_absent(options) + } + + async fn build_draft_driver( + state: &S, + policy: &storage_policy::Model, + ) -> Result> { + let _ = state; + Ok(Box::new(SftpDriver::new(policy)?)) + } + + fn upload_transport(policy: &storage_policy::Model) -> StorageConnectorUploadTransport { + let _ = policy; + StorageConnectorUploadTransport::Sftp + } +} diff --git a/src/storage/connectors/tests.rs b/src/storage/connectors/tests.rs index f976cf0aa..cd354ce67 100644 --- a/src/storage/connectors/tests.rs +++ b/src/storage/connectors/tests.rs @@ -168,14 +168,27 @@ async fn azure_blob_draft_connection_rejects_saved_credential_driver_mismatch() assert_saved_credentials_driver_mismatch_for_driver(DriverType::AzureBlob).await; } +#[tokio::test] +async fn sftp_draft_connection_can_merge_saved_credentials() { + assert!(SftpConnector::supports_saved_draft_credentials()); + assert_saved_credentials_merge_for_driver(DriverType::Sftp).await; +} + +#[tokio::test] +async fn sftp_draft_connection_rejects_saved_credential_driver_mismatch() { + assert!(SftpConnector::supports_saved_draft_credentials()); + assert_saved_credentials_driver_mismatch_for_driver(DriverType::Sftp).await; +} + #[test] fn descriptors_cover_every_storage_driver() { let descriptors = list_storage_driver_descriptors(); - assert_eq!(descriptors.len(), 6); + assert_eq!(descriptors.len(), 7); for driver_type in [ DriverType::Local, DriverType::S3, + DriverType::Sftp, DriverType::AzureBlob, DriverType::TencentCos, DriverType::Remote, @@ -255,6 +268,15 @@ fn descriptors_expose_connector_owned_ui_metadata() { onedrive.ui.config_step_title_key, "policy_wizard_step_onedrive_title" ); + + let sftp = descriptor(DriverType::Sftp); + assert_eq!(sftp.ui.label_key, "driver_type_sftp"); + assert_eq!(sftp.ui.helper_key, "policy_wizard_sftp_helper"); + assert_eq!( + sftp.ui.config_step_description_key, + "policy_wizard_step_sftp_desc" + ); + assert_eq!(sftp.ui.base_path_empty_display, "core:root"); } #[test] @@ -262,6 +284,7 @@ fn connector_registry_covers_every_builtin_storage_driver() { for driver_type in [ DriverType::Local, DriverType::S3, + DriverType::Sftp, DriverType::AzureBlob, DriverType::TencentCos, DriverType::Remote, @@ -307,6 +330,14 @@ fn transfer_strategy_policy_options_are_declared_by_descriptors() { let remote = descriptor(DriverType::Remote); assert!(has_policy_option(&remote, "remote_download_strategy")); assert!(has_policy_option(&remote, "remote_upload_strategy")); + + let sftp = descriptor(DriverType::Sftp); + assert!(!has_policy_option(&sftp, "object_storage_upload_strategy")); + assert!(!has_policy_option( + &sftp, + "object_storage_download_strategy" + )); + assert!(!has_policy_option(&sftp, "s3_path_style")); } #[test] @@ -422,6 +453,34 @@ fn object_storage_connection_field_display_metadata_is_connector_owned() { assert!(!has_policy_option(&tencent_cos, "s3_path_style")); } +#[test] +fn sftp_connection_field_display_metadata_is_connector_owned() { + let sftp = descriptor(DriverType::Sftp); + + assert_eq!(sftp.ui.label_key, "driver_type_sftp"); + assert_eq!(sftp.ui.helper_key, "policy_wizard_sftp_helper"); + assert_eq!(sftp.ui.edit_context_key, "policy_edit_context_sftp_desc"); + assert_eq!( + field(&sftp, "endpoint").placeholder.as_deref(), + Some("sftp://example.com:22") + ); + assert_eq!( + field(&sftp, "endpoint").help_key.as_deref(), + Some("sftp_endpoint_hint") + ); + assert_eq!( + field(&sftp, "endpoint") + .invalid_protocol_message_key + .as_deref(), + Some("sftp_endpoint_protocol_required_error") + ); + assert_eq!(field(&sftp, "access_key").label_key, "access_key"); + assert_eq!(field(&sftp, "secret_key").label_key, "secret_key"); + assert!(sftp.fields.iter().all(|field| field.name != "bucket")); + assert!(!sftp.upload_workflows.presigned_upload); + assert!(!sftp.upload_workflows.object_multipart_upload); +} + #[test] fn object_storage_multipart_etag_requirements_are_connector_owned() { for (driver_type, expected_etag_required) in [ @@ -624,6 +683,7 @@ fn runtime_credential_requirement_is_connector_owned() { for driver_type in [ DriverType::Local, DriverType::S3, + DriverType::Sftp, DriverType::AzureBlob, DriverType::TencentCos, DriverType::Remote, @@ -1007,6 +1067,7 @@ fn non_local_upload_transports_expose_opaque_blob_hash_prefix() { StorageConnectorUploadTransport::Remote(RemoteUploadStrategy::RelayStream), StorageConnectorUploadTransport::Remote(RemoteUploadStrategy::Presigned), StorageConnectorUploadTransport::StreamUpload, + StorageConnectorUploadTransport::Sftp, ] { assert!( transport.opaque_blob_hash_prefix().is_some(), @@ -1032,6 +1093,7 @@ fn presigned_download_policy_is_connector_owned() { 1024, r#"{"object_storage_download_strategy":"relay_stream"}"#, ); + let sftp = mock_policy(DriverType::Sftp, 1024, "{}"); assert!(presigned_download_enabled(&s3).expect("presigned download support should resolve")); assert!( @@ -1040,6 +1102,7 @@ fn presigned_download_policy_is_connector_owned() { assert!( !presigned_download_enabled(&relay_s3).expect("presigned download support should resolve") ); + assert!(!presigned_download_enabled(&sftp).expect("presigned download support should resolve")); } #[test] @@ -1282,6 +1345,45 @@ fn onedrive_uses_server_relay_without_presigned_or_multipart_tracking() { ); } +#[test] +fn sftp_uses_server_relay_without_presigned_or_multipart_tracking() { + let policy = mock_policy(DriverType::Sftp, 1024, "{}"); + let transport = + resolve_policy_upload_transport(&policy).expect("upload transport should resolve"); + + assert_eq!(transport, StorageConnectorUploadTransport::Sftp); + assert_eq!( + transport.resolve_init_mode(&policy, 1024), + UploadMode::Direct + ); + assert_eq!( + transport.resolve_init_mode(&policy, 1025), + UploadMode::Chunked + ); + assert!(!transport.supports_streaming_direct_upload(&policy, 0)); + assert!(transport.supports_streaming_direct_upload(&policy, 1024)); + assert!(!transport.supports_streaming_direct_upload(&policy, 1025)); + assert!(!transport.uses_relay_multipart_tracking()); + assert_eq!(transport.opaque_blob_hash_prefix(), Some("sftp")); + assert_eq!( + transport.chunked_completion(), + StorageConnectorChunkedCompletion::RelayLocalChunksToStreamUpload + ); +} + +#[test] +fn sftp_zero_chunk_size_uses_single_streaming_request() { + let policy = mock_policy(DriverType::Sftp, 0, "{}"); + let transport = + resolve_policy_upload_transport(&policy).expect("upload transport should resolve"); + + assert_eq!( + transport.resolve_init_mode(&policy, i64::MAX), + UploadMode::Direct + ); + assert!(transport.supports_streaming_direct_upload(&policy, i64::MAX)); +} + #[test] fn upload_workflow_descriptors_match_default_connector_transports() { assert_upload_workflow_alignment( @@ -1334,6 +1436,20 @@ fn upload_workflow_descriptors_match_default_connector_transports() { chunked_completion: StorageConnectorChunkedCompletion::RelayLocalChunksToStreamUpload, }, ); + assert_upload_workflow_alignment( + DriverType::Sftp, + "{}", + ExpectedUploadWorkflow { + transport: StorageConnectorUploadTransport::Sftp, + object_multipart: false, + provider_resumable: false, + presigned: false, + frontend_direct_provider_resumable: false, + small_mode: UploadMode::Direct, + large_mode: UploadMode::Chunked, + chunked_completion: StorageConnectorChunkedCompletion::RelayLocalChunksToStreamUpload, + }, + ); assert_upload_workflow_alignment( DriverType::OneDrive, "{}", @@ -1411,7 +1527,8 @@ fn assert_upload_workflow_alignment( StorageConnectorUploadTransport::ObjectStorage(_) => OBJECT_STORAGE_LARGE_UPLOAD_SIZE, StorageConnectorUploadTransport::Local | StorageConnectorUploadTransport::Remote(_) - | StorageConnectorUploadTransport::StreamUpload => 2048, + | StorageConnectorUploadTransport::StreamUpload + | StorageConnectorUploadTransport::Sftp => 2048, }; let descriptor = descriptor(driver_type); let workflows = descriptor.upload_workflows; diff --git a/src/storage/connectors/upload.rs b/src/storage/connectors/upload.rs index 63c909b07..768a26576 100644 --- a/src/storage/connectors/upload.rs +++ b/src/storage/connectors/upload.rs @@ -21,6 +21,9 @@ pub enum StorageConnectorUploadTransport { /// driver may create Microsoft Graph upload sessions internally, but the /// upload service only sees a generic stream-upload target. StreamUpload, + /// SFTP can only be reached by the server. Browsers never receive a + /// provider-native upload URL; uploads are relayed through StreamUploadDriver. + Sftp, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -38,7 +41,7 @@ impl StorageConnectorUploadTransport { pub fn effective_chunk_size(self, policy: &storage_policy::Model) -> i64 { match self { Self::ObjectStorage(_) => effective_object_multipart_chunk_size(policy.chunk_size), - Self::Local | Self::Remote(_) | Self::StreamUpload => policy.chunk_size, + Self::Local | Self::Remote(_) | Self::StreamUpload | Self::Sftp => policy.chunk_size, } } @@ -82,6 +85,7 @@ impl StorageConnectorUploadTransport { Self::Remote(RemoteUploadStrategy::RelayStream) | Self::Remote(RemoteUploadStrategy::Presigned) => true, Self::StreamUpload => true, + Self::Sftp => self.fits_single_request(policy, declared_size), } } @@ -109,6 +113,7 @@ impl StorageConnectorUploadTransport { Self::ObjectStorage(_) => Some("s3"), Self::Remote(_) => Some("remote"), Self::StreamUpload => Some("provider"), + Self::Sftp => Some("sftp"), } } @@ -122,7 +127,7 @@ impl StorageConnectorUploadTransport { // full temp file. They can relay stored chunks into the connector's // stream-upload implementation, which lets the concrete driver own // any provider-native resumable/session behavior internally. - Self::Remote(_) | Self::StreamUpload => { + Self::Remote(_) | Self::StreamUpload | Self::Sftp => { StorageConnectorChunkedCompletion::RelayLocalChunksToStreamUpload } Self::Local | Self::ObjectStorage(_) => { diff --git a/src/storage/drivers/mod.rs b/src/storage/drivers/mod.rs index e090fb66d..00b29b15d 100644 --- a/src/storage/drivers/mod.rs +++ b/src/storage/drivers/mod.rs @@ -9,4 +9,5 @@ pub mod remote; pub mod s3; pub mod s3_compatible; pub mod s3_config; +pub mod sftp; pub mod tencent_cos; diff --git a/src/storage/drivers/sftp.rs b/src/storage/drivers/sftp.rs new file mode 100644 index 000000000..200b677b0 --- /dev/null +++ b/src/storage/drivers/sftp.rs @@ -0,0 +1,839 @@ +//! Storage driver implementation for `sftp`. + +use async_trait::async_trait; +use russh::client::{self, Handler}; +use russh_sftp::client::{Config as SftpClientConfig, SftpSession, error::Error as SftpError}; +use russh_sftp::protocol::StatusCode; +use std::io::SeekFrom; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, ReadBuf}; + +use crate::entities::storage_policy; +use crate::errors::{AsterError, Result}; +use crate::storage::error::{StorageErrorKind, storage_driver_error}; +use crate::storage::{BlobMetadata, StorageDriver, StreamUploadDriver}; + +const DEFAULT_SFTP_PORT: u16 = 22; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const IO_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone)] +struct SftpEndpoint { + host: String, + port: u16, +} + +#[derive(Debug, Clone)] +pub struct SftpDriver { + endpoint: SftpEndpoint, + username: String, + password: String, + base_path: String, +} + +struct TrustServerKeyClient; + +impl Handler for TrustServerKeyClient { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + _server_public_key: &russh::keys::PublicKey, + ) -> std::result::Result { + Ok(true) + } +} + +struct SftpConnection { + _ssh: client::Handle, + sftp: SftpSession, +} + +struct SftpFileReader { + _connection: SftpConnection, + file: russh_sftp::client::fs::File, +} + +impl AsyncRead for SftpFileReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.file).poll_read(cx, buf) + } +} + +impl SftpDriver { + pub fn validate_policy(policy: &storage_policy::Model) -> Result<()> { + Self::validate_connection_parts( + &policy.endpoint, + &policy.access_key, + &policy.secret_key, + &policy.base_path, + ) + } + + pub(crate) fn validate_connection_parts( + endpoint: &str, + username: &str, + password: &str, + base_path: &str, + ) -> Result<()> { + parse_sftp_endpoint(endpoint)?; + validate_connection_secret(username, "access_key")?; + validate_connection_secret(password, "secret_key")?; + normalize_remote_base_path(base_path)?; + Ok(()) + } + + pub(crate) fn normalize_endpoint(endpoint: &str) -> Result { + let endpoint = endpoint.trim(); + parse_sftp_endpoint(endpoint)?; + Ok(endpoint.to_string()) + } + + pub fn new(policy: &storage_policy::Model) -> Result { + Self::validate_policy(policy)?; + Ok(Self { + endpoint: parse_sftp_endpoint(&policy.endpoint)?, + username: policy.access_key.clone(), + password: policy.secret_key.clone(), + base_path: normalize_remote_base_path(&policy.base_path)?, + }) + } + + async fn connect(&self) -> Result { + let mut config = russh::client::Config::default(); + config.inactivity_timeout = Some(IO_TIMEOUT); + config.keepalive_interval = Some(Duration::from_secs(10)); + config.nodelay = true; + + let address = (self.endpoint.host.clone(), self.endpoint.port); + let mut ssh = timeout_io( + "connect SFTP endpoint", + CONNECT_TIMEOUT, + russh::client::connect(Arc::new(config), address, TrustServerKeyClient), + ) + .await? + .map_err(|error| map_ssh_error("connect SFTP endpoint failed", error))?; + + let auth = timeout_io( + "SFTP authentication", + IO_TIMEOUT, + ssh.authenticate_password(self.username.clone(), self.password.clone()), + ) + .await? + .map_err(|error| map_ssh_error("SFTP authentication failed", error))?; + if !auth.success() { + return Err(storage_driver_error( + StorageErrorKind::Auth, + "SFTP authentication failed", + )); + } + + let channel = timeout_io( + "open SSH session channel", + IO_TIMEOUT, + ssh.channel_open_session(), + ) + .await? + .map_err(|error| map_ssh_error("open SSH session channel failed", error))?; + timeout_io( + "open SFTP subsystem", + IO_TIMEOUT, + channel.request_subsystem(true, "sftp"), + ) + .await? + .map_err(|error| map_ssh_error("open SFTP subsystem failed", error))?; + + let sftp_config = SftpClientConfig { + request_timeout_secs: IO_TIMEOUT.as_secs(), + ..Default::default() + }; + let sftp = timeout_io( + "initialize SFTP session", + IO_TIMEOUT, + SftpSession::new_with_config(channel.into_stream(), sftp_config), + ) + .await? + .map_err(|error| map_sftp_error("initialize SFTP session failed", error))?; + sftp.set_timeout(IO_TIMEOUT.as_secs()); + + Ok(SftpConnection { _ssh: ssh, sftp }) + } + + fn full_path(&self, path: &str) -> Result { + let relative = sanitize_relative_storage_path(path)?; + join_remote_path(&self.base_path, &relative) + } + + async fn open_reader(&self, path: &str, offset: u64) -> Result { + let remote_path = self.full_path(path)?; + let connection = self.connect().await?; + let mut file = connection + .sftp + .open(remote_path) + .await + .map_err(|error| map_sftp_error("SFTP open failed", error))?; + if offset > 0 { + file.seek(SeekFrom::Start(offset)) + .await + .map_err(|error| map_io_error("SFTP seek failed", error))?; + } + Ok(SftpFileReader { + _connection: connection, + file, + }) + } +} + +#[async_trait] +impl StorageDriver for SftpDriver { + async fn put(&self, path: &str, data: &[u8]) -> Result { + let remote_path = self.full_path(path)?; + let connection = self.connect().await?; + ensure_remote_parent_dir(&connection.sftp, &remote_path).await?; + let mut file = connection + .sftp + .create(remote_path) + .await + .map_err(|error| map_sftp_error("SFTP create failed", error))?; + file.write_all(data) + .await + .map_err(|error| map_io_error("SFTP write failed", error))?; + file.flush() + .await + .map_err(|error| map_io_error("SFTP flush failed", error))?; + file.shutdown() + .await + .map_err(|error| map_io_error("SFTP close failed", error))?; + Ok(path.to_string()) + } + + async fn get(&self, path: &str) -> Result> { + let remote_path = self.full_path(path)?; + let connection = self.connect().await?; + connection + .sftp + .read(remote_path) + .await + .map_err(|error| map_sftp_error("SFTP read failed", error)) + } + + async fn get_stream(&self, path: &str) -> Result> { + Ok(Box::new(self.open_reader(path, 0).await?)) + } + + async fn get_range( + &self, + path: &str, + offset: u64, + length: Option, + ) -> Result> { + if length == Some(0) { + return Ok(Box::new(tokio::io::empty())); + } + + let reader = self.open_reader(path, offset).await?; + Ok(match length { + Some(len) => Box::new(reader.take(len)), + None => Box::new(reader), + }) + } + + fn supports_efficient_range(&self) -> bool { + true + } + + async fn delete(&self, path: &str) -> Result<()> { + let remote_path = self.full_path(path)?; + let connection = self.connect().await?; + connection + .sftp + .remove_file(remote_path) + .await + .map_err(|error| map_sftp_error("SFTP delete failed", error)) + } + + async fn exists(&self, path: &str) -> Result { + let remote_path = self.full_path(path)?; + let connection = self.connect().await?; + match connection.sftp.metadata(remote_path).await { + Ok(_) => Ok(true), + Err(error) if is_sftp_not_found(&error) => Ok(false), + Err(error) => Err(map_sftp_error("SFTP stat failed", error)), + } + } + + async fn metadata(&self, path: &str) -> Result { + let remote_path = self.full_path(path)?; + let connection = self.connect().await?; + let stat = connection + .sftp + .metadata(remote_path) + .await + .map_err(|error| map_sftp_error("SFTP stat failed", error))?; + Ok(BlobMetadata { + size: stat.size.unwrap_or(0), + content_type: None, + }) + } + + async fn copy_object(&self, src_path: &str, dest_path: &str) -> Result { + let src_remote_path = self.full_path(src_path)?; + let dest_remote_path = self.full_path(dest_path)?; + let connection = self.connect().await?; + ensure_remote_parent_dir(&connection.sftp, &dest_remote_path).await?; + let mut src = connection + .sftp + .open(src_remote_path) + .await + .map_err(|error| map_sftp_error("SFTP source open failed", error))?; + let mut dest = connection + .sftp + .create(dest_remote_path) + .await + .map_err(|error| map_sftp_error("SFTP destination create failed", error))?; + tokio::io::copy(&mut src, &mut dest) + .await + .map_err(|error| map_io_error("SFTP copy failed", error))?; + dest.flush() + .await + .map_err(|error| map_io_error("SFTP copy flush failed", error))?; + dest.shutdown() + .await + .map_err(|error| map_io_error("SFTP copy close failed", error))?; + Ok(dest_path.to_string()) + } + + fn as_stream_upload(&self) -> Option<&dyn StreamUploadDriver> { + Some(self) + } +} + +#[async_trait] +impl StreamUploadDriver for SftpDriver { + async fn put_reader( + &self, + storage_path: &str, + mut reader: Box, + _size: i64, + ) -> Result { + let remote_path = self.full_path(storage_path)?; + let connection = self.connect().await?; + ensure_remote_parent_dir(&connection.sftp, &remote_path).await?; + let mut remote_file = connection + .sftp + .create(remote_path) + .await + .map_err(|error| map_sftp_error("SFTP create failed", error))?; + tokio::io::copy(&mut reader, &mut remote_file) + .await + .map_err(|error| map_io_error("SFTP stream upload failed", error))?; + remote_file + .flush() + .await + .map_err(|error| map_io_error("SFTP stream flush failed", error))?; + remote_file + .shutdown() + .await + .map_err(|error| map_io_error("SFTP stream close failed", error))?; + Ok(storage_path.to_string()) + } + + async fn put_file(&self, storage_path: &str, local_path: &str) -> Result { + let local_file = tokio::fs::File::open(local_path) + .await + .map_err(|error| map_io_error("open local upload file failed", error))?; + self.put_reader(storage_path, Box::new(local_file), -1) + .await + } +} + +async fn timeout_io(context: &'static str, duration: Duration, future: F) -> Result +where + F: std::future::Future, +{ + tokio::time::timeout(duration, future).await.map_err(|_| { + storage_driver_error( + StorageErrorKind::Transient, + format!("{context}: timed out after {}s", duration.as_secs()), + ) + }) +} + +async fn ensure_remote_parent_dir(sftp: &SftpSession, remote_path: &str) -> Result<()> { + let Some(parent) = remote_parent_dir(remote_path) else { + return Ok(()); + }; + ensure_remote_dir(sftp, &parent).await +} + +fn remote_parent_dir(remote_path: &str) -> Option { + let trimmed = remote_path.trim_end_matches('/'); + let index = trimmed.rfind('/')?; + if index == 0 { + Some("/".to_string()) + } else { + Some(trimmed[..index].to_string()) + } +} + +async fn ensure_remote_dir(sftp: &SftpSession, dir: &str) -> Result<()> { + if dir.is_empty() || dir == "." || dir == "/" { + return Ok(()); + } + + let is_absolute = dir.starts_with('/'); + let (_, segments) = sanitize_remote_path_segments(dir, true)?; + let mut current = String::new(); + for segment in segments { + if current.is_empty() { + current = if is_absolute { + format!("/{segment}") + } else { + segment + }; + } else if current == "/" { + current = format!("/{segment}"); + } else { + current.push('/'); + current.push_str(&segment); + } + + match sftp.create_dir(current.clone()).await { + Ok(()) => {} + Err(error) => match sftp.metadata(current.clone()).await { + Ok(metadata) if metadata.file_type().is_dir() => {} + Ok(_) => { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + format!("SFTP mkdir failed: {current} exists and is not a directory"), + )); + } + Err(_) => return Err(map_sftp_error("SFTP mkdir failed", error)), + }, + } + } + Ok(()) +} + +fn parse_sftp_endpoint(endpoint: &str) -> Result { + let endpoint = endpoint.trim(); + if endpoint.is_empty() { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP endpoint is required", + )); + } + + let url_text = if endpoint.contains("://") { + endpoint.to_string() + } else { + format!("sftp://{endpoint}") + }; + let url = url::Url::parse(&url_text).map_err(|error| { + storage_driver_error( + StorageErrorKind::Misconfigured, + format!("invalid SFTP endpoint: {error}"), + ) + })?; + + if url.scheme() != "sftp" { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP endpoint must use the sftp scheme", + )); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP endpoint must not contain credentials; use access_key and secret_key", + )); + } + if url.path() != "/" && !url.path().is_empty() { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP endpoint path is not supported; use base_path for the remote root", + )); + } + if url.query().is_some() || url.fragment().is_some() { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP endpoint must not contain query or fragment", + )); + } + + let host = url.host_str().ok_or_else(|| { + storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP endpoint host is required", + ) + })?; + let port = url.port().unwrap_or(DEFAULT_SFTP_PORT); + + Ok(SftpEndpoint { + host: host + .strip_prefix('[') + .and_then(|host| host.strip_suffix(']')) + .unwrap_or(host) + .to_string(), + port, + }) +} + +fn validate_connection_secret(value: &str, field: &str) -> Result<()> { + if value.trim().is_empty() { + return Err(storage_driver_error( + StorageErrorKind::Auth, + format!("{field} is required for SFTP storage policies"), + )); + } + Ok(()) +} + +fn sanitize_remote_path_segments(path: &str, allow_absolute: bool) -> Result<(bool, Vec)> { + if path.contains('\\') || path.contains('\0') { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + format!("invalid SFTP path: {path}"), + )); + } + + let is_absolute = path.starts_with('/'); + if is_absolute && !allow_absolute { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + format!("SFTP object path must be relative: {path}"), + )); + } + + let mut segments = Vec::new(); + for segment in path.split('/') { + let segment = segment.trim(); + if segment.is_empty() || segment == "." { + continue; + } + if segment == ".." { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + format!("SFTP path escapes base path: {path}"), + )); + } + segments.push(segment.to_string()); + } + + Ok((is_absolute, segments)) +} + +fn sanitize_relative_storage_path(path: &str) -> Result { + let (_, segments) = sanitize_remote_path_segments(path.trim_start_matches('/'), false)?; + Ok(segments.join("/")) +} + +fn normalize_remote_base_path(path: &str) -> Result { + let trimmed = path.trim(); + if trimmed.is_empty() { + return Ok(String::new()); + } + let (is_absolute, segments) = sanitize_remote_path_segments(trimmed, true)?; + if segments.is_empty() { + return Ok(if is_absolute { + "/".to_string() + } else { + String::new() + }); + } + let normalized = segments.join("/"); + Ok(if is_absolute { + format!("/{normalized}") + } else { + normalized + }) +} + +fn join_remote_path(base_path: &str, relative_path: &str) -> Result { + if relative_path.is_empty() { + if base_path.is_empty() { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP storage path cannot be empty", + )); + } + return Ok(base_path.to_string()); + } + + Ok(if base_path.is_empty() { + relative_path.to_string() + } else if base_path == "/" { + format!("/{relative_path}") + } else { + format!("{base_path}/{relative_path}") + }) +} + +fn map_ssh_error(context: &'static str, error: russh::Error) -> AsterError { + storage_driver_error( + classify_error_message(&error.to_string()), + format!("{context}: {error}"), + ) +} + +fn map_sftp_error(context: &'static str, error: SftpError) -> AsterError { + storage_driver_error(classify_sftp_error(&error), format!("{context}: {error}")) +} + +fn map_io_error(context: &'static str, error: std::io::Error) -> AsterError { + storage_driver_error(classify_io_error(&error), format!("{context}: {error}")) +} + +fn classify_sftp_error(error: &SftpError) -> StorageErrorKind { + match error { + SftpError::Status(status) => match status.status_code { + StatusCode::NoSuchFile => StorageErrorKind::NotFound, + StatusCode::PermissionDenied => StorageErrorKind::Permission, + StatusCode::NoConnection | StatusCode::ConnectionLost => StorageErrorKind::Transient, + _ => classify_error_message(&status.error_message), + }, + SftpError::Timeout => StorageErrorKind::Transient, + SftpError::IO(message) + | SftpError::Limited(message) + | SftpError::UnexpectedBehavior(message) => classify_error_message(message), + SftpError::UnexpectedPacket => StorageErrorKind::Unknown, + } +} + +fn classify_io_error(error: &std::io::Error) -> StorageErrorKind { + match error.kind() { + std::io::ErrorKind::NotFound => StorageErrorKind::NotFound, + std::io::ErrorKind::PermissionDenied => StorageErrorKind::Permission, + std::io::ErrorKind::TimedOut + | std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::WouldBlock => StorageErrorKind::Transient, + _ => classify_error_message(&error.to_string()), + } +} + +fn classify_error_message(message: &str) -> StorageErrorKind { + let message = message.to_ascii_lowercase(); + if message.contains("no such file") || message.contains("not found") { + StorageErrorKind::NotFound + } else if message.contains("auth") + || message.contains("password") + || message.contains("permission denied (publickey,password") + { + StorageErrorKind::Auth + } else if message.contains("permission denied") || message.contains("access denied") { + StorageErrorKind::Permission + } else if message.contains("connection") + || message.contains("timed out") + || message.contains("timeout") + || message.contains("eof") + || message.contains("closed") + || message.contains("reset") + { + StorageErrorKind::Transient + } else { + StorageErrorKind::Unknown + } +} + +fn is_sftp_not_found(error: &SftpError) -> bool { + matches!( + error, + SftpError::Status(status) if status.status_code == StatusCode::NoSuchFile + ) || error + .to_string() + .to_ascii_lowercase() + .contains("no such file") +} + +#[cfg(test)] +mod tests { + use super::{ + classify_sftp_error, join_remote_path, normalize_remote_base_path, parse_sftp_endpoint, + sanitize_relative_storage_path, + }; + use crate::storage::error::StorageErrorKind; + use crate::storage::{StorageDriver, StreamUploadDriver}; + use crate::types::{DriverType, StoredStoragePolicyAllowedTypes}; + use chrono::Utc; + use russh_sftp::client::error::Error as SftpError; + use russh_sftp::protocol::{Status, StatusCode}; + use tokio::io::AsyncReadExt; + + #[test] + fn parses_plain_sftp_endpoint_with_default_port() { + let endpoint = parse_sftp_endpoint("example.com").unwrap(); + assert_eq!(endpoint.host, "example.com"); + assert_eq!(endpoint.port, 22); + } + + #[test] + fn parses_sftp_endpoint_with_explicit_port() { + let endpoint = parse_sftp_endpoint("sftp://example.com:2222").unwrap(); + assert_eq!(endpoint.host, "example.com"); + assert_eq!(endpoint.port, 2222); + } + + #[test] + fn parses_ipv6_sftp_endpoint() { + let endpoint = parse_sftp_endpoint("sftp://[::1]:2222").unwrap(); + assert_eq!(endpoint.host, "::1"); + assert_eq!(endpoint.port, 2222); + } + + #[test] + fn rejects_endpoint_credentials_paths_query_and_fragment() { + assert!(parse_sftp_endpoint("").is_err()); + assert!(parse_sftp_endpoint("ftp://example.com").is_err()); + assert!(parse_sftp_endpoint("sftp://user@example.com").is_err()); + assert!(parse_sftp_endpoint("sftp://example.com/uploads").is_err()); + assert!(parse_sftp_endpoint("sftp://example.com?x=1").is_err()); + assert!(parse_sftp_endpoint("sftp://example.com#frag").is_err()); + } + + #[test] + fn normalizes_remote_base_path() { + assert_eq!(normalize_remote_base_path("").unwrap(), ""); + assert_eq!(normalize_remote_base_path("/").unwrap(), "/"); + assert_eq!( + normalize_remote_base_path("/data//uploads/").unwrap(), + "/data/uploads" + ); + assert_eq!( + normalize_remote_base_path("data/./uploads").unwrap(), + "data/uploads" + ); + assert!(normalize_remote_base_path("../data").is_err()); + assert!(normalize_remote_base_path("data\\uploads").is_err()); + assert!(normalize_remote_base_path("data\0uploads").is_err()); + } + + #[test] + fn sanitizes_storage_path_as_relative_path() { + assert_eq!( + sanitize_relative_storage_path("/files/./blob.bin").unwrap(), + "files/blob.bin" + ); + assert!(sanitize_relative_storage_path("../blob.bin").is_err()); + assert!(sanitize_relative_storage_path("folder\\blob.bin").is_err()); + assert!(sanitize_relative_storage_path("folder\0blob.bin").is_err()); + } + + #[test] + fn joins_base_and_relative_paths() { + assert_eq!(join_remote_path("", "files/a.bin").unwrap(), "files/a.bin"); + assert_eq!( + join_remote_path("/data", "files/a.bin").unwrap(), + "/data/files/a.bin" + ); + assert_eq!( + join_remote_path("/", "files/a.bin").unwrap(), + "/files/a.bin" + ); + assert!(join_remote_path("", "").is_err()); + assert_eq!(join_remote_path("/data", "").unwrap(), "/data"); + } + + #[test] + fn classifies_sftp_status_errors() { + let status = |status_code, error_message: &str| { + SftpError::Status(Status { + id: 1, + status_code, + error_message: error_message.to_string(), + language_tag: String::new(), + }) + }; + + assert_eq!( + classify_sftp_error(&status(StatusCode::NoSuchFile, "missing")), + StorageErrorKind::NotFound + ); + assert_eq!( + classify_sftp_error(&status(StatusCode::PermissionDenied, "denied")), + StorageErrorKind::Permission + ); + assert_eq!( + classify_sftp_error(&status(StatusCode::ConnectionLost, "lost")), + StorageErrorKind::Transient + ); + } + + fn env_policy() -> Option { + let endpoint = std::env::var("ASTER_SFTP_TEST_ENDPOINT").ok()?; + let username = std::env::var("ASTER_SFTP_TEST_USERNAME").ok()?; + let password = std::env::var("ASTER_SFTP_TEST_PASSWORD").ok()?; + let base_path = std::env::var("ASTER_SFTP_TEST_BASE_PATH").ok()?; + Some(crate::entities::storage_policy::Model { + id: 1, + name: "sftp acceptance".to_string(), + driver_type: DriverType::Sftp, + endpoint, + bucket: String::new(), + access_key: username, + secret_key: password, + base_path, + remote_node_id: None, + remote_storage_target_key: None, + max_file_size: 0, + allowed_types: StoredStoragePolicyAllowedTypes::empty(), + options: crate::types::StoredStoragePolicyOptions::empty(), + is_default: false, + chunk_size: 1024, + created_at: Utc::now(), + updated_at: Utc::now(), + }) + } + + #[tokio::test] + #[ignore = "requires ASTER_SFTP_TEST_* environment variables and a reachable SFTP server"] + async fn real_sftp_driver_round_trip_uses_streaming_upload() { + let Some(policy) = env_policy() else { + eprintln!("skipping real SFTP test because ASTER_SFTP_TEST_* is not set"); + return; + }; + let driver = super::SftpDriver::new(&policy).unwrap(); + let test_root = format!("codex-acceptance/{}", uuid::Uuid::new_v4()); + let object_path = format!("{test_root}/streamed.bin"); + let copy_path = format!("{test_root}/copied.bin"); + let payload = b"hello from asterdrive sftp streaming"; + + driver + .put_reader( + &object_path, + Box::new(std::io::Cursor::new(payload.to_vec())), + payload.len() as i64, + ) + .await + .unwrap(); + + assert!(driver.exists(&object_path).await.unwrap()); + assert_eq!( + driver.metadata(&object_path).await.unwrap().size, + payload.len() as u64 + ); + assert_eq!(driver.get(&object_path).await.unwrap(), payload); + + let mut range = driver.get_range(&object_path, 6, Some(4)).await.unwrap(); + let mut range_bytes = Vec::new(); + range.read_to_end(&mut range_bytes).await.unwrap(); + assert_eq!(range_bytes, b"from"); + + driver.copy_object(&object_path, ©_path).await.unwrap(); + assert_eq!(driver.get(©_path).await.unwrap(), payload); + + driver.delete(&object_path).await.unwrap(); + driver.delete(©_path).await.unwrap(); + assert!(!driver.exists(&object_path).await.unwrap()); + } +} diff --git a/src/storage/registry.rs b/src/storage/registry.rs index 5face5d95..ffe9b7f44 100644 --- a/src/storage/registry.rs +++ b/src/storage/registry.rs @@ -5,6 +5,7 @@ use super::drivers::azure_blob::AzureBlobDriver; use super::drivers::local::LocalDriver; use super::drivers::remote::RemoteDriver; use super::drivers::s3::S3Driver; +use super::drivers::sftp::SftpDriver; use super::drivers::tencent_cos::TencentCosDriver; use super::error::storage_driver_error; use super::metrics_driver::{MetricsMultipartStorageDriver, MetricsStorageDriver}; @@ -369,6 +370,10 @@ impl DriverRegistry { let multipart: Arc = driver; Ok(self.build_entry(policy.driver_type, storage, Some(multipart))) } + DriverType::Sftp => { + let driver: Arc = Arc::new(SftpDriver::new(policy)?); + Ok(self.build_entry(policy.driver_type, driver, None)) + } DriverType::AzureBlob => { let driver = Arc::new(AzureBlobDriver::new(policy)?); let storage: Arc = driver.clone(); diff --git a/src/types/storage_policy.rs b/src/types/storage_policy.rs index 3698f084f..a7f6942fe 100644 --- a/src/types/storage_policy.rs +++ b/src/types/storage_policy.rs @@ -20,6 +20,8 @@ pub enum DriverType { Local, #[sea_orm(string_value = "s3")] S3, + #[sea_orm(string_value = "sftp")] + Sftp, #[sea_orm(string_value = "azure_blob")] AzureBlob, #[sea_orm(string_value = "tencent_cos")] @@ -35,6 +37,7 @@ impl DriverType { match self { Self::Local => "local", Self::S3 => "s3", + Self::Sftp => "sftp", Self::AzureBlob => "azure_blob", Self::TencentCos => "tencent_cos", Self::Remote => "remote", @@ -46,6 +49,7 @@ impl DriverType { match value { "local" => Some(Self::Local), "s3" => Some(Self::S3), + "sftp" => Some(Self::Sftp), "azure_blob" => Some(Self::AzureBlob), "tencent_cos" => Some(Self::TencentCos), "remote" => Some(Self::Remote), From 42dc9022c80f70dfc6d489e4ccbfd553823f601e Mon Sep 17 00:00:00 2001 From: AptS-1547 Date: Mon, 6 Jul 2026 20:51:48 +0800 Subject: [PATCH 2/5] refactor(storage): derive SFTP connection fields and validation from descriptor metadata Remove hardcoded SFTP/S3/Azure field matrices from frontend predicates and normalization logic. Replace form-driven capability inference with descriptor-first field derivation: - **Backend descriptor enhancement** - Add `allowed_endpoint_protocols` and `allow_endpoint_without_protocol` to field descriptor schema - Mark SFTP endpoint as scheme-optional with `sftp:` protocol constraint - Add custom labels `sftp_username` and `sftp_password` for SFTP credentials - Set `trim_on_blur: true` for SFTP access_key field - **Frontend normalization refactor** - Replace `shouldUse*Connection()` predicates that check form content with pure `supports*Connection()` descriptor queries - Remove `hasStaticSecretConnectionFields()` and `hasObjectStorageConnectionFields()` form-based inference helpers - Derive endpoint validation rules (`allowed_protocols`, `allow_endpoint_without_protocol`) directly from descriptor field metadata - Fix `hasEndpointUrlScheme()` regex to require `://` instead of accepting bare `:` - **Test coverage** - Add SFTP wizard integration test verifying SSH credential labels and scheme-less endpoint acceptance - Update existing S3/Azure test fixtures to pass explicit descriptors to all payload/normalization/validation helpers - Add backend unit test assertions for SFTP field `label_key`, `allowed_endpoint_protocols`, and `allow_endpoint_without_protocol` - Add new SFTP driver integration test covering upload, download, range requests, copy, stream upload, and file upload operations - **I18n** - Add `sftp_username` and `sftp_password` translation keys to English and Chinese policy locales This change enforces the AGENTS.md rule prohibiting new driver-type white-list predicates and ensures connection field behavior is governed by backend connector metadata, not frontend assumptions. --- frontend-panel/AGENTS.md | 1 + .../connectionNormalization.ts | 97 ++----- .../admin/storagePolicyDialogShared.test.ts | 256 +++++++++++++----- .../src/i18n/locales/en/admin/policies.json | 2 + .../src/i18n/locales/zh/admin/policies.json | 2 + .../pages/admin/AdminPoliciesPage.test.tsx | 79 +++++- frontend-panel/src/services/api.generated.ts | 4 + src/storage/connector_descriptor.rs | 26 ++ src/storage/connectors/sftp.rs | 48 +++- src/storage/connectors/tests.rs | 10 +- src/storage/drivers/sftp.rs | 4 +- tests/test_sftp.rs | 179 ++++++++++++ 12 files changed, 537 insertions(+), 171 deletions(-) create mode 100644 tests/test_sftp.rs diff --git a/frontend-panel/AGENTS.md b/frontend-panel/AGENTS.md index 96da92bad..643a8e607 100644 --- a/frontend-panel/AGENTS.md +++ b/frontend-panel/AGENTS.md @@ -63,6 +63,7 @@ import type { UserInfo, ShareInfo, StoragePolicy } from "@/types/api"; ### 存储策略能力 - 存储策略 UI 的连接测试、字段显示、上传工作流、授权入口、远端节点绑定、S3 传输策略、存储原生处理、driver action 可用性等能力判断,必须优先来自后端 storage connector descriptor 的 `capabilities`、`fields`、`upload_workflows`、`actions`、`connection_tests` 等元数据。 - 不要在前端新增 `driver_type === "s3" || driver_type === "azure_blob" || ...` 这类白名单/矩阵来推断能力。已有临时兜底也只能用于 descriptor 缺失时的保守兼容,不能扩大成新的事实来源。 +- 存储策略表单的连接字段、字段标签、credential mode、endpoint 协议、是否允许裸 host、测试 payload 和 create/update payload,都必须从 descriptor 字段和后端 schema 派生;不要在 `storage-policy-dialog`、`descriptorPredicates`、`connectionNormalization` 或页面测试 fixture 里硬编码 S3/Azure/COS/SFTP/Remote/OneDrive 的字段矩阵。 - descriptor 缺失时,如果必须根据表单内容做兼容判断,只能使用字段是否存在、用户是否实际填写了相关字段、URL scheme 这类局部信号;不要把 S3/Azure/COS/Remote/OneDrive 的能力关系重新硬编码回前端。 ### 组件库 diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts b/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts index 192c258f6..83c14106a 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts +++ b/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts @@ -101,10 +101,7 @@ export function normalizePolicyForm( form: PolicyFormData, descriptor?: StorageConnectorDescriptor | null, ): PolicyFormData { - const shouldNormalizeConnection = shouldUseStaticSecretConnection( - form, - descriptor, - ); + const shouldNormalizeConnection = supportsStaticSecretConnection(descriptor); const shouldNormalizeMicrosoftGraph = shouldUseMicrosoftGraphConfig( form, descriptor, @@ -142,10 +139,8 @@ export function normalizePolicyForm( : normalized; } - const usesObjectStorageConnection = shouldUseObjectStorageConnection( - form, - descriptor, - ); + const usesObjectStorageConnection = + supportsObjectStorageConnection(descriptor); const normalized = usesObjectStorageConnection ? normalizeObjectStorageConnectionFields(form.endpoint, form.bucket) : { @@ -199,9 +194,7 @@ export function hasConnectionFieldChanges( return true; } - if ( - shouldUseStaticSecretConnection(normalizedForm, descriptor, editingPolicy) - ) { + if (supportsStaticSecretConnection(descriptor)) { return ( normalizedForm.endpoint !== editingPolicy.endpoint || normalizedForm.bucket !== editingPolicy.bucket || @@ -211,7 +204,7 @@ export function hasConnectionFieldChanges( ); } - if (shouldUseRemoteNodeBinding(descriptor)) { + if (supportsRemoteNodeBinding(descriptor)) { return ( parseRemoteNodeId(normalizedForm.remote_node_id) !== editingPolicy.remote_node_id || @@ -258,11 +251,7 @@ export function getEndpointValidationMessage( t: (key: string) => string, descriptor?: StorageConnectorDescriptor | null, ) { - if ( - descriptor - ? !supportsStaticSecretConnection(descriptor) - : !hasStaticSecretConnectionFields(form) - ) { + if (!supportsStaticSecretConnection(descriptor)) { return null; } @@ -270,14 +259,19 @@ export function getEndpointValidationMessage( if (!trimmedEndpoint) { return null; } + const endpointField = descriptor?.fields.find( + (field) => field.scope === "connection" && field.name === "endpoint", + ); const endpointProtocolMessage = - descriptor?.fields.find( - (field) => field.scope === "connection" && field.name === "endpoint", - )?.invalid_protocol_message_key ?? "s3_endpoint_protocol_required_error"; + endpointField?.invalid_protocol_message_key ?? + "s3_endpoint_protocol_required_error"; + const allowedProtocols = + endpointField?.allowed_endpoint_protocols?.length === 0 + ? ["http:", "https:"] + : (endpointField?.allowed_endpoint_protocols ?? ["http:", "https:"]); - const allowedProtocols = allowedEndpointProtocols(descriptor); if (!hasEndpointUrlScheme(trimmedEndpoint)) { - return allowedProtocols.includes("sftp:") + return endpointField?.allow_endpoint_without_protocol ? null : t(endpointProtocolMessage); } @@ -296,26 +290,6 @@ export function getEndpointValidationMessage( return null; } -function shouldUseObjectStorageConnection( - form: PolicyFormData, - descriptor?: StorageConnectorDescriptor | null, - editingPolicy?: StoragePolicy | null, -) { - return descriptor - ? supportsObjectStorageConnection(descriptor) - : hasObjectStorageConnectionFields(form, editingPolicy); -} - -function shouldUseStaticSecretConnection( - form: PolicyFormData, - descriptor?: StorageConnectorDescriptor | null, - editingPolicy?: StoragePolicy | null, -) { - return descriptor - ? supportsStaticSecretConnection(descriptor) - : hasStaticSecretConnectionFields(form, editingPolicy); -} - function shouldUseMicrosoftGraphConfig( form: PolicyFormData, descriptor?: StorageConnectorDescriptor | null, @@ -328,45 +302,8 @@ function shouldUseMicrosoftGraphConfig( hasMicrosoftGraphPolicyOptions(editingPolicy); } -function shouldUseRemoteNodeBinding( - descriptor: StorageConnectorDescriptor | null | undefined, -) { - return descriptor ? supportsRemoteNodeBinding(descriptor) : false; -} - -function hasObjectStorageConnectionFields( - form: PolicyFormData, - editingPolicy?: StoragePolicy | null, -) { - return Boolean( - hasEndpointUrlScheme(form.endpoint) || - hasText(form.bucket) || - hasText(form.access_key) || - hasText(form.secret_key) || - hasEndpointUrlScheme(editingPolicy?.endpoint) || - hasText(editingPolicy?.bucket), - ); -} - -function hasStaticSecretConnectionFields( - form: PolicyFormData, - editingPolicy?: StoragePolicy | null, -) { - return Boolean( - hasObjectStorageConnectionFields(form, editingPolicy) || - hasText(form.endpoint) || - hasText(editingPolicy?.endpoint), - ); -} - function hasEndpointUrlScheme(endpoint?: string | null) { - return /^[a-z][a-z0-9+.-]*:/i.test(endpoint?.trim() ?? ""); -} - -function allowedEndpointProtocols( - descriptor?: StorageConnectorDescriptor | null, -) { - return descriptor?.driver_type === "sftp" ? ["sftp:"] : ["http:", "https:"]; + return /^[a-z][a-z0-9+.-]*:\/\//i.test(endpoint?.trim() ?? ""); } function shouldTrimConnectionField( diff --git a/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts b/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts index 7be75bace..7cf47da98 100644 --- a/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts +++ b/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts @@ -53,6 +53,52 @@ describe("storage policy dialog helper modules", () => { object_multipart_upload: false, }, } as never; + const staticSecretConnectionDescriptor = ({ + protocolMessage = "s3_endpoint_protocol_required_error", + allowedProtocols = ["http:", "https:"], + allowEndpointWithoutProtocol = false, + includeS3PathStyle = false, + }: { + protocolMessage?: string; + allowedProtocols?: string[]; + allowEndpointWithoutProtocol?: boolean; + includeS3PathStyle?: boolean; + } = {}) => + ({ + credential_mode: "static_secret", + fields: [ + { + allow_endpoint_without_protocol: allowEndpointWithoutProtocol, + allowed_endpoint_protocols: allowedProtocols, + invalid_protocol_message_key: protocolMessage, + name: "endpoint", + scope: "connection", + }, + { name: "bucket", scope: "connection" }, + { name: "access_key", scope: "connection" }, + { name: "secret_key", scope: "connection" }, + { + name: "object_storage_upload_strategy", + scope: "policy_options", + }, + { + name: "object_storage_download_strategy", + scope: "policy_options", + }, + ...(includeS3PathStyle + ? [{ name: "s3_path_style", scope: "policy_options" }] + : []), + ], + upload_workflows: { + object_multipart_upload: true, + }, + }) as never; + const sftpConnectionDescriptor = () => + staticSecretConnectionDescriptor({ + allowedProtocols: ["sftp:"], + allowEndpointWithoutProtocol: true, + protocolMessage: "sftp_endpoint_protocol_required_error", + }); it("uses connector descriptor endpoint rules for specialized driver promotion", () => { const s3Descriptor = { @@ -359,28 +405,34 @@ describe("storage policy dialog helper modules", () => { }); it("builds create payloads with trimmed S3 fields", () => { + const s3Descriptor = staticSecretConnectionDescriptor({ + includeS3PathStyle: true, + }); expect( - buildCreatePolicyPayload({ - name: "Media", - driver_type: "s3", - endpoint: " https://s3.example.test/custom/path ", - bucket: " photos ", - access_key: "AKIA", - secret_key: "SECRET", - base_path: "videos", - remote_node_id: "", - max_file_size: "2048", - chunk_size: "6", - is_default: false, - content_dedup: false, - remote_download_strategy: "relay_stream", - remote_upload_strategy: "relay_stream", - object_storage_upload_strategy: "presigned", - object_storage_download_strategy: "relay_stream", - storage_native_processing_enabled: false, - thumbnail_processor: null, - thumbnail_extensions: [], - }), + buildCreatePolicyPayload( + { + name: "Media", + driver_type: "s3", + endpoint: " https://s3.example.test/custom/path ", + bucket: " photos ", + access_key: "AKIA", + secret_key: "SECRET", + base_path: "videos", + remote_node_id: "", + max_file_size: "2048", + chunk_size: "6", + is_default: false, + content_dedup: false, + remote_download_strategy: "relay_stream", + remote_upload_strategy: "relay_stream", + object_storage_upload_strategy: "presigned", + object_storage_download_strategy: "relay_stream", + storage_native_processing_enabled: false, + thumbnail_processor: null, + thumbnail_extensions: [], + }, + s3Descriptor, + ), ).toEqual({ name: "Media", driver_type: "s3", @@ -393,35 +445,42 @@ describe("storage policy dialog helper modules", () => { chunk_size: 6 * 1024 * 1024, is_default: false, options: { + object_storage_download_strategy: "relay_stream", object_storage_upload_strategy: "presigned", }, remote_node_id: undefined, + remote_storage_target_key: undefined, }); expect( - buildCreatePolicyPayload({ - name: "Virtual Hosted S3", - driver_type: "s3", - endpoint: "https://s3.example.test", - bucket: "photos", - access_key: "AKIA", - secret_key: "SECRET", - base_path: "", - remote_node_id: "", - max_file_size: "", - chunk_size: "5", - is_default: false, - content_dedup: false, - remote_download_strategy: "relay_stream", - remote_upload_strategy: "relay_stream", - object_storage_upload_strategy: "relay_stream", - object_storage_download_strategy: "relay_stream", - s3_path_style: false, - storage_native_processing_enabled: false, - thumbnail_processor: null, - thumbnail_extensions: [], - }).options, + buildCreatePolicyPayload( + { + name: "Virtual Hosted S3", + driver_type: "s3", + endpoint: "https://s3.example.test", + bucket: "photos", + access_key: "AKIA", + secret_key: "SECRET", + base_path: "", + remote_node_id: "", + max_file_size: "", + chunk_size: "5", + is_default: false, + content_dedup: false, + remote_download_strategy: "relay_stream", + remote_upload_strategy: "relay_stream", + object_storage_upload_strategy: "relay_stream", + object_storage_download_strategy: "relay_stream", + s3_path_style: false, + storage_native_processing_enabled: false, + thumbnail_processor: null, + thumbnail_extensions: [], + }, + s3Descriptor, + ).options, ).toEqual({ + object_storage_upload_strategy: "relay_stream", + object_storage_download_strategy: "relay_stream", s3_path_style: false, }); }); @@ -653,20 +712,23 @@ describe("storage policy dialog helper modules", () => { storage_native_media_metadata_enabled: false, media_metadata_extensions: [], }; + const s3Descriptor = staticSecretConnectionDescriptor(); - expect(getEndpointValidationMessage(baseForm, t)).toBe( + expect(getEndpointValidationMessage(baseForm, t, s3Descriptor)).toBe( "s3_endpoint_protocol_required_error", ); expect( getEndpointValidationMessage( { ...baseForm, endpoint: "http://s3.example.com" }, t, + s3Descriptor, ), ).toBeNull(); expect( getEndpointValidationMessage( { ...baseForm, endpoint: "https://s3.example.com" }, t, + s3Descriptor, ), ).toBeNull(); expect( @@ -677,6 +739,7 @@ describe("storage policy dialog helper modules", () => { endpoint: "cos.ap-guangzhou.myqcloud.com", }, t, + s3Descriptor, ), ).toBe("s3_endpoint_protocol_required_error"); expect( @@ -689,9 +752,10 @@ describe("storage policy dialog helper modules", () => { secret_key: "", }, t, - null, + s3Descriptor, ), ).toBe("s3_endpoint_protocol_required_error"); + expect(getEndpointValidationMessage(baseForm, t, null)).toBeNull(); expect( getEndpointValidationMessage( { @@ -729,22 +793,9 @@ describe("storage policy dialog helper modules", () => { storage_native_media_metadata_enabled: false, media_metadata_extensions: [], }; - const azureDescriptor = { - fields: [ - { - invalid_protocol_message_key: - "azure_blob_endpoint_protocol_required_error", - name: "endpoint", - scope: "connection", - }, - { name: "bucket", scope: "connection" }, - { name: "access_key", scope: "connection" }, - { name: "secret_key", scope: "connection" }, - ], - upload_workflows: { - object_multipart_upload: true, - }, - } as never; + const azureDescriptor = staticSecretConnectionDescriptor({ + protocolMessage: "azure_blob_endpoint_protocol_required_error", + }); expect(getEndpointValidationMessage(baseForm, t, azureDescriptor)).toBe( "azure_blob_endpoint_protocol_required_error", @@ -771,10 +822,54 @@ describe("storage policy dialog helper modules", () => { endpoint: "ftp://s3.example.com", }, t, + staticSecretConnectionDescriptor(), ), ).toBe("s3_endpoint_protocol_required_error"); }); + it("treats bare SFTP host and host port endpoints as scheme-less", () => { + const baseForm = { + name: "SFTP", + driver_type: "sftp" as const, + endpoint: "sftp.example.com:2222", + bucket: "", + access_key: "aster", + secret_key: "secret", + base_path: "/upload", + remote_node_id: "", + max_file_size: "", + chunk_size: "5", + is_default: false, + content_dedup: false, + remote_download_strategy: "relay_stream" as const, + remote_upload_strategy: "relay_stream" as const, + object_storage_upload_strategy: "relay_stream" as const, + object_storage_download_strategy: "relay_stream" as const, + storage_native_processing_enabled: false, + thumbnail_processor: null, + thumbnail_extensions: [], + storage_native_media_metadata_enabled: false, + media_metadata_extensions: [], + }; + const descriptor = sftpConnectionDescriptor(); + + expect(getEndpointValidationMessage(baseForm, t, descriptor)).toBeNull(); + expect( + getEndpointValidationMessage( + { ...baseForm, endpoint: "sftp://sftp.example.com:2222" }, + t, + descriptor, + ), + ).toBeNull(); + expect( + getEndpointValidationMessage( + { ...baseForm, endpoint: "https://sftp.example.com" }, + t, + descriptor, + ), + ).toBe("sftp_endpoint_protocol_required_error"); + }); + it("builds Tencent COS CORS draft action payloads with optional saved policy reuse", () => { const form = { name: "COS Media", @@ -799,8 +894,9 @@ describe("storage policy dialog helper modules", () => { storage_native_media_metadata_enabled: true, media_metadata_extensions: [" mp4 "], }; + const tencentCosDescriptor = staticSecretConnectionDescriptor(); - expect(buildTencentCosCorsPayload(form, 34)).toEqual({ + expect(buildTencentCosCorsPayload(form, 34, tencentCosDescriptor)).toEqual({ action: "configure_tencent_cos_cors", policy_id: 34, driver_type: "tencent_cos", @@ -810,6 +906,7 @@ describe("storage policy dialog helper modules", () => { secret_key: undefined, base_path: "tenant-a", remote_node_id: undefined, + remote_storage_target_key: undefined, options: { object_storage_upload_strategy: "presigned", object_storage_download_strategy: "presigned", @@ -820,7 +917,9 @@ describe("storage policy dialog helper modules", () => { media_metadata_extensions: ["mp4"], }, }); - expect(buildTencentCosCorsPayload(form, null).policy_id).toBeUndefined(); + expect( + buildTencentCosCorsPayload(form, null, tencentCosDescriptor).policy_id, + ).toBeUndefined(); }); it("omits empty credentials from update payloads", () => { @@ -1029,9 +1128,15 @@ describe("storage policy dialog helper modules", () => { storage_native_media_metadata_enabled: false, media_metadata_extensions: [], }; - const payload = buildCreatePolicyPayload({ - ...azureForm, + const azureDescriptor = staticSecretConnectionDescriptor({ + protocolMessage: "azure_blob_endpoint_protocol_required_error", }); + const payload = buildCreatePolicyPayload( + { + ...azureForm, + }, + azureDescriptor, + ); expect(payload).toEqual({ name: "Azure Archive", @@ -1046,18 +1151,19 @@ describe("storage policy dialog helper modules", () => { chunk_size: 8 * 1024 * 1024, is_default: false, options: { + object_storage_download_strategy: "relay_stream", object_storage_upload_strategy: "presigned", - s3_path_style: false, }, + remote_storage_target_key: undefined, }); - expect(normalizePolicyForm(azureForm)).toEqual({ + expect(normalizePolicyForm(azureForm, azureDescriptor)).toEqual({ ...azureForm, endpoint: "https://acct.blob.core.windows.net/", bucket: "container-a", access_key: "account-name", secret_key: "account-key", }); - expect(getPolicyConnectionTestKey(azureForm)).toBe( + expect(getPolicyConnectionTestKey(azureForm, azureDescriptor)).toBe( JSON.stringify({ driver_type: "azure_blob", endpoint: "https://acct.blob.core.windows.net/", @@ -1068,7 +1174,7 @@ describe("storage policy dialog helper modules", () => { remote_node_id: undefined, options: { object_storage_upload_strategy: "presigned", - s3_path_style: false, + object_storage_download_strategy: "relay_stream", }, }), ); @@ -1120,6 +1226,7 @@ describe("storage policy dialog helper modules", () => { created_at: "", updated_at: "", } as StoragePolicy, + azureDescriptor, ), ).toBe(false); }); @@ -1549,10 +1656,17 @@ describe("storage policy dialog helper modules", () => { }, } as StoragePolicy; const s3Form = getPolicyForm(s3Policy); + const s3Descriptor = staticSecretConnectionDescriptor(); - expect(hasConnectionFieldChanges(s3Form, s3Policy)).toBe(false); + expect(hasConnectionFieldChanges(s3Form, s3Policy, s3Descriptor)).toBe( + false, + ); expect( - hasConnectionFieldChanges({ ...s3Form, secret_key: "SECRET" }, s3Policy), + hasConnectionFieldChanges( + { ...s3Form, secret_key: "SECRET" }, + s3Policy, + s3Descriptor, + ), ).toBe(true); const oneDrivePolicy = { diff --git a/frontend-panel/src/i18n/locales/en/admin/policies.json b/frontend-panel/src/i18n/locales/en/admin/policies.json index 69c30a3f5..ded7abcec 100644 --- a/frontend-panel/src/i18n/locales/en/admin/policies.json +++ b/frontend-panel/src/i18n/locales/en/admin/policies.json @@ -167,6 +167,7 @@ "driver_type_azure_blob": "Azure Blob", "driver_type_onedrive": "OneDrive", "access_key": "Access Key", + "sftp_username": "SSH Username", "azure_blob_account_name": "Storage Account Name", "azure_blob_account_key": "Storage Account Key", "base_url": "Base URL", @@ -174,6 +175,7 @@ "remote_node": "Remote Node", "select_remote_node_placeholder": "Select a remote node", "secret_key": "Secret Key", + "sftp_password": "SSH Password", "policy_wizard_progress": "Step {{current}} of {{total}}", "policy_wizard_step_storage_title": "Choose Storage", "policy_wizard_step_storage_desc": "Pick the storage backend for this policy.", diff --git a/frontend-panel/src/i18n/locales/zh/admin/policies.json b/frontend-panel/src/i18n/locales/zh/admin/policies.json index e0ba6bf07..dfc90c154 100644 --- a/frontend-panel/src/i18n/locales/zh/admin/policies.json +++ b/frontend-panel/src/i18n/locales/zh/admin/policies.json @@ -167,6 +167,7 @@ "driver_type_azure_blob": "Azure Blob", "driver_type_onedrive": "OneDrive", "access_key": "访问密钥", + "sftp_username": "SSH 用户名", "azure_blob_account_name": "存储账户名", "azure_blob_account_key": "存储账户密钥", "base_url": "基础地址", @@ -174,6 +175,7 @@ "remote_node": "远程节点", "select_remote_node_placeholder": "选择远程节点", "secret_key": "密钥", + "sftp_password": "SSH 密码", "policy_wizard_progress": "第 {{current}} 步,共 {{total}} 步", "policy_wizard_step_storage_title": "选择存储类型", "policy_wizard_step_storage_desc": "先确定这个策略的存储后端。", diff --git a/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx b/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx index f378f73aa..405746af2 100644 --- a/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx +++ b/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx @@ -80,6 +80,10 @@ const mockTranslate = vi.hoisted( return "Account Name"; case "azure_blob_account_key": return "Account Key"; + case "sftp_username": + return "SSH Username"; + case "sftp_password": + return "SSH Password"; case "access_key": return "Access Key"; case "secret_key": @@ -995,6 +999,7 @@ function createStorageDriverDescriptors() { presigned_download: true, object_storage_transfer_strategy: true, }, + credential_mode: "static_secret", driver_recommendations: [ { target_driver_type: "tencent_cos", @@ -1020,6 +1025,8 @@ function createStorageDriverDescriptors() { credential_mode: "static_secret", fields: [ fieldDescriptor("endpoint", "connection", "text", { + allowed_endpoint_protocols: ["sftp:"], + allow_endpoint_without_protocol: true, help_key: "sftp_endpoint_hint", invalid_protocol_message_key: "sftp_endpoint_protocol_required_error", placeholder: "sftp://example.com:22", @@ -1027,9 +1034,12 @@ function createStorageDriverDescriptors() { trim_on_blur: true, }), fieldDescriptor("access_key", "connection", "text", { + label_key: "sftp_username", required: true, + trim_on_blur: true, }), fieldDescriptor("secret_key", "connection", "secret", { + label_key: "sftp_password", required: true, }), fieldDescriptor("base_path", "connection", "text"), @@ -1072,6 +1082,7 @@ function createStorageDriverDescriptors() { storage_native_media_metadata: true, storage_native_thumbnail: true, }, + credential_mode: "static_secret", fields: [ ...objectStorageConnectionFields("tencent_cos"), ...objectStoragePolicyOptionFields, @@ -1088,6 +1099,7 @@ function createStorageDriverDescriptors() { presigned_download: true, object_storage_transfer_strategy: true, }, + credential_mode: "static_secret", fields: [ ...objectStorageConnectionFields("azure_blob"), ...objectStoragePolicyOptionFields, @@ -1184,7 +1196,7 @@ function openCreateWizard( } else if (driver === "s3") { fireEvent.click(screen.getByRole("button", { name: /^S3\b/ })); } else if (driver === "sftp") { - fireEvent.click(screen.getByRole("button", { name: /^SFTP\b/ })); + fireEvent.click(screen.getByRole("button", { name: /\bSFTP\b/ })); } else if (driver === "tencent_cos") { fireEvent.click(screen.getByRole("button", { name: /Tencent COS/ })); } else if (driver === "azure_blob") { @@ -2284,7 +2296,7 @@ describe("AdminPoliciesPage", () => { }); expect(screen.queryByLabelText("bucket")).not.toBeInTheDocument(); - expect(screen.queryByLabelText("endpoint")).not.toBeInTheDocument(); + expect(screen.getByLabelText("endpoint")).toBeInTheDocument(); fireEvent.click( screen.getByRole("button", { name: "policy_wizard_review" }), @@ -3047,6 +3059,69 @@ describe("AdminPoliciesPage", () => { expect(mockState.toastSuccess).toHaveBeenCalledWith("policy_created"); }); + it("tests and creates an SFTP policy with SSH credentials", async () => { + render(); + + openCreateWizard("sftp"); + + expect(screen.getByText("policy_wizard_sftp_helper")).toBeInTheDocument(); + expect(screen.getByText("sftp_endpoint_hint")).toBeInTheDocument(); + + fireEvent.change(screen.getByLabelText("core:name"), { + target: { value: "SFTP Archive" }, + }); + fireEvent.change(screen.getByLabelText("base_path"), { + target: { value: "/upload/archive" }, + }); + fireEvent.change(screen.getByLabelText("endpoint"), { + target: { value: "sftp.example.com:2222" }, + }); + fireEvent.change(screen.getByLabelText("SSH Username"), { + target: { value: "aster" }, + }); + fireEvent.change(screen.getByLabelText("SSH Password"), { + target: { value: "SFTPSECRET" }, + }); + + fireEvent.click(screen.getByRole("button", { name: /test_connection/i })); + + await waitFor(() => { + expect(mockState.testParams).toHaveBeenCalledWith({ + access_key: "aster", + base_path: "/upload/archive", + bucket: undefined, + driver_type: "sftp", + endpoint: "sftp.example.com:2222", + options: {}, + remote_node_id: undefined, + secret_key: "SFTPSECRET", + }); + }); + + fireEvent.click( + screen.getByRole("button", { name: "policy_wizard_review" }), + ); + fireEvent.click(screen.getByRole("button", { name: /core:create/i })); + + await waitFor(() => { + expect(mockState.create).toHaveBeenCalledWith({ + access_key: "aster", + base_path: "/upload/archive", + bucket: "", + chunk_size: 5 * 1024 * 1024, + driver_type: "sftp", + endpoint: "sftp.example.com:2222", + is_default: false, + max_file_size: undefined, + name: "SFTP Archive", + options: {}, + remote_node_id: undefined, + secret_key: "SFTPSECRET", + }); + }); + expect(mockState.toastSuccess).toHaveBeenCalledWith("policy_created"); + }); + it("suggests the specialized driver when a generic S3 create uses a known provider endpoint", () => { render(); diff --git a/frontend-panel/src/services/api.generated.ts b/frontend-panel/src/services/api.generated.ts index 902211739..e9b0db3a0 100644 --- a/frontend-panel/src/services/api.generated.ts +++ b/frontend-panel/src/services/api.generated.ts @@ -7351,6 +7351,10 @@ export interface components { equals?: string | null; }; StorageConnectorFieldDescriptor: { + /** @description true 表示 endpoint 可以省略 URL scheme,由 connector 在后端补齐或解释。 */ + allow_endpoint_without_protocol?: boolean; + /** @description endpoint 允许的 URL protocol,取值与浏览器 `URL.protocol` 一致,例如 `https:`。 */ + allowed_endpoint_protocols?: string[]; /** @description 可选 help 文案 key。 */ help_key?: string | null; /** @description endpoint 协议不合法时的前端文案 key。 */ diff --git a/src/storage/connector_descriptor.rs b/src/storage/connector_descriptor.rs index d6ce7bcce..30af104d9 100644 --- a/src/storage/connector_descriptor.rs +++ b/src/storage/connector_descriptor.rs @@ -360,6 +360,12 @@ pub struct StorageConnectorFieldDescriptor { /// endpoint 协议不合法时的前端文案 key。 #[serde(skip_serializing_if = "Option::is_none")] pub invalid_protocol_message_key: Option, + /// endpoint 允许的 URL protocol,取值与浏览器 `URL.protocol` 一致,例如 `https:`。 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub allowed_endpoint_protocols: Vec, + /// true 表示 endpoint 可以省略 URL scheme,由 connector 在后端补齐或解释。 + #[serde(default)] + pub allow_endpoint_without_protocol: bool, /// true 表示该字段失焦时前端可以安全 trim。 #[serde(default)] pub trim_on_blur: bool, @@ -501,6 +507,8 @@ pub(crate) fn object_storage_connector_descriptor( help_key: Some(input.fields.endpoint_help_key), required_message_key: None, invalid_protocol_message_key: Some(input.fields.endpoint_protocol_error_key), + allowed_endpoint_protocols: vec!["http:", "https:"], + allow_endpoint_without_protocol: false, trim_on_blur: false, visible_when_driver_types: Vec::new(), }), @@ -515,6 +523,8 @@ pub(crate) fn object_storage_connector_descriptor( help_key: None, required_message_key: Some(input.fields.bucket_required_message_key), invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, trim_on_blur: false, visible_when_driver_types: Vec::new(), }), @@ -529,6 +539,8 @@ pub(crate) fn object_storage_connector_descriptor( help_key: None, required_message_key: None, invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, trim_on_blur: input.fields.access_key_trim_on_blur, visible_when_driver_types: Vec::new(), }), @@ -543,6 +555,8 @@ pub(crate) fn object_storage_connector_descriptor( help_key: None, required_message_key: None, invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, trim_on_blur: false, visible_when_driver_types: Vec::new(), }), @@ -583,6 +597,8 @@ pub(crate) fn object_storage_connector_descriptor( help_key: Some("s3_path_style_desc"), required_message_key: None, invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, trim_on_blur: false, visible_when_driver_types: vec![DriverType::S3], }, @@ -767,6 +783,8 @@ pub(crate) fn storage_connector_field( help_key: None, required_message_key: None, invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, trim_on_blur: false, visible_when_driver_types: Vec::new(), }) @@ -783,6 +801,8 @@ pub(crate) struct StorageConnectorFieldDisplayInput<'a> { pub(crate) help_key: Option<&'a str>, pub(crate) required_message_key: Option<&'a str>, pub(crate) invalid_protocol_message_key: Option<&'a str>, + pub(crate) allowed_endpoint_protocols: Vec<&'a str>, + pub(crate) allow_endpoint_without_protocol: bool, pub(crate) trim_on_blur: bool, pub(crate) visible_when_driver_types: Vec, } @@ -804,6 +824,12 @@ pub(crate) fn storage_connector_field_with_display( help_key: input.help_key.map(ToOwned::to_owned), required_message_key: input.required_message_key.map(ToOwned::to_owned), invalid_protocol_message_key: input.invalid_protocol_message_key.map(ToOwned::to_owned), + allowed_endpoint_protocols: input + .allowed_endpoint_protocols + .into_iter() + .map(ToOwned::to_owned) + .collect(), + allow_endpoint_without_protocol: input.allow_endpoint_without_protocol, trim_on_blur: input.trim_on_blur, required: semantics.required, secret: semantics.secret, diff --git a/src/storage/connectors/sftp.rs b/src/storage/connectors/sftp.rs index bbddddfd1..af1e2a4b4 100644 --- a/src/storage/connectors/sftp.rs +++ b/src/storage/connectors/sftp.rs @@ -75,23 +75,43 @@ impl StorageConnectorDescriptorProvider for SftpConnector { help_key: Some("sftp_endpoint_hint"), required_message_key: None, invalid_protocol_message_key: Some("sftp_endpoint_protocol_required_error"), + allowed_endpoint_protocols: vec!["sftp:"], + allow_endpoint_without_protocol: true, trim_on_blur: true, visible_when_driver_types: Vec::new(), }), - storage_connector_field( - "access_key", - StorageConnectorFieldScope::Connection, - StorageConnectorFieldKind::Text, - true, - false, - ), - storage_connector_field( - "secret_key", - StorageConnectorFieldScope::Connection, - StorageConnectorFieldKind::Secret, - true, - true, - ), + storage_connector_field_with_display(StorageConnectorFieldDisplayInput { + name: "access_key", + scope: StorageConnectorFieldScope::Connection, + kind: StorageConnectorFieldKind::Text, + required: true, + secret: false, + label_key: "sftp_username", + placeholder: None, + help_key: None, + required_message_key: None, + invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, + trim_on_blur: true, + visible_when_driver_types: Vec::new(), + }), + storage_connector_field_with_display(StorageConnectorFieldDisplayInput { + name: "secret_key", + scope: StorageConnectorFieldScope::Connection, + kind: StorageConnectorFieldKind::Secret, + required: true, + secret: true, + label_key: "sftp_password", + placeholder: None, + help_key: None, + required_message_key: None, + invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, + trim_on_blur: false, + visible_when_driver_types: Vec::new(), + }), storage_connector_field( "base_path", StorageConnectorFieldScope::Connection, diff --git a/src/storage/connectors/tests.rs b/src/storage/connectors/tests.rs index cd354ce67..d76a81eb1 100644 --- a/src/storage/connectors/tests.rs +++ b/src/storage/connectors/tests.rs @@ -474,8 +474,14 @@ fn sftp_connection_field_display_metadata_is_connector_owned() { .as_deref(), Some("sftp_endpoint_protocol_required_error") ); - assert_eq!(field(&sftp, "access_key").label_key, "access_key"); - assert_eq!(field(&sftp, "secret_key").label_key, "secret_key"); + assert_eq!( + field(&sftp, "endpoint").allowed_endpoint_protocols, + vec!["sftp:"] + ); + assert!(field(&sftp, "endpoint").allow_endpoint_without_protocol); + assert_eq!(field(&sftp, "access_key").label_key, "sftp_username"); + assert!(field(&sftp, "access_key").trim_on_blur); + assert_eq!(field(&sftp, "secret_key").label_key, "sftp_password"); assert!(sftp.fields.iter().all(|field| field.name != "bucket")); assert!(!sftp.upload_workflows.presigned_upload); assert!(!sftp.upload_workflows.object_multipart_upload); diff --git a/src/storage/drivers/sftp.rs b/src/storage/drivers/sftp.rs index 200b677b0..88144c010 100644 --- a/src/storage/drivers/sftp.rs +++ b/src/storage/drivers/sftp.rs @@ -84,8 +84,8 @@ impl SftpDriver { base_path: &str, ) -> Result<()> { parse_sftp_endpoint(endpoint)?; - validate_connection_secret(username, "access_key")?; - validate_connection_secret(password, "secret_key")?; + validate_connection_secret(username, "username")?; + validate_connection_secret(password, "password")?; normalize_remote_base_path(base_path)?; Ok(()) } diff --git a/tests/test_sftp.rs b/tests/test_sftp.rs new file mode 100644 index 000000000..8f8de8b7b --- /dev/null +++ b/tests/test_sftp.rs @@ -0,0 +1,179 @@ +//! SFTP storage driver integration test using testcontainers. + +use std::time::Duration; + +use aster_drive::storage::drivers::sftp::SftpDriver; +use aster_drive::storage::{StorageDriver, StorageErrorKind, StreamUploadDriver}; +use testcontainers::{GenericImage, ImageExt, core::IntoContainerPort, runners::AsyncRunner}; +use tokio::io::AsyncReadExt as _; + +const SFTP_IMAGE: &str = "atmoz/sftp"; +const SFTP_TAG: &str = "alpine"; +const SFTP_PORT: u16 = 22; +const SFTP_USERNAME: &str = "aster"; +const SFTP_PASSWORD: &str = "asterpass"; + +fn sftp_policy(endpoint: &str, base_path: &str) -> aster_drive::entities::storage_policy::Model { + use chrono::Utc; + + aster_drive::entities::storage_policy::Model { + id: 997, + name: "Test SFTP".to_string(), + driver_type: aster_drive::types::DriverType::Sftp, + endpoint: endpoint.to_string(), + bucket: String::new(), + access_key: SFTP_USERNAME.to_string(), + secret_key: SFTP_PASSWORD.to_string(), + base_path: base_path.to_string(), + remote_node_id: None, + remote_storage_target_key: None, + max_file_size: 0, + allowed_types: aster_drive::types::StoredStoragePolicyAllowedTypes::empty(), + options: aster_drive::types::StoredStoragePolicyOptions::empty(), + is_default: false, + chunk_size: 1024 * 1024, + created_at: Utc::now(), + updated_at: Utc::now(), + } +} + +async fn wait_for_sftp(driver: &SftpDriver) { + let mut last_error = None; + let ready = tokio::time::timeout(Duration::from_secs(45), async { + loop { + match tokio::time::timeout( + Duration::from_secs(5), + driver.put("readiness/probe.txt", b"ready"), + ) + .await + { + Ok(Ok(_)) => { + let _ = driver.delete("readiness/probe.txt").await; + break; + } + Ok(Err(error)) => last_error = Some(error.to_string()), + Err(_) => last_error = Some("readiness upload attempt timed out".to_string()), + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await; + + if ready.is_err() { + panic!( + "timed out waiting for SFTP test server: {}", + last_error.unwrap_or_else(|| "unknown error".to_string()) + ); + } +} + +#[tokio::test] +async fn test_sftp_driver_upload_download_round_trip() { + let container = GenericImage::new(SFTP_IMAGE, SFTP_TAG) + .with_exposed_port(IntoContainerPort::tcp(SFTP_PORT)) + .with_cmd(vec![format!("{SFTP_USERNAME}:{SFTP_PASSWORD}:::upload")]) + .start() + .await + .expect("failed to start sftp container"); + + let port = container + .get_host_port_ipv4(IntoContainerPort::tcp(SFTP_PORT)) + .await + .expect("resolve mapped sftp port"); + let endpoint = format!("sftp://127.0.0.1:{port}"); + let base_path = format!("/upload/asterdrive-itest-{}", uuid::Uuid::new_v4()); + let driver = SftpDriver::new(&sftp_policy(&endpoint, &base_path)).expect("create SftpDriver"); + wait_for_sftp(&driver).await; + + let data = b"hello sftp world"; + driver.put("docs/hello.txt", data).await.unwrap(); + assert!(driver.exists("docs/hello.txt").await.unwrap()); + assert!(!driver.exists("docs/missing.txt").await.unwrap()); + assert_eq!(driver.get("docs/hello.txt").await.unwrap(), data); + + let missing_meta = driver + .metadata("docs/missing.txt") + .await + .expect_err("missing sftp object metadata should fail"); + assert_eq!( + missing_meta.storage_error_kind(), + Some(StorageErrorKind::NotFound) + ); + + let meta = driver.metadata("docs/hello.txt").await.unwrap(); + assert_eq!(meta.size, u64::try_from(data.len()).unwrap()); + + let unicode_path = "docs/space dir/中文+plus.txt"; + driver.put(unicode_path, b"encoded path").await.unwrap(); + assert_eq!(driver.get(unicode_path).await.unwrap(), b"encoded path"); + + let mut full_stream = driver.get_stream("docs/hello.txt").await.unwrap(); + let mut full_body = Vec::new(); + full_stream.read_to_end(&mut full_body).await.unwrap(); + assert_eq!(full_body, data); + + let mut empty_range = driver + .get_range("docs/hello.txt", 0, Some(0)) + .await + .unwrap(); + let mut empty_range_body = Vec::new(); + empty_range + .read_to_end(&mut empty_range_body) + .await + .unwrap(); + assert!(empty_range_body.is_empty()); + + let mut range = driver + .get_range("docs/hello.txt", 6, Some(4)) + .await + .unwrap(); + let mut range_body = Vec::new(); + range.read_to_end(&mut range_body).await.unwrap(); + assert_eq!(range_body, b"sftp"); + + let mut tail = driver.get_range("docs/hello.txt", 11, None).await.unwrap(); + let mut tail_body = Vec::new(); + tail.read_to_end(&mut tail_body).await.unwrap(); + assert_eq!(tail_body, b"world"); + + driver + .copy_object("docs/hello.txt", "docs/copied.txt") + .await + .unwrap(); + assert_eq!(driver.get("docs/copied.txt").await.unwrap(), data); + + driver + .put_reader( + "stream/reader.bin", + Box::new(std::io::Cursor::new(b"stream upload".to_vec())), + 13, + ) + .await + .unwrap(); + assert_eq!( + driver.get("stream/reader.bin").await.unwrap(), + b"stream upload" + ); + + let temp_dir = std::env::temp_dir().join(format!("asterdrive-sftp-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&temp_dir).expect("create temp dir"); + let local_upload = temp_dir.join("upload.bin"); + std::fs::write(&local_upload, b"file upload").expect("write local upload"); + driver + .put_file( + "stream/from-file.bin", + local_upload + .to_str() + .expect("temp upload path should be valid utf-8"), + ) + .await + .unwrap(); + assert_eq!( + driver.get("stream/from-file.bin").await.unwrap(), + b"file upload" + ); + let _ = std::fs::remove_dir_all(&temp_dir); + + driver.delete("docs/hello.txt").await.unwrap(); + assert!(!driver.exists("docs/hello.txt").await.unwrap()); +} From a11b41d5011ea2a7f509e520345e6699a2b975b7 Mon Sep 17 00:00:00 2001 From: AptS-1547 Date: Mon, 6 Jul 2026 21:44:25 +0800 Subject: [PATCH 3/5] fix(storage): require pinned SFTP host keys --- .../StoragePolicyObjectStorageFields.tsx | 67 ++++++++ .../connectionNormalization.ts | 32 +++- .../admin/storage-policy-dialog/formTypes.ts | 19 +++ .../policyFormTransition.ts | 25 +++ .../storagePolicyOptions.ts | 16 ++ .../admin/storagePolicyDialogShared.test.ts | 78 +++++++++ .../src/i18n/locales/en/admin/policies.json | 2 + .../src/i18n/locales/zh/admin/policies.json | 2 + .../pages/admin/AdminPoliciesPage.test.tsx | 19 ++- frontend-panel/src/services/api.generated.ts | 1 + src/storage/connectors/common.rs | 12 ++ src/storage/connectors/mod.rs | 3 +- src/storage/connectors/sftp.rs | 22 ++- src/storage/connectors/tests.rs | 31 ++++ src/storage/drivers/sftp.rs | 150 ++++++++++++++++-- src/storage/error.rs | 1 + src/types/storage_policy.rs | 12 ++ tests/test_sftp.rs | 74 ++++++++- 18 files changed, 549 insertions(+), 17 deletions(-) diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyObjectStorageFields.tsx b/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyObjectStorageFields.tsx index 4ce3b49e7..f2c46f2f6 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyObjectStorageFields.tsx +++ b/frontend-panel/src/components/admin/storage-policy-dialog/StoragePolicyObjectStorageFields.tsx @@ -38,6 +38,10 @@ export function ObjectStorageConnectionFields({ pathStyleField, form.driver_type, ); + const policyOptionTextFields = policyOptionTextFieldDescriptors( + storageDriverDescriptor, + form.driver_type, + ); const hasBucketField = bucketField != null; return ( @@ -138,10 +142,59 @@ export function ObjectStorageConnectionFields({ /> + {policyOptionTextFields.map((field) => ( + + ))} ); } +function PolicyOptionTextField({ + field, + form, + onFieldChange, + t, +}: SharedFieldProps & { + field: StorageConnectorFieldDescriptor; +}) { + const optionValues = form.policy_option_values ?? {}; + const setPolicyOptionValue = (value: string) => { + onFieldChange("policy_option_values", { + ...optionValues, + [field.name]: value, + }); + }; + + return ( +
+ + setPolicyOptionValue(event.target.value)} + onBlur={(event) => { + if (field.trim_on_blur === true) { + setPolicyOptionValue(event.target.value.trim()); + } + }} + autoComplete="off" + className={ADMIN_CONTROL_HEIGHT_CLASS} + placeholder={field.placeholder ?? undefined} + /> + {field.help_key ? ( +

{t(field.help_key)}

+ ) : null} +
+ ); +} + function S3PathStyleField({ field, form, @@ -176,6 +229,20 @@ function fieldDescriptor( return descriptor?.fields.find((field) => field.name === name) ?? null; } +function policyOptionTextFieldDescriptors( + descriptor: StorageConnectorDescriptor | null | undefined, + driverType: string, +) { + return ( + descriptor?.fields.filter( + (field): field is StorageConnectorFieldDescriptor => + field.scope === "policy_options" && + (field.kind === "text" || field.kind === "secret") && + isFieldVisibleForDriver(field, driverType), + ) ?? [] + ); +} + function fieldLabelKey( field: StorageConnectorFieldDescriptor | null, fallback: string, diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts b/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts index 83c14106a..720e3fc97 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts +++ b/frontend-panel/src/components/admin/storage-policy-dialog/connectionNormalization.ts @@ -183,6 +183,31 @@ function getComparableOneDrivePolicyOptions( return buildPolicyOptions(getPolicyForm(policy)); } +function hasStaticSecretPolicyOptionConnectionChanges( + form: PolicyFormData, + editingPolicy: StoragePolicy, + descriptor?: StorageConnectorDescriptor | null, +) { + const optionFields = + descriptor?.fields.filter( + (field) => + field.scope === "policy_options" && + (field.kind === "text" || field.kind === "secret"), + ) ?? []; + if (optionFields.length === 0) { + return false; + } + + const policyOptions = editingPolicy.options as Record; + return optionFields.some((field) => { + const formValue = String( + form.policy_option_values?.[field.name] ?? "", + ).trim(); + const savedValue = String(policyOptions[field.name] ?? "").trim(); + return formValue !== savedValue; + }); +} + export function hasConnectionFieldChanges( form: PolicyFormData, editingPolicy: StoragePolicy | null, @@ -200,7 +225,12 @@ export function hasConnectionFieldChanges( normalizedForm.bucket !== editingPolicy.bucket || normalizedForm.base_path !== editingPolicy.base_path || normalizedForm.access_key !== "" || - normalizedForm.secret_key !== "" + normalizedForm.secret_key !== "" || + hasStaticSecretPolicyOptionConnectionChanges( + normalizedForm, + editingPolicy, + descriptor, + ) ); } diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/formTypes.ts b/frontend-panel/src/components/admin/storage-policy-dialog/formTypes.ts index cab197b11..bc0a31238 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/formTypes.ts +++ b/frontend-panel/src/components/admin/storage-policy-dialog/formTypes.ts @@ -52,6 +52,7 @@ export interface PolicyFormData { onedrive_root_item_id: string; onedrive_site_id: string; onedrive_group_id: string; + policy_option_values?: Record; application_credentials: StorageApplicationCredentialForm; storage_native_processing_enabled: boolean; storage_native_media_metadata_enabled?: boolean; @@ -96,6 +97,7 @@ export function getPolicyForm(policy: StoragePolicy): PolicyFormData { onedrive_root_item_id: options.onedrive_root_item_id ?? "", onedrive_site_id: options.onedrive_site_id ?? "", onedrive_group_id: options.onedrive_group_id ?? "", + policy_option_values: scalarPolicyOptionValues(options), application_credentials: { microsoft_graph: { cloud: options.onedrive_cloud ?? "global", @@ -151,6 +153,7 @@ export const emptyForm: PolicyFormData = { onedrive_root_item_id: "", onedrive_site_id: "", onedrive_group_id: "", + policy_option_values: {}, application_credentials: { microsoft_graph: { cloud: "global", @@ -166,3 +169,19 @@ export const emptyForm: PolicyFormData = { thumbnail_extensions: [], media_metadata_extensions: [], }; + +function scalarPolicyOptionValues( + options: StoragePolicyOptions, +): Record { + const values: Record = {}; + for (const [key, value] of Object.entries(options)) { + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + values[key] = String(value); + } + } + return values; +} diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/policyFormTransition.ts b/frontend-panel/src/components/admin/storage-policy-dialog/policyFormTransition.ts index 0c18231c4..a78464289 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/policyFormTransition.ts +++ b/frontend-panel/src/components/admin/storage-policy-dialog/policyFormTransition.ts @@ -56,11 +56,16 @@ export function applyPolicyDriverTransition( form; const nextSupportsStorageNativeProcessing = supportsStorageNativeProcessing(nextDriverDescriptor); + const nextPolicyOptionValues = policyOptionValuesForDescriptor( + form.policy_option_values ?? {}, + nextDriverDescriptor, + ); if (supportsObjectStorageConnection(nextDriverDescriptor)) { return { ...formWithoutS3PathStyle, driver_type: driverType, + policy_option_values: nextPolicyOptionValues, remote_node_id: "", remote_storage_target_key: "", storage_native_processing_enabled: nextSupportsStorageNativeProcessing @@ -92,6 +97,7 @@ export function applyPolicyDriverTransition( bucket: "", access_key: "", secret_key: "", + policy_option_values: nextPolicyOptionValues, content_dedup: false, storage_native_processing_enabled: false, thumbnail_processor: null, @@ -112,6 +118,7 @@ export function applyPolicyDriverTransition( bucket: "", access_key: "", secret_key: "", + policy_option_values: nextPolicyOptionValues, remote_node_id: "", remote_storage_target_key: "", content_dedup: false, @@ -150,6 +157,7 @@ export function applyPolicyDriverTransition( bucket: "", access_key: "", secret_key: "", + policy_option_values: nextPolicyOptionValues, remote_node_id: "", remote_storage_target_key: "", storage_native_processing_enabled: false, @@ -163,3 +171,20 @@ export function applyPolicyDriverTransition( object_storage_download_strategy: "relay_stream", }; } + +function policyOptionValuesForDescriptor( + values: Record, + descriptor: StorageConnectorDescriptor | null | undefined, +) { + const nextValues: Record = {}; + for (const field of descriptor?.fields ?? []) { + if ( + field.scope === "policy_options" && + (field.kind === "text" || field.kind === "secret") && + Object.hasOwn(values, field.name) + ) { + nextValues[field.name] = values[field.name]; + } + } + return nextValues; +} diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts b/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts index 7678c2760..8522e7a40 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts +++ b/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts @@ -25,6 +25,7 @@ export interface StoragePolicyOptionsForm { onedrive_root_item_id: string; onedrive_site_id: string; onedrive_group_id: string; + policy_option_values?: Record; storage_native_processing_enabled: boolean; storage_native_media_metadata_enabled?: boolean; thumbnail_processor: StoragePolicyOptions["thumbnail_processor"]; @@ -132,6 +133,21 @@ function buildDescriptorPolicyOptions( Object.assign(options, buildOneDrivePolicyOptions(form)); } + const optionRecord = options as Record; + for (const field of descriptor.fields) { + if ( + field.scope !== "policy_options" || + (field.kind !== "text" && field.kind !== "secret") || + Object.hasOwn(optionRecord, field.name) + ) { + continue; + } + const value = form.policy_option_values?.[field.name]?.trim() ?? ""; + if (value) { + optionRecord[field.name] = value; + } + } + return options; } diff --git a/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts b/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts index 7cf47da98..050edeedd 100644 --- a/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts +++ b/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts @@ -387,6 +387,11 @@ describe("storage policy dialog helper modules", () => { onedrive_root_item_id: "", onedrive_site_id: "", onedrive_group_id: "", + policy_option_values: { + content_dedup: "true", + storage_native_processing_enabled: "true", + thumbnail_processor: "storage_native", + }, application_credentials: { microsoft_graph: { cloud: "global", @@ -1361,6 +1366,33 @@ describe("storage policy dialog helper modules", () => { object_storage_upload_strategy: "presigned", object_storage_download_strategy: "presigned", }); + + const sftpDescriptor = { + fields: [ + { + name: "sftp_host_key_fingerprint", + kind: "text", + scope: "policy_options", + }, + ], + } as never; + expect( + buildCreatePolicyPayload( + { + ...remoteForm, + driver_type: "sftp" as const, + endpoint: "sftp.example.com:22", + access_key: "aster", + secret_key: "secret", + policy_option_values: { + sftp_host_key_fingerprint: " SHA256:trusted ", + }, + }, + sftpDescriptor, + ).options, + ).toEqual({ + sftp_host_key_fingerprint: "SHA256:trusted", + }); }); it("uses application credential descriptor fields for Microsoft Graph app config", () => { @@ -1669,6 +1701,52 @@ describe("storage policy dialog helper modules", () => { ), ).toBe(true); + const sftpPolicy = { + ...localPolicy, + id: 16, + name: "SFTP", + driver_type: "sftp", + endpoint: "sftp.example.com:22", + access_key: "aster", + secret_key: "", + base_path: "uploads", + options: { + sftp_host_key_fingerprint: "SHA256:old-key", + }, + } as StoragePolicy; + const sftpForm = getPolicyForm(sftpPolicy); + const sftpDescriptor = { + credential_mode: "static_secret", + fields: [ + { name: "endpoint", scope: "connection" }, + { name: "access_key", scope: "connection" }, + { name: "secret_key", scope: "connection" }, + { + name: "sftp_host_key_fingerprint", + kind: "text", + scope: "policy_options", + }, + ], + upload_workflows: {}, + } as never; + + expect( + hasConnectionFieldChanges(sftpForm, sftpPolicy, sftpDescriptor), + ).toBe(false); + expect( + hasConnectionFieldChanges( + { + ...sftpForm, + policy_option_values: { + ...sftpForm.policy_option_values, + sftp_host_key_fingerprint: " SHA256:new-key ", + }, + }, + sftpPolicy, + sftpDescriptor, + ), + ).toBe(true); + const oneDrivePolicy = { ...localPolicy, id: 14, diff --git a/frontend-panel/src/i18n/locales/en/admin/policies.json b/frontend-panel/src/i18n/locales/en/admin/policies.json index ded7abcec..536bb22f0 100644 --- a/frontend-panel/src/i18n/locales/en/admin/policies.json +++ b/frontend-panel/src/i18n/locales/en/admin/policies.json @@ -89,6 +89,8 @@ "cos_endpoint_hint": "For Tencent COS, use the bucket domain such as https://.cos..myqcloud.com; enter the bucket separately, for example .", "azure_blob_endpoint_hint": "Enter the Azure Blob service endpoint, for example https://.blob.core.windows.net. Use the bucket field for the container name.", "sftp_endpoint_hint": "Enter an SFTP server endpoint such as sftp://example.com:22. Put the remote root directory in base path.", + "sftp_host_key_fingerprint": "SSH Host Key Fingerprint", + "sftp_host_key_fingerprint_hint": "Run the connection test once. If the host key is unknown, confirm the reported SHA256 fingerprint here before saving.", "policy_wizard_endpoint_required": "Endpoint is required.", "s3_endpoint_protocol_required_error": "S3 endpoint must include http:// or https://.", "azure_blob_endpoint_protocol_required_error": "Azure Blob endpoint must include http:// or https://.", diff --git a/frontend-panel/src/i18n/locales/zh/admin/policies.json b/frontend-panel/src/i18n/locales/zh/admin/policies.json index dfc90c154..340d34549 100644 --- a/frontend-panel/src/i18n/locales/zh/admin/policies.json +++ b/frontend-panel/src/i18n/locales/zh/admin/policies.json @@ -89,6 +89,8 @@ "cos_endpoint_hint": "腾讯云 COS 请填写 bucket 域名,例如 https://.cos..myqcloud.com;bucket 请单独填写,例如 。", "azure_blob_endpoint_hint": "填写 Azure Blob 服务 endpoint,例如 https://.blob.core.windows.net;bucket 字段填写容器名称。", "sftp_endpoint_hint": "填写 SFTP 服务器 endpoint,例如 sftp://example.com:22;远程根目录请填写在基础路径里。", + "sftp_host_key_fingerprint": "SSH 主机密钥指纹", + "sftp_host_key_fingerprint_hint": "先运行一次连接测试。若主机密钥未知,请确认错误信息里的 SHA256 指纹并填到这里再保存。", "policy_wizard_endpoint_required": "必须填写端点。", "s3_endpoint_protocol_required_error": "S3 endpoint 必须包含 http:// 或 https://。", "azure_blob_endpoint_protocol_required_error": "Azure Blob endpoint 必须包含 http:// 或 https://。", diff --git a/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx b/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx index 69590917c..564b452d6 100644 --- a/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx +++ b/frontend-panel/src/pages/admin/AdminPoliciesPage.test.tsx @@ -84,6 +84,8 @@ const mockTranslate = vi.hoisted( return "SSH Username"; case "sftp_password": return "SSH Password"; + case "sftp_host_key_fingerprint": + return "SSH Host Key Fingerprint"; case "access_key": return "Access Key"; case "secret_key": @@ -1043,6 +1045,12 @@ function createStorageDriverDescriptors() { required: true, }), fieldDescriptor("base_path", "connection", "text"), + fieldDescriptor("sftp_host_key_fingerprint", "policy_options", "text", { + help_key: "sftp_host_key_fingerprint_hint", + label_key: "sftp_host_key_fingerprint", + placeholder: "SHA256:...", + trim_on_blur: true, + }), ], }), createStorageDriverDescriptor("tencent_cos", { @@ -3077,6 +3085,9 @@ describe("AdminPoliciesPage", () => { fireEvent.change(screen.getByLabelText("SSH Password"), { target: { value: "SFTPSECRET" }, }); + fireEvent.change(screen.getByLabelText("SSH Host Key Fingerprint"), { + target: { value: " SHA256:trusted-host-key " }, + }); fireEvent.click(screen.getByRole("button", { name: /test_connection/i })); @@ -3087,7 +3098,9 @@ describe("AdminPoliciesPage", () => { bucket: undefined, driver_type: "sftp", endpoint: "sftp.example.com:2222", - options: {}, + options: { + sftp_host_key_fingerprint: "SHA256:trusted-host-key", + }, remote_node_id: undefined, secret_key: "SFTPSECRET", }); @@ -3109,7 +3122,9 @@ describe("AdminPoliciesPage", () => { is_default: false, max_file_size: undefined, name: "SFTP Archive", - options: {}, + options: { + sftp_host_key_fingerprint: "SHA256:trusted-host-key", + }, remote_node_id: undefined, secret_key: "SFTPSECRET", }); diff --git a/frontend-panel/src/services/api.generated.ts b/frontend-panel/src/services/api.generated.ts index 5bcbb8637..44a2f6372 100644 --- a/frontend-panel/src/services/api.generated.ts +++ b/frontend-panel/src/services/api.generated.ts @@ -7722,6 +7722,7 @@ export interface components { s3_path_style?: boolean | null; /** Format: int64 */ s3_read_timeout_secs?: number | null; + sftp_host_key_fingerprint?: string | null; storage_native_media_metadata_enabled?: boolean | null; storage_native_processing_enabled?: boolean | null; thumbnail_extensions?: string[]; diff --git a/src/storage/connectors/common.rs b/src/storage/connectors/common.rs index 9002ca8e7..9a926b434 100644 --- a/src/storage/connectors/common.rs +++ b/src/storage/connectors/common.rs @@ -194,6 +194,18 @@ pub(super) fn ensure_onedrive_options_absent( Ok(()) } +pub(super) fn ensure_sftp_options_absent( + options: &crate::types::StoragePolicyOptions, +) -> Result<()> { + if options.sftp_host_key_fingerprint.is_some() { + return Err(validation_error_with_code( + ApiErrorCode::BadRequest, + "SFTP host key options are only valid for SFTP storage policies", + )); + } + Ok(()) +} + pub(super) fn validate_onedrive_options( options: &crate::types::StoragePolicyOptions, ) -> Result<()> { diff --git a/src/storage/connectors/mod.rs b/src/storage/connectors/mod.rs index a3724bcad..3ed0d6eb5 100644 --- a/src/storage/connectors/mod.rs +++ b/src/storage/connectors/mod.rs @@ -129,7 +129,8 @@ trait StorageConnector: StorageConnectorDescriptorProvider + Send + Sync + Sized Self::storage_connector_descriptor(), options, )?; - common::ensure_onedrive_options_absent(options) + common::ensure_onedrive_options_absent(options)?; + common::ensure_sftp_options_absent(options) } /// 持久化 provider application credential/config。 diff --git a/src/storage/connectors/sftp.rs b/src/storage/connectors/sftp.rs index af1e2a4b4..248bbb177 100644 --- a/src/storage/connectors/sftp.rs +++ b/src/storage/connectors/sftp.rs @@ -119,6 +119,22 @@ impl StorageConnectorDescriptorProvider for SftpConnector { false, false, ), + storage_connector_field_with_display(StorageConnectorFieldDisplayInput { + name: "sftp_host_key_fingerprint", + scope: StorageConnectorFieldScope::PolicyOptions, + kind: StorageConnectorFieldKind::Text, + required: false, + secret: false, + label_key: "sftp_host_key_fingerprint", + placeholder: Some("SHA256:..."), + help_key: Some("sftp_host_key_fingerprint_hint"), + required_message_key: None, + invalid_protocol_message_key: None, + allowed_endpoint_protocols: Vec::new(), + allow_endpoint_without_protocol: false, + trim_on_blur: true, + visible_when_driver_types: Vec::new(), + }), ], actions: vec![ draft_connection_test_action_descriptor(), @@ -161,7 +177,11 @@ impl StorageConnector for SftpConnector { options: &crate::types::StoragePolicyOptions, ) -> Result<()> { let _ = (db, remote_node_id); - ensure_onedrive_options_absent(options) + ensure_onedrive_options_absent(options)?; + if let Some(fingerprint) = options.sftp_host_key_fingerprint.as_deref() { + SftpDriver::validate_host_key_fingerprint(fingerprint)?; + } + Ok(()) } async fn build_draft_driver( diff --git a/src/storage/connectors/tests.rs b/src/storage/connectors/tests.rs index d76a81eb1..ca23fead6 100644 --- a/src/storage/connectors/tests.rs +++ b/src/storage/connectors/tests.rs @@ -482,6 +482,20 @@ fn sftp_connection_field_display_metadata_is_connector_owned() { assert_eq!(field(&sftp, "access_key").label_key, "sftp_username"); assert!(field(&sftp, "access_key").trim_on_blur); assert_eq!(field(&sftp, "secret_key").label_key, "sftp_password"); + assert_eq!( + field(&sftp, "sftp_host_key_fingerprint").scope, + StorageConnectorFieldScope::PolicyOptions + ); + assert_eq!( + field(&sftp, "sftp_host_key_fingerprint").label_key, + "sftp_host_key_fingerprint" + ); + assert_eq!( + field(&sftp, "sftp_host_key_fingerprint") + .help_key + .as_deref(), + Some("sftp_host_key_fingerprint_hint") + ); assert!(sftp.fields.iter().all(|field| field.name != "bucket")); assert!(!sftp.upload_workflows.presigned_upload); assert!(!sftp.upload_workflows.object_multipart_upload); @@ -825,6 +839,23 @@ fn onedrive_options_are_rejected_for_non_onedrive_connector() { ); } +#[test] +fn sftp_host_key_options_are_rejected_for_non_sftp_connector() { + let options = StoragePolicyOptions { + sftp_host_key_fingerprint: Some("SHA256:abc123".to_string()), + ..Default::default() + }; + + let error = common::ensure_sftp_options_absent(&options).unwrap_err(); + + assert_eq!(error.api_error_code(), ApiErrorCode::BadRequest); + assert!( + error + .to_string() + .contains("SFTP host key options are only valid for SFTP") + ); +} + #[test] fn onedrive_connector_accepts_automatic_default_drive() { let options = StoragePolicyOptions { diff --git a/src/storage/drivers/sftp.rs b/src/storage/drivers/sftp.rs index 88144c010..744ed8db2 100644 --- a/src/storage/drivers/sftp.rs +++ b/src/storage/drivers/sftp.rs @@ -2,11 +2,12 @@ use async_trait::async_trait; use russh::client::{self, Handler}; +use russh::keys::{HashAlg, PublicKey}; use russh_sftp::client::{Config as SftpClientConfig, SftpSession, error::Error as SftpError}; use russh_sftp::protocol::StatusCode; use std::io::SeekFrom; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; use std::time::Duration; use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, ReadBuf}; @@ -15,6 +16,7 @@ use crate::entities::storage_policy; use crate::errors::{AsterError, Result}; use crate::storage::error::{StorageErrorKind, storage_driver_error}; use crate::storage::{BlobMetadata, StorageDriver, StreamUploadDriver}; +use crate::types::parse_storage_policy_options; const DEFAULT_SFTP_PORT: u16 = 22; const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); @@ -32,18 +34,46 @@ pub struct SftpDriver { username: String, password: String, base_path: String, + host_key_fingerprint: Option, } -struct TrustServerKeyClient; +#[derive(Debug, Clone, PartialEq, Eq)] +enum HostKeyRejection { + MissingPin { actual: String }, + Mismatch { expected: String, actual: String }, +} + +#[derive(Clone)] +struct TrustServerKeyClient { + expected_fingerprint: Option, + rejection: Arc>>, +} impl Handler for TrustServerKeyClient { type Error = russh::Error; async fn check_server_key( &mut self, - _server_public_key: &russh::keys::PublicKey, + server_public_key: &PublicKey, ) -> std::result::Result { - Ok(true) + let actual = host_key_fingerprint(server_public_key); + let Some(expected) = self.expected_fingerprint.as_deref() else { + record_host_key_rejection(&self.rejection, HostKeyRejection::MissingPin { actual }); + return Ok(false); + }; + + if host_key_fingerprint_matches(expected, &actual) { + return Ok(true); + } + + record_host_key_rejection( + &self.rejection, + HostKeyRejection::Mismatch { + expected: normalize_host_key_fingerprint(expected), + actual, + }, + ); + Ok(false) } } @@ -103,9 +133,22 @@ impl SftpDriver { username: policy.access_key.clone(), password: policy.secret_key.clone(), base_path: normalize_remote_base_path(&policy.base_path)?, + host_key_fingerprint: parse_storage_policy_options(policy.options.as_ref()) + .sftp_host_key_fingerprint, }) } + pub fn validate_host_key_fingerprint(fingerprint: &str) -> Result<()> { + let normalized = normalize_host_key_fingerprint(fingerprint); + if !is_valid_host_key_fingerprint(&normalized) { + return Err(storage_driver_error( + StorageErrorKind::Misconfigured, + "SFTP host key fingerprint must use the SHA256: format", + )); + } + Ok(()) + } + async fn connect(&self) -> Result { let mut config = russh::client::Config::default(); config.inactivity_timeout = Some(IO_TIMEOUT); @@ -113,13 +156,21 @@ impl SftpDriver { config.nodelay = true; let address = (self.endpoint.host.clone(), self.endpoint.port); + let host_key_rejection = Arc::new(Mutex::new(None)); + let client = TrustServerKeyClient { + expected_fingerprint: self.host_key_fingerprint.clone(), + rejection: Arc::clone(&host_key_rejection), + }; let mut ssh = timeout_io( "connect SFTP endpoint", CONNECT_TIMEOUT, - russh::client::connect(Arc::new(config), address, TrustServerKeyClient), + russh::client::connect(Arc::new(config), address, client), ) .await? - .map_err(|error| map_ssh_error("connect SFTP endpoint failed", error))?; + .map_err(|error| { + host_key_rejection_error(&self.endpoint, &host_key_rejection) + .unwrap_or_else(|| map_ssh_error("connect SFTP endpoint failed", error)) + })?; let auth = timeout_io( "SFTP authentication", @@ -354,6 +405,65 @@ impl StreamUploadDriver for SftpDriver { } } +fn host_key_fingerprint(public_key: &PublicKey) -> String { + public_key.fingerprint(HashAlg::Sha256).to_string() +} + +fn normalize_host_key_fingerprint(fingerprint: &str) -> String { + let trimmed = fingerprint.trim(); + trimmed + .strip_prefix("sha256:") + .map(|value| format!("SHA256:{value}")) + .unwrap_or_else(|| trimmed.to_string()) +} + +fn is_valid_host_key_fingerprint(fingerprint: &str) -> bool { + fingerprint + .strip_prefix("SHA256:") + .is_some_and(|value| !value.is_empty() && !value.chars().any(char::is_whitespace)) +} + +fn host_key_fingerprint_matches(expected: &str, actual: &str) -> bool { + normalize_host_key_fingerprint(expected) == normalize_host_key_fingerprint(actual) +} + +fn record_host_key_rejection( + rejection: &Arc>>, + value: HostKeyRejection, +) { + match rejection.lock() { + Ok(mut guard) => { + *guard = Some(value); + } + Err(error) => { + tracing::warn!("failed to record SFTP host key rejection: {error}"); + } + } +} + +fn host_key_rejection_error( + endpoint: &SftpEndpoint, + rejection: &Arc>>, +) -> Option { + let rejection = rejection.lock().ok()?.clone()?; + Some(match rejection { + HostKeyRejection::MissingPin { actual } => storage_driver_error( + StorageErrorKind::Precondition, + format!( + "SFTP host key is not trusted for {}:{}. Confirm fingerprint {actual} and save it as sftp_host_key_fingerprint before testing again", + endpoint.host, endpoint.port + ), + ), + HostKeyRejection::Mismatch { expected, actual } => storage_driver_error( + StorageErrorKind::Precondition, + format!( + "SFTP host key mismatch for {}:{}. Expected {expected}, got {actual}. Verify the server identity before updating sftp_host_key_fingerprint", + endpoint.host, endpoint.port + ), + ), + }) +} + async fn timeout_io(context: &'static str, duration: Duration, future: F) -> Result where F: std::future::Future, @@ -659,8 +769,9 @@ fn is_sftp_not_found(error: &SftpError) -> bool { #[cfg(test)] mod tests { use super::{ - classify_sftp_error, join_remote_path, normalize_remote_base_path, parse_sftp_endpoint, - sanitize_relative_storage_path, + classify_sftp_error, host_key_fingerprint_matches, is_valid_host_key_fingerprint, + join_remote_path, normalize_host_key_fingerprint, normalize_remote_base_path, + parse_sftp_endpoint, sanitize_relative_storage_path, }; use crate::storage::error::StorageErrorKind; use crate::storage::{StorageDriver, StreamUploadDriver}; @@ -769,11 +880,26 @@ mod tests { ); } + #[test] + fn validates_sftp_host_key_fingerprint_format() { + assert!(is_valid_host_key_fingerprint( + &normalize_host_key_fingerprint("SHA256:abc123+/=") + )); + assert!(host_key_fingerprint_matches( + "sha256:abc123+/=", + "SHA256:abc123+/=" + )); + assert!(!is_valid_host_key_fingerprint("MD5:aa:bb")); + assert!(!is_valid_host_key_fingerprint("SHA256:")); + assert!(!is_valid_host_key_fingerprint("SHA256:abc def")); + } + fn env_policy() -> Option { let endpoint = std::env::var("ASTER_SFTP_TEST_ENDPOINT").ok()?; let username = std::env::var("ASTER_SFTP_TEST_USERNAME").ok()?; let password = std::env::var("ASTER_SFTP_TEST_PASSWORD").ok()?; let base_path = std::env::var("ASTER_SFTP_TEST_BASE_PATH").ok()?; + let host_key_fingerprint = std::env::var("ASTER_SFTP_TEST_HOST_KEY_FINGERPRINT").ok()?; Some(crate::entities::storage_policy::Model { id: 1, name: "sftp acceptance".to_string(), @@ -787,7 +913,13 @@ mod tests { remote_storage_target_key: None, max_file_size: 0, allowed_types: StoredStoragePolicyAllowedTypes::empty(), - options: crate::types::StoredStoragePolicyOptions::empty(), + options: crate::types::serialize_storage_policy_options( + &crate::types::StoragePolicyOptions { + sftp_host_key_fingerprint: Some(host_key_fingerprint), + ..Default::default() + }, + ) + .expect("serialize SFTP host key options"), is_default: false, chunk_size: 1024, created_at: Utc::now(), diff --git a/src/storage/error.rs b/src/storage/error.rs index 820a788a5..33fce2c88 100644 --- a/src/storage/error.rs +++ b/src/storage/error.rs @@ -161,6 +161,7 @@ fn infer_storage_error_kind(message: &str) -> StorageErrorKind { "is disabled", "precondition failed", "master binding is disabled", + "host key", ], ) { return StorageErrorKind::Precondition; diff --git a/src/types/storage_policy.rs b/src/types/storage_policy.rs index a7f6942fe..18356705f 100644 --- a/src/types/storage_policy.rs +++ b/src/types/storage_policy.rs @@ -359,6 +359,8 @@ pub struct StoragePolicyOptions { pub onedrive_site_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub onedrive_group_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sftp_host_key_fingerprint: Option, } impl StoragePolicyOptions { @@ -421,6 +423,7 @@ impl StoragePolicyOptions { trim_empty_option_string(&mut self.onedrive_root_item_id); trim_empty_option_string(&mut self.onedrive_site_id); trim_empty_option_string(&mut self.onedrive_group_id); + trim_empty_option_string(&mut self.sftp_host_key_fingerprint); } pub fn normalized(mut self) -> Self { @@ -1116,6 +1119,15 @@ mod tests { }) .unwrap(); assert_eq!(json, r#"{"s3_operation_timeout_secs":600}"#); + + let json = String::from( + serialize_storage_policy_options(&StoragePolicyOptions { + sftp_host_key_fingerprint: Some(" SHA256:abc123 ".to_string()), + ..Default::default() + }) + .unwrap(), + ); + assert_eq!(json, r#"{"sftp_host_key_fingerprint":"SHA256:abc123"}"#); } #[test] diff --git a/tests/test_sftp.rs b/tests/test_sftp.rs index 8f8de8b7b..87e4a5763 100644 --- a/tests/test_sftp.rs +++ b/tests/test_sftp.rs @@ -13,9 +13,25 @@ const SFTP_PORT: u16 = 22; const SFTP_USERNAME: &str = "aster"; const SFTP_PASSWORD: &str = "asterpass"; -fn sftp_policy(endpoint: &str, base_path: &str) -> aster_drive::entities::storage_policy::Model { +fn sftp_policy( + endpoint: &str, + base_path: &str, + host_key_fingerprint: Option<&str>, +) -> aster_drive::entities::storage_policy::Model { use chrono::Utc; + let options = host_key_fingerprint + .map(|fingerprint| { + aster_drive::types::serialize_storage_policy_options( + &aster_drive::types::StoragePolicyOptions { + sftp_host_key_fingerprint: Some(fingerprint.to_string()), + ..Default::default() + }, + ) + .expect("serialize SFTP host key options") + }) + .unwrap_or_else(aster_drive::types::StoredStoragePolicyOptions::empty); + aster_drive::entities::storage_policy::Model { id: 997, name: "Test SFTP".to_string(), @@ -29,7 +45,7 @@ fn sftp_policy(endpoint: &str, base_path: &str) -> aster_drive::entities::storag remote_storage_target_key: None, max_file_size: 0, allowed_types: aster_drive::types::StoredStoragePolicyAllowedTypes::empty(), - options: aster_drive::types::StoredStoragePolicyOptions::empty(), + options, is_default: false, chunk_size: 1024 * 1024, created_at: Utc::now(), @@ -37,6 +53,47 @@ fn sftp_policy(endpoint: &str, base_path: &str) -> aster_drive::entities::storag } } +fn extract_sftp_host_key_fingerprint(error: &aster_drive::errors::AsterError) -> String { + let message = error.message(); + let marker = "Confirm fingerprint "; + let (_, rest) = message + .split_once(marker) + .expect("untrusted host key error should include fingerprint"); + rest.split_whitespace() + .next() + .expect("fingerprint should be present") + .to_string() +} + +async fn wait_for_sftp_host_key_fingerprint(driver: &SftpDriver) -> String { + let mut last_error = None; + let fingerprint = tokio::time::timeout(Duration::from_secs(45), async { + loop { + match tokio::time::timeout(Duration::from_secs(5), driver.exists("readiness/probe.txt")) + .await + { + Ok(Ok(_)) => last_error = Some("untrusted host key was accepted".to_string()), + Ok(Err(error)) + if error.storage_error_kind() == Some(StorageErrorKind::Precondition) => + { + break extract_sftp_host_key_fingerprint(&error); + } + Ok(Err(error)) => last_error = Some(error.to_string()), + Err(_) => last_error = Some("host key probe attempt timed out".to_string()), + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await; + + fingerprint.unwrap_or_else(|_| { + panic!( + "timed out waiting for SFTP host key fingerprint: {}", + last_error.unwrap_or_else(|| "unknown error".to_string()) + ) + }) +} + async fn wait_for_sftp(driver: &SftpDriver) { let mut last_error = None; let ready = tokio::time::timeout(Duration::from_secs(45), async { @@ -82,7 +139,18 @@ async fn test_sftp_driver_upload_download_round_trip() { .expect("resolve mapped sftp port"); let endpoint = format!("sftp://127.0.0.1:{port}"); let base_path = format!("/upload/asterdrive-itest-{}", uuid::Uuid::new_v4()); - let driver = SftpDriver::new(&sftp_policy(&endpoint, &base_path)).expect("create SftpDriver"); + let untrusted_driver = + SftpDriver::new(&sftp_policy(&endpoint, &base_path, None)).expect("create SftpDriver"); + let host_key_fingerprint = wait_for_sftp_host_key_fingerprint(&untrusted_driver).await; + SftpDriver::validate_host_key_fingerprint(&host_key_fingerprint) + .expect("reported host key fingerprint should be valid"); + + let driver = SftpDriver::new(&sftp_policy( + &endpoint, + &base_path, + Some(&host_key_fingerprint), + )) + .expect("create SftpDriver"); wait_for_sftp(&driver).await; let data = b"hello sftp world"; From 0e3522c2b73eb4e73588aeff12603db36946ee74 Mon Sep 17 00:00:00 2001 From: AptS-1547 Date: Mon, 6 Jul 2026 22:14:27 +0800 Subject: [PATCH 4/5] fix(storage): tighten SFTP review follow-ups --- .../storagePolicyOptions.ts | 3 +- .../admin/storagePolicyDialogShared.test.ts | 8 ++++ frontend-panel/src/i18n/index.test.ts | 11 ++--- .../src/i18n/locales/en/errors/storage.json | 1 + .../src/i18n/locales/zh/errors/storage.json | 1 + frontend-panel/src/services/api.generated.ts | 2 +- frontend-panel/src/types/api-helpers.ts | 1 + src/api/api_error_code.rs | 1 + src/errors.rs | 28 +++++++++++- src/storage/connectors/common.rs | 2 +- src/storage/connectors/tests.rs | 5 ++- src/storage/drivers/sftp.rs | 44 +++++++++++++++++-- src/storage/error.rs | 16 +++++++ tests/test_sftp.rs | 31 ++++++++----- 14 files changed, 130 insertions(+), 24 deletions(-) diff --git a/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts b/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts index 8522e7a40..f19fc2ce4 100644 --- a/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts +++ b/frontend-panel/src/components/admin/storage-policy-dialog/storagePolicyOptions.ts @@ -142,7 +142,8 @@ function buildDescriptorPolicyOptions( ) { continue; } - const value = form.policy_option_values?.[field.name]?.trim() ?? ""; + const rawValue = form.policy_option_values?.[field.name] ?? ""; + const value = field.trim_on_blur === true ? rawValue.trim() : rawValue; if (value) { optionRecord[field.name] = value; } diff --git a/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts b/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts index 050edeedd..b6c7924aa 100644 --- a/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts +++ b/frontend-panel/src/components/admin/storagePolicyDialogShared.test.ts @@ -1373,6 +1373,12 @@ describe("storage policy dialog helper modules", () => { name: "sftp_host_key_fingerprint", kind: "text", scope: "policy_options", + trim_on_blur: true, + }, + { + name: "api_secret", + kind: "secret", + scope: "policy_options", }, ], } as never; @@ -1385,12 +1391,14 @@ describe("storage policy dialog helper modules", () => { access_key: "aster", secret_key: "secret", policy_option_values: { + api_secret: " keep-space ", sftp_host_key_fingerprint: " SHA256:trusted ", }, }, sftpDescriptor, ).options, ).toEqual({ + api_secret: " keep-space ", sftp_host_key_fingerprint: "SHA256:trusted", }); }); diff --git a/frontend-panel/src/i18n/index.test.ts b/frontend-panel/src/i18n/index.test.ts index 6a61d5833..0966bc4a9 100644 --- a/frontend-panel/src/i18n/index.test.ts +++ b/frontend-panel/src/i18n/index.test.ts @@ -166,11 +166,12 @@ describe("i18n", () => { ApiErrorCode.PolicyRemoteNodeDisabled, ApiErrorCode.PolicyRemoteNodeBaseUrlRequired, ApiErrorCode.PolicyRemoteNodeTransferStrategyUnsupported, - "policy.onedrive_options_unsupported", - "policy.onedrive_account_mode_required", - "policy.onedrive_personal_china_cloud_unsupported", - "policy.onedrive_sharepoint_site_required", - "policy.onedrive_group_required", + ApiErrorCode.PolicyOneDriveOptionsUnsupported, + ApiErrorCode.PolicySftpOptionsUnsupported, + ApiErrorCode.PolicyOneDriveAccountModeRequired, + ApiErrorCode.PolicyOneDrivePersonalChinaCloudUnsupported, + ApiErrorCode.PolicyOneDriveSharepointSiteRequired, + ApiErrorCode.PolicyOneDriveGroupRequired, ApiErrorCode.PolicyNativeThumbnailUnsupported, ApiErrorCode.PolicyNativeMediaMetadataUnsupported, ApiErrorCode.PolicyPromotionSourceUnsupported, diff --git a/frontend-panel/src/i18n/locales/en/errors/storage.json b/frontend-panel/src/i18n/locales/en/errors/storage.json index 4b0af7fad..c5b825340 100644 --- a/frontend-panel/src/i18n/locales/en/errors/storage.json +++ b/frontend-panel/src/i18n/locales/en/errors/storage.json @@ -19,6 +19,7 @@ "policy_remote_node_base_url_required": "The selected direct remote node needs a base URL before it can be used by a storage policy", "policy_remote_node_transfer_strategy_unsupported": "This remote node does not support presigned browser transfer strategies. Use relay streaming instead", "policy_onedrive_options_unsupported": "OneDrive options can only be used with OneDrive storage policies", + "policy_sftp_options_unsupported": "SFTP host key options can only be used with SFTP storage policies", "policy_onedrive_account_mode_required": "Select a OneDrive account mode before saving this storage policy", "policy_onedrive_personal_china_cloud_unsupported": "Personal Microsoft accounts must use the Global Microsoft Graph cloud", "policy_onedrive_sharepoint_site_required": "SharePoint site mode needs a Site ID unless a Drive ID is already set", diff --git a/frontend-panel/src/i18n/locales/zh/errors/storage.json b/frontend-panel/src/i18n/locales/zh/errors/storage.json index f45fc632a..12c0e9eb7 100644 --- a/frontend-panel/src/i18n/locales/zh/errors/storage.json +++ b/frontend-panel/src/i18n/locales/zh/errors/storage.json @@ -19,6 +19,7 @@ "policy_remote_node_base_url_required": "选中的直连远程节点需要先配置 Base URL 才能用于存储策略", "policy_remote_node_transfer_strategy_unsupported": "该远程节点不支持浏览器预签名传输策略,请改用中继流式传输", "policy_onedrive_options_unsupported": "OneDrive 选项只能用于 OneDrive 存储策略", + "policy_sftp_options_unsupported": "SFTP 主机密钥选项只能用于 SFTP 存储策略", "policy_onedrive_account_mode_required": "保存 OneDrive 存储策略前需要先选择账号模式", "policy_onedrive_personal_china_cloud_unsupported": "Microsoft 个人账号必须使用 Global Microsoft Graph 云", "policy_onedrive_sharepoint_site_required": "SharePoint 站点模式需要填写 Site ID,除非已指定 Drive ID", diff --git a/frontend-panel/src/services/api.generated.ts b/frontend-panel/src/services/api.generated.ts index 44a2f6372..0692fcbd2 100644 --- a/frontend-panel/src/services/api.generated.ts +++ b/frontend-panel/src/services/api.generated.ts @@ -4610,7 +4610,7 @@ export interface components { AdminUserSortBy: "id" | "username" | "email" | "role" | "status" | "storage_used" | "storage_quota" | "created_at" | "updated_at"; ApiEmptyData: Record; /** @enum {string} */ - ApiErrorCode: "success" | "bad_request" | "not_found" | "internal_server_error" | "database.error" | "config.error" | "endpoint.not_found" | "rate_limited" | "mail.not_configured" | "mail.delivery_failed" | "conflict" | "config.public_site_url_required" | "config.public_site_url_invalid" | "auth.failed" | "auth.token_expired" | "auth.token_invalid" | "forbidden" | "auth.pending_activation" | "auth.contact_verification_invalid" | "auth.contact_verification_expired" | "auth.token_missing" | "auth.credentials_failed" | "auth.mfa_failed" | "auth.refresh_token_stale" | "auth.refresh_token_reuse_detected" | "file.not_found" | "file.too_large" | "file.type_not_allowed" | "file.upload_failed" | "upload.session_not_found" | "upload.session_expired" | "upload.chunk_failed" | "upload.assembly_failed" | "thumbnail.failed" | "resource.locked" | "precondition_failed" | "upload.assembling" | "storage.policy_not_found" | "storage.driver_error" | "storage.quota_exceeded" | "storage.unsupported_driver" | "storage.auth_failed" | "storage.permission_denied" | "storage.misconfigured" | "storage.object_not_found" | "storage.rate_limited" | "storage.transient_failure" | "storage.precondition_failed" | "storage.operation_unsupported" | "folder.not_found" | "share.not_found" | "share.expired" | "share.password_required" | "share.download_limit_reached" | "archive_preview.disabled" | "archive_preview.user_disabled" | "archive_preview.share_disabled" | "archive_preview.source_too_large" | "archive_preview.invalid_archive" | "archive_preview.manifest_too_large" | "archive_preview.unsupported_type" | "archive_preview.rejected" | "archive_preview.source_size_mismatch" | "archive_compress.disabled" | "archive_download.user_disabled" | "archive_download.share_disabled" | "auth.username_exists" | "auth.email_exists" | "auth.identifier_exists" | "auth.admin_required" | "auth.account_disabled" | "auth.request_source_untrusted" | "auth.request_origin_untrusted" | "auth.request_referer_untrusted" | "auth.request_source_missing" | "auth.session_user_mismatch" | "auth.csrf_cookie_missing" | "auth.csrf_header_missing" | "auth.csrf_token_invalid" | "auth.passkey_login_disabled" | "auth.registration_disabled" | "auth.email_blocked" | "auth.email_not_allowlisted" | "auth.mfa_flow_invalid" | "auth.mfa_flow_expired" | "auth.mfa_code_invalid" | "auth.mfa_attempts_exceeded" | "auth.mfa_factor_required" | "auth.mfa_factor_already_exists" | "auth.mfa_recovery_code_used" | "auth.mfa_email_code_required" | "auth.mfa_email_code_expired" | "auth.invitation_invalid" | "auth.invitation_expired" | "auth.invitation_revoked" | "auth.invitation_accepted" | "auth.password_change_required" | "avatar.file_required" | "avatar.upload_read_failed" | "avatar.processor_unavailable" | "avatar.empty_image" | "avatar.render_failed" | "avatar.output_invalid" | "file.name_conflict" | "file.etag_mismatch" | "file.modified_during_write" | "folder.name_conflict" | "lock.not_owner" | "share.scope_denied" | "managed_ingress.binding_mismatch" | "managed_ingress.default_delete_requires_replacement" | "managed_ingress.default_error" | "managed_ingress.default_missing" | "managed_ingress.default_not_applied" | "managed_ingress.default_update_requires_replacement" | "managed_ingress.driver_unsupported" | "managed_ingress.local_path_invalid" | "managed_ingress.required" | "managed_ingress.single_primary_required" | "remote_storage_target.not_found" | "master_binding.disabled" | "passkey.name_invalid" | "passkey.name_too_long" | "passkey.not_discoverable" | "team.not_member" | "team.owner_required" | "team.admin_or_owner_required" | "policy.upload_sessions_exist" | "policy.storage_access_key_required" | "policy.storage_secret_key_required" | "policy.storage_bucket_required" | "policy.storage_endpoint_invalid" | "policy.remote_node_required" | "policy.remote_node_unexpected" | "policy.remote_storage_target_required" | "policy.remote_node_disabled" | "policy.remote_node_base_url_required" | "policy.remote_node_transfer_strategy_unsupported" | "policy.onedrive_options_unsupported" | "policy.onedrive_account_mode_required" | "policy.onedrive_personal_china_cloud_unsupported" | "policy.onedrive_sharepoint_site_required" | "policy.onedrive_group_required" | "policy.native_thumbnail_unsupported" | "policy.native_media_metadata_unsupported" | "policy.promotion_source_unsupported" | "policy.promotion_target_unsupported" | "policy.promotion_bucket_change_denied" | "policy.action_unsupported" | "policy.action_parameter_required" | "policy.action_parameter_invalid" | "workspace.scope_denied" | "external_auth.provider_disabled" | "external_auth.policy_denied" | "external_auth.callback_redirect_uri_required" | "offline_download.aria2_rpc_auth_failed" | "offline_download.aria2_rpc_probe_failed" | "remote_node.disabled" | "remote_node.enrollment_required" | "remote_node.unique_conflict" | "storage.auth" | "storage.not_found" | "storage.permission" | "storage.precondition" | "storage.transient" | "storage.unsupported" | "storage.unknown" | "task.lease_lost" | "task.lease_renewal_timed_out" | "task.worker_shutdown_requested" | "task.retry_status_conflict" | "task.retry_not_allowed" | "team.member_exists" | "thumbnail.format_guess_failed" | "thumbnail.decode_failed" | "thumbnail.encode_failed" | "thumbnail.source_open_failed" | "thumbnail.source_stream_failed" | "thumbnail.task_panicked" | "thumbnail.source_too_large" | "thumbnail.processor_unavailable" | "thumbnail.render_failed" | "thumbnail.output_invalid" | "thumbnail.source_temp_create_failed" | "thumbnail.source_temp_flush_failed" | "thumbnail.source_temp_copy_failed" | "wopi.public_site_url_required" | "wopi.app_disabled" | "wopi.request_origin_untrusted" | "wopi.request_referer_untrusted" | "upload.temp_dir_create_failed" | "upload.temp_file_create_failed" | "upload.temp_file_write_failed" | "upload.temp_file_flush_failed" | "upload.request_body_read_failed" | "upload.request_body_size_overflow" | "upload.request_size_mismatch" | "upload.hash_temp_open_failed" | "upload.hash_temp_read_failed" | "upload.field_read_failed" | "upload.local_staging_path_resolve_failed" | "upload.local_staging_dir_create_failed" | "upload.local_staging_file_create_failed" | "upload.local_staging_write_failed" | "upload.local_staging_flush_failed" | "upload.direct_relay_write_failed" | "upload.direct_relay_shutdown_failed" | "upload.direct_relay_task_failed" | "upload.body_size_overflow" | "upload.declared_size_invalid" | "upload.empty_file" | "upload.chunk_persist_failed" | "upload.chunk_relay_failed" | "upload.chunk_transport_mismatch" | "upload.chunk_session_invalid" | "upload.chunk_number_out_of_range" | "upload.chunk_size_mismatch" | "upload.chunk_too_large" | "upload.chunk_size_overflow" | "upload.status_conflict" | "upload.completed_file_missing" | "upload.previous_failure" | "upload.parts_required" | "upload.incomplete_chunks" | "upload.incomplete_parts" | "upload.missing_part" | "upload.temp_object_missing" | "upload.temp_object_size_mismatch" | "upload.final_object_size_mismatch" | "upload.session_corrupted" | "upload.part_numbers_empty" | "upload.part_numbers_too_many" | "upload.part_number_out_of_range" | "upload.assembly_io_failed" | "upload.assembly_size_overflow" | "webdav.username_exists" | "wopi.max_expected_size_exceeded" | "validation.request_origin_invalid" | "validation.request_referer_invalid" | "validation.request_host_invalid" | "validation.request_scheme_invalid" | "validation.request_header_value_invalid" | "validation.system_already_initialized" | "search.query_empty" | "search.type_invalid" | "search.tag_match_invalid" | "search.size_range_invalid" | "search.file_filter_type_conflict" | "search.mime_type_empty" | "search.category_invalid" | "search.extensions_invalid" | "search.tag_ids_invalid" | "search.date_invalid" | "search.date_range_invalid" | "internal_storage.range_length_invalid" | "internal_storage.range_empty_object" | "internal_storage.range_offset_out_of_bounds" | "internal_storage.range_header_invalid" | "internal_storage.range_multiple_unsupported" | "internal_storage.range_bounds_invalid" | "internal_storage.content_length_required" | "internal_storage.content_length_invalid" | "internal_storage.compose_parts_required" | "internal_storage.compose_expected_size_invalid"; + ApiErrorCode: "success" | "bad_request" | "not_found" | "internal_server_error" | "database.error" | "config.error" | "endpoint.not_found" | "rate_limited" | "mail.not_configured" | "mail.delivery_failed" | "conflict" | "config.public_site_url_required" | "config.public_site_url_invalid" | "auth.failed" | "auth.token_expired" | "auth.token_invalid" | "forbidden" | "auth.pending_activation" | "auth.contact_verification_invalid" | "auth.contact_verification_expired" | "auth.token_missing" | "auth.credentials_failed" | "auth.mfa_failed" | "auth.refresh_token_stale" | "auth.refresh_token_reuse_detected" | "file.not_found" | "file.too_large" | "file.type_not_allowed" | "file.upload_failed" | "upload.session_not_found" | "upload.session_expired" | "upload.chunk_failed" | "upload.assembly_failed" | "thumbnail.failed" | "resource.locked" | "precondition_failed" | "upload.assembling" | "storage.policy_not_found" | "storage.driver_error" | "storage.quota_exceeded" | "storage.unsupported_driver" | "storage.auth_failed" | "storage.permission_denied" | "storage.misconfigured" | "storage.object_not_found" | "storage.rate_limited" | "storage.transient_failure" | "storage.precondition_failed" | "storage.operation_unsupported" | "folder.not_found" | "share.not_found" | "share.expired" | "share.password_required" | "share.download_limit_reached" | "archive_preview.disabled" | "archive_preview.user_disabled" | "archive_preview.share_disabled" | "archive_preview.source_too_large" | "archive_preview.invalid_archive" | "archive_preview.manifest_too_large" | "archive_preview.unsupported_type" | "archive_preview.rejected" | "archive_preview.source_size_mismatch" | "archive_compress.disabled" | "archive_download.user_disabled" | "archive_download.share_disabled" | "auth.username_exists" | "auth.email_exists" | "auth.identifier_exists" | "auth.admin_required" | "auth.account_disabled" | "auth.request_source_untrusted" | "auth.request_origin_untrusted" | "auth.request_referer_untrusted" | "auth.request_source_missing" | "auth.session_user_mismatch" | "auth.csrf_cookie_missing" | "auth.csrf_header_missing" | "auth.csrf_token_invalid" | "auth.passkey_login_disabled" | "auth.registration_disabled" | "auth.email_blocked" | "auth.email_not_allowlisted" | "auth.mfa_flow_invalid" | "auth.mfa_flow_expired" | "auth.mfa_code_invalid" | "auth.mfa_attempts_exceeded" | "auth.mfa_factor_required" | "auth.mfa_factor_already_exists" | "auth.mfa_recovery_code_used" | "auth.mfa_email_code_required" | "auth.mfa_email_code_expired" | "auth.invitation_invalid" | "auth.invitation_expired" | "auth.invitation_revoked" | "auth.invitation_accepted" | "auth.password_change_required" | "avatar.file_required" | "avatar.upload_read_failed" | "avatar.processor_unavailable" | "avatar.empty_image" | "avatar.render_failed" | "avatar.output_invalid" | "file.name_conflict" | "file.etag_mismatch" | "file.modified_during_write" | "folder.name_conflict" | "lock.not_owner" | "share.scope_denied" | "managed_ingress.binding_mismatch" | "managed_ingress.default_delete_requires_replacement" | "managed_ingress.default_error" | "managed_ingress.default_missing" | "managed_ingress.default_not_applied" | "managed_ingress.default_update_requires_replacement" | "managed_ingress.driver_unsupported" | "managed_ingress.local_path_invalid" | "managed_ingress.required" | "managed_ingress.single_primary_required" | "remote_storage_target.not_found" | "master_binding.disabled" | "passkey.name_invalid" | "passkey.name_too_long" | "passkey.not_discoverable" | "team.not_member" | "team.owner_required" | "team.admin_or_owner_required" | "policy.upload_sessions_exist" | "policy.storage_access_key_required" | "policy.storage_secret_key_required" | "policy.storage_bucket_required" | "policy.storage_endpoint_invalid" | "policy.remote_node_required" | "policy.remote_node_unexpected" | "policy.remote_storage_target_required" | "policy.remote_node_disabled" | "policy.remote_node_base_url_required" | "policy.remote_node_transfer_strategy_unsupported" | "policy.onedrive_options_unsupported" | "policy.sftp_options_unsupported" | "policy.onedrive_account_mode_required" | "policy.onedrive_personal_china_cloud_unsupported" | "policy.onedrive_sharepoint_site_required" | "policy.onedrive_group_required" | "policy.native_thumbnail_unsupported" | "policy.native_media_metadata_unsupported" | "policy.promotion_source_unsupported" | "policy.promotion_target_unsupported" | "policy.promotion_bucket_change_denied" | "policy.action_unsupported" | "policy.action_parameter_required" | "policy.action_parameter_invalid" | "workspace.scope_denied" | "external_auth.provider_disabled" | "external_auth.policy_denied" | "external_auth.callback_redirect_uri_required" | "offline_download.aria2_rpc_auth_failed" | "offline_download.aria2_rpc_probe_failed" | "remote_node.disabled" | "remote_node.enrollment_required" | "remote_node.unique_conflict" | "storage.auth" | "storage.not_found" | "storage.permission" | "storage.precondition" | "storage.transient" | "storage.unsupported" | "storage.unknown" | "task.lease_lost" | "task.lease_renewal_timed_out" | "task.worker_shutdown_requested" | "task.retry_status_conflict" | "task.retry_not_allowed" | "team.member_exists" | "thumbnail.format_guess_failed" | "thumbnail.decode_failed" | "thumbnail.encode_failed" | "thumbnail.source_open_failed" | "thumbnail.source_stream_failed" | "thumbnail.task_panicked" | "thumbnail.source_too_large" | "thumbnail.processor_unavailable" | "thumbnail.render_failed" | "thumbnail.output_invalid" | "thumbnail.source_temp_create_failed" | "thumbnail.source_temp_flush_failed" | "thumbnail.source_temp_copy_failed" | "wopi.public_site_url_required" | "wopi.app_disabled" | "wopi.request_origin_untrusted" | "wopi.request_referer_untrusted" | "upload.temp_dir_create_failed" | "upload.temp_file_create_failed" | "upload.temp_file_write_failed" | "upload.temp_file_flush_failed" | "upload.request_body_read_failed" | "upload.request_body_size_overflow" | "upload.request_size_mismatch" | "upload.hash_temp_open_failed" | "upload.hash_temp_read_failed" | "upload.field_read_failed" | "upload.local_staging_path_resolve_failed" | "upload.local_staging_dir_create_failed" | "upload.local_staging_file_create_failed" | "upload.local_staging_write_failed" | "upload.local_staging_flush_failed" | "upload.direct_relay_write_failed" | "upload.direct_relay_shutdown_failed" | "upload.direct_relay_task_failed" | "upload.body_size_overflow" | "upload.declared_size_invalid" | "upload.empty_file" | "upload.chunk_persist_failed" | "upload.chunk_relay_failed" | "upload.chunk_transport_mismatch" | "upload.chunk_session_invalid" | "upload.chunk_number_out_of_range" | "upload.chunk_size_mismatch" | "upload.chunk_too_large" | "upload.chunk_size_overflow" | "upload.status_conflict" | "upload.completed_file_missing" | "upload.previous_failure" | "upload.parts_required" | "upload.incomplete_chunks" | "upload.incomplete_parts" | "upload.missing_part" | "upload.temp_object_missing" | "upload.temp_object_size_mismatch" | "upload.final_object_size_mismatch" | "upload.session_corrupted" | "upload.part_numbers_empty" | "upload.part_numbers_too_many" | "upload.part_number_out_of_range" | "upload.assembly_io_failed" | "upload.assembly_size_overflow" | "webdav.username_exists" | "wopi.max_expected_size_exceeded" | "validation.request_origin_invalid" | "validation.request_referer_invalid" | "validation.request_host_invalid" | "validation.request_scheme_invalid" | "validation.request_header_value_invalid" | "validation.system_already_initialized" | "search.query_empty" | "search.type_invalid" | "search.tag_match_invalid" | "search.size_range_invalid" | "search.file_filter_type_conflict" | "search.mime_type_empty" | "search.category_invalid" | "search.extensions_invalid" | "search.tag_ids_invalid" | "search.date_invalid" | "search.date_range_invalid" | "internal_storage.range_length_invalid" | "internal_storage.range_empty_object" | "internal_storage.range_offset_out_of_bounds" | "internal_storage.range_header_invalid" | "internal_storage.range_multiple_unsupported" | "internal_storage.range_bounds_invalid" | "internal_storage.content_length_required" | "internal_storage.content_length_invalid" | "internal_storage.compose_parts_required" | "internal_storage.compose_expected_size_invalid"; ApiErrorDiagnostic: { kind: string; message: string; diff --git a/frontend-panel/src/types/api-helpers.ts b/frontend-panel/src/types/api-helpers.ts index 7f6813365..0e3928571 100644 --- a/frontend-panel/src/types/api-helpers.ts +++ b/frontend-panel/src/types/api-helpers.ts @@ -150,6 +150,7 @@ export const ApiErrorCode = { PolicyRemoteNodeTransferStrategyUnsupported: "policy.remote_node_transfer_strategy_unsupported", PolicyOneDriveOptionsUnsupported: "policy.onedrive_options_unsupported", + PolicySftpOptionsUnsupported: "policy.sftp_options_unsupported", PolicyOneDriveAccountModeRequired: "policy.onedrive_account_mode_required", PolicyOneDrivePersonalChinaCloudUnsupported: "policy.onedrive_personal_china_cloud_unsupported", diff --git a/src/api/api_error_code.rs b/src/api/api_error_code.rs index e1dbce6b2..91237b97c 100644 --- a/src/api/api_error_code.rs +++ b/src/api/api_error_code.rs @@ -210,6 +210,7 @@ define_api_error_codes! { PolicyRemoteNodeBaseUrlRequired => "policy.remote_node_base_url_required", PolicyRemoteNodeTransferStrategyUnsupported => "policy.remote_node_transfer_strategy_unsupported", PolicyOneDriveOptionsUnsupported => "policy.onedrive_options_unsupported", + PolicySftpOptionsUnsupported => "policy.sftp_options_unsupported", PolicyOneDriveAccountModeRequired => "policy.onedrive_account_mode_required", PolicyOneDrivePersonalChinaCloudUnsupported => "policy.onedrive_personal_china_cloud_unsupported", PolicyOneDriveSharePointSiteRequired => "policy.onedrive_sharepoint_site_required", diff --git a/src/errors.rs b/src/errors.rs index ec9983f16..73ac73460 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -6,13 +6,15 @@ use std::any::Any; use crate::api::api_error_code::ApiErrorCode; use crate::api::response::{ApiErrorDiagnostic, ApiErrorInfo}; use crate::storage::error::{ - StorageErrorKind, storage_driver_error_display_message, storage_driver_error_kind_from_message, + StorageErrorContext, StorageErrorKind, storage_driver_error_display_message, + storage_driver_error_kind_from_message, }; #[derive(Debug, Clone)] pub struct AsterErrorPayload { message: String, api_code: Option, + storage_context: Option, } impl AsterErrorPayload { @@ -20,6 +22,7 @@ impl AsterErrorPayload { Self { message: message.into(), api_code: None, + storage_context: None, } } @@ -35,6 +38,15 @@ impl AsterErrorPayload { fn api_code(&self) -> Option { self.api_code } + + fn with_storage_context(mut self, context: StorageErrorContext) -> Self { + self.storage_context = Some(context); + self + } + + fn storage_context(&self) -> Option<&StorageErrorContext> { + self.storage_context.as_ref() + } } impl PartialEq<&str> for AsterErrorPayload { @@ -94,6 +106,12 @@ macro_rules! define_errors { } } + pub(crate) fn storage_error_context(&self) -> Option<&StorageErrorContext> { + match self { + $(AsterError::$variant(payload) => payload.storage_context(),)* + } + } + pub fn with_api_error_code(self, api_code: ApiErrorCode) -> Self { match self { $(AsterError::$variant(payload) => { @@ -102,6 +120,14 @@ macro_rules! define_errors { } } + pub(crate) fn with_storage_error_context(self, context: StorageErrorContext) -> Self { + match self { + $(AsterError::$variant(payload) => { + AsterError::$variant(payload.with_storage_context(context)) + })* + } + } + /// 错误类型名称 pub fn error_type(&self) -> &'static str { match self { diff --git a/src/storage/connectors/common.rs b/src/storage/connectors/common.rs index 9a926b434..5364c22be 100644 --- a/src/storage/connectors/common.rs +++ b/src/storage/connectors/common.rs @@ -199,7 +199,7 @@ pub(super) fn ensure_sftp_options_absent( ) -> Result<()> { if options.sftp_host_key_fingerprint.is_some() { return Err(validation_error_with_code( - ApiErrorCode::BadRequest, + ApiErrorCode::PolicySftpOptionsUnsupported, "SFTP host key options are only valid for SFTP storage policies", )); } diff --git a/src/storage/connectors/tests.rs b/src/storage/connectors/tests.rs index ca23fead6..80dd5540b 100644 --- a/src/storage/connectors/tests.rs +++ b/src/storage/connectors/tests.rs @@ -848,7 +848,10 @@ fn sftp_host_key_options_are_rejected_for_non_sftp_connector() { let error = common::ensure_sftp_options_absent(&options).unwrap_err(); - assert_eq!(error.api_error_code(), ApiErrorCode::BadRequest); + assert_eq!( + error.api_error_code(), + ApiErrorCode::PolicySftpOptionsUnsupported + ); assert!( error .to_string() diff --git a/src/storage/drivers/sftp.rs b/src/storage/drivers/sftp.rs index 744ed8db2..2b7a18395 100644 --- a/src/storage/drivers/sftp.rs +++ b/src/storage/drivers/sftp.rs @@ -14,7 +14,9 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, ReadBuf}; use crate::entities::storage_policy; use crate::errors::{AsterError, Result}; -use crate::storage::error::{StorageErrorKind, storage_driver_error}; +use crate::storage::error::{ + StorageErrorContext, StorageErrorKind, storage_driver_error, storage_driver_error_with_context, +}; use crate::storage::{BlobMetadata, StorageDriver, StreamUploadDriver}; use crate::types::parse_storage_policy_options; @@ -37,12 +39,33 @@ pub struct SftpDriver { host_key_fingerprint: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SftpHostKeyRejection { + pub expected: Option, + pub actual: String, +} + #[derive(Debug, Clone, PartialEq, Eq)] enum HostKeyRejection { MissingPin { actual: String }, Mismatch { expected: String, actual: String }, } +impl HostKeyRejection { + fn expected(&self) -> Option<&str> { + match self { + Self::MissingPin { .. } => None, + Self::Mismatch { expected, .. } => Some(expected), + } + } + + fn actual(&self) -> &str { + match self { + Self::MissingPin { actual } | Self::Mismatch { actual, .. } => actual, + } + } +} + #[derive(Clone)] struct TrustServerKeyClient { expected_fingerprint: Option, @@ -149,6 +172,15 @@ impl SftpDriver { Ok(()) } + pub fn host_key_rejection(error: &AsterError) -> Option { + let StorageErrorContext::SftpHostKeyRejected { expected, actual } = + error.storage_error_context()?; + Some(SftpHostKeyRejection { + expected: expected.clone(), + actual: actual.clone(), + }) + } + async fn connect(&self) -> Result { let mut config = russh::client::Config::default(); config.inactivity_timeout = Some(IO_TIMEOUT); @@ -446,20 +478,26 @@ fn host_key_rejection_error( rejection: &Arc>>, ) -> Option { let rejection = rejection.lock().ok()?.clone()?; + let context = StorageErrorContext::SftpHostKeyRejected { + expected: rejection.expected().map(ToOwned::to_owned), + actual: rejection.actual().to_string(), + }; Some(match rejection { - HostKeyRejection::MissingPin { actual } => storage_driver_error( + HostKeyRejection::MissingPin { actual } => storage_driver_error_with_context( StorageErrorKind::Precondition, format!( "SFTP host key is not trusted for {}:{}. Confirm fingerprint {actual} and save it as sftp_host_key_fingerprint before testing again", endpoint.host, endpoint.port ), + context, ), - HostKeyRejection::Mismatch { expected, actual } => storage_driver_error( + HostKeyRejection::Mismatch { expected, actual } => storage_driver_error_with_context( StorageErrorKind::Precondition, format!( "SFTP host key mismatch for {}:{}. Expected {expected}, got {actual}. Verify the server identity before updating sftp_host_key_fingerprint", endpoint.host, endpoint.port ), + context, ), }) } diff --git a/src/storage/error.rs b/src/storage/error.rs index 33fce2c88..622fffe24 100644 --- a/src/storage/error.rs +++ b/src/storage/error.rs @@ -20,6 +20,14 @@ pub enum StorageErrorKind { Unknown, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StorageErrorContext { + SftpHostKeyRejected { + expected: Option, + actual: String, + }, +} + impl StorageErrorKind { pub fn as_str(self) -> &'static str { match self { @@ -55,6 +63,14 @@ pub fn storage_driver_error(kind: StorageErrorKind, message: impl Into) AsterError::storage_driver_error(encode_storage_driver_error_message(kind, message.into())) } +pub fn storage_driver_error_with_context( + kind: StorageErrorKind, + message: impl Into, + context: StorageErrorContext, +) -> AsterError { + storage_driver_error(kind, message).with_storage_error_context(context) +} + pub fn storage_driver_error_with_code( kind: StorageErrorKind, api_code: ApiErrorCode, diff --git a/tests/test_sftp.rs b/tests/test_sftp.rs index 87e4a5763..040e075b1 100644 --- a/tests/test_sftp.rs +++ b/tests/test_sftp.rs @@ -53,16 +53,15 @@ fn sftp_policy( } } -fn extract_sftp_host_key_fingerprint(error: &aster_drive::errors::AsterError) -> String { - let message = error.message(); - let marker = "Confirm fingerprint "; - let (_, rest) = message - .split_once(marker) - .expect("untrusted host key error should include fingerprint"); - rest.split_whitespace() - .next() - .expect("fingerprint should be present") - .to_string() +fn docker_sftp_test_enabled() -> bool { + std::env::var("ASTER_SFTP_TEST_DOCKER") + .map(|value| { + !matches!( + value.as_str(), + "0" | "false" | "FALSE" | "no" | "NO" | "off" | "OFF" + ) + }) + .unwrap_or(true) } async fn wait_for_sftp_host_key_fingerprint(driver: &SftpDriver) -> String { @@ -76,7 +75,10 @@ async fn wait_for_sftp_host_key_fingerprint(driver: &SftpDriver) -> String { Ok(Err(error)) if error.storage_error_kind() == Some(StorageErrorKind::Precondition) => { - break extract_sftp_host_key_fingerprint(&error); + let rejection = SftpDriver::host_key_rejection(&error) + .expect("untrusted host key error should expose rejection details"); + assert_eq!(rejection.expected, None); + break rejection.actual; } Ok(Err(error)) => last_error = Some(error.to_string()), Err(_) => last_error = Some("host key probe attempt timed out".to_string()), @@ -126,6 +128,13 @@ async fn wait_for_sftp(driver: &SftpDriver) { #[tokio::test] async fn test_sftp_driver_upload_download_round_trip() { + if !docker_sftp_test_enabled() { + eprintln!( + "skipping SFTP docker integration test because ASTER_SFTP_TEST_DOCKER disables it" + ); + return; + } + let container = GenericImage::new(SFTP_IMAGE, SFTP_TAG) .with_exposed_port(IntoContainerPort::tcp(SFTP_PORT)) .with_cmd(vec![format!("{SFTP_USERNAME}:{SFTP_PASSWORD}:::upload")]) From ddc9073b1fa3a7b563d7970a1e07b9b9ea010a70 Mon Sep 17 00:00:00 2001 From: AptS-1547 Date: Mon, 6 Jul 2026 22:49:49 +0800 Subject: [PATCH 5/5] docs(storage): document SFTP policy setup --- developer-docs/en/api/admin.md | 10 +- developer-docs/en/architecture.md | 7 +- ...orage-descriptor-normalization-contract.md | 4 + developer-docs/en/testing.md | 20 +++ developer-docs/zh-CN/api/admin.md | 10 +- developer-docs/zh-CN/architecture.md | 7 +- ...orage-descriptor-normalization-contract.md | 4 + developer-docs/zh-CN/testing.md | 20 +++ docs/.vitepress/config.ts | 2 + docs/config/storage.md | 17 +- docs/en/config/storage.md | 17 +- docs/en/features/upload-storage.md | 8 +- docs/en/guide/upload-modes.md | 19 +- docs/en/index.md | 6 +- docs/en/storage/index.md | 3 +- docs/en/storage/sftp.md | 169 ++++++++++++++++++ docs/features/upload-storage.md | 8 +- docs/guide/upload-modes.md | 19 +- docs/index.md | 4 +- docs/storage/index.md | 3 +- docs/storage/sftp.md | 169 ++++++++++++++++++ 21 files changed, 489 insertions(+), 37 deletions(-) create mode 100644 docs/en/storage/sftp.md create mode 100644 docs/storage/sftp.md diff --git a/developer-docs/en/api/admin.md b/developer-docs/en/api/admin.md index 19b5215d7..e4b63406b 100644 --- a/developer-docs/en/api/admin.md +++ b/developer-docs/en/api/admin.md @@ -76,7 +76,7 @@ Create example: Current notes: -- `driver_type` currently supports `local`, `s3`, `azure_blob`, `tencent_cos`, `remote`, and `onedrive` +- `driver_type` currently supports `local`, `s3`, `sftp`, `azure_blob`, `tencent_cos`, `remote`, and `one_drive` - `GET /admin/policies/storage-drivers` returns `StorageConnectorDescriptor` entries. The frontend should use descriptor `capabilities`, `fields`, `upload_workflows`, `actions`, and `credential_mode` to decide forms, connection tests, upload/download strategies, and action affordances instead of maintaining a hard-coded driver capability matrix. - create and update both honor request `chunk_size` - `options` carries policy-level behavior: @@ -88,11 +88,13 @@ Current notes: - storage-native thumbnails / image previews with `storage_native_processing_enabled`, `thumbnail_processor`, and `thumbnail_extensions` - storage-native media metadata with `storage_native_media_metadata_enabled` and `media_metadata_extensions` - OneDrive location options: `onedrive_account_mode`, `onedrive_tenant`, `onedrive_site_id`, `onedrive_drive_id`, `onedrive_group_id`, and `onedrive_root_item_id` + - SFTP host key pinning: `sftp_host_key_fingerprint` - `application_config.microsoft_graph` stores OneDrive / Microsoft Graph app settings. Client secrets are stored encrypted; API responses expose only `client_secret_configured`. - `driver_type = "azure_blob"` uses Azure Block Blob capabilities. Presigned browser upload uses SAS URLs and requires the client to send `x-ms-blob-type: BlockBlob`. -- `driver_type = "onedrive"` uses Microsoft Graph OAuth credentials. Save the policy and `application_config.microsoft_graph` before starting authorization. +- `driver_type = "one_drive"` uses Microsoft Graph OAuth credentials. Save the policy and `application_config.microsoft_graph` before starting authorization. +- `driver_type = "sftp"` uses SSH username / password credentials to connect to an SFTP server. Endpoint supports `sftp://host:port`, bare `host`, and `host:port`; the remote root belongs in `base_path`. Unknown or mismatched SSH host keys are rejected as `StorageErrorKind::Precondition` with diagnostics that include actual / expected fingerprints; the confirmed fingerprint is stored in `options.sftp_host_key_fingerprint`. - `driver_type = "tencent_cos"` uses the S3-compatible object path for normal reads and writes, validates Tencent COS endpoint shape, and can expose COS CI storage-native thumbnail / image-preview / media-metadata capabilities when the policy opts in -- built-in Local, S3-compatible, Azure Blob, OneDrive, and Remote drivers do not expose storage-native thumbnail, image-preview, or media-metadata capabilities +- built-in Local, S3-compatible, SFTP, Azure Blob, OneDrive, and Remote drivers do not expose storage-native thumbnail, image-preview, or media-metadata capabilities - legacy `{"presigned_upload":true}` remains compatible with object-storage presigned upload - `allowed_types` can be managed through REST - `driver_type = "remote"` requires `remote_node_id` @@ -130,7 +132,7 @@ Failed connection tests no longer return a `StoragePolicyProbeResult` success pa } ``` -Draft test requests support optional `policy_id`. While editing a saved policy, blank sensitive fields such as `access_key` or `secret_key` can be filled from the saved policy by S3-compatible, Azure Blob, and Tencent COS connectors. Unsaved new policies must still provide complete credentials. +Draft test requests support optional `policy_id`. While editing a saved policy, blank sensitive fields such as `access_key` or `secret_key` can be filled from the saved policy by S3-compatible, SFTP, Azure Blob, and Tencent COS connectors. Unsaved new policies must still provide complete credentials. ### Storage OAuth Credentials diff --git a/developer-docs/en/architecture.md b/developer-docs/en/architecture.md index ddd04940c..5014edec7 100644 --- a/developer-docs/en/architecture.md +++ b/developer-docs/en/architecture.md @@ -148,8 +148,9 @@ WebDAV does not go through `src/api/routes/**`. Instead: │ Infrastructure layer │ │ - SeaORM + migration │ │ - StorageConnector descriptor / action │ -│ - StorageDriver(Local/S3/Azure/OneDrive) │ -│ - StorageDriver(Tencent COS / Remote) │ +│ - StorageDriver(Local/S3/SFTP/Azure) │ +│ - StorageDriver(Tencent COS/OneDrive) │ +│ - StorageDriver(Remote) │ │ - CacheBackend(Memory / Redis) │ ├─────────────────────────────────────────────┤ │ Data layer │ @@ -190,7 +191,7 @@ The practical rule of thumb in this repository remains: | `src/api/routes/remote_tunnel.rs` | Primary-side remote-node reverse tunnel internal entry | | `src/services/` | Central business rule layer | | `src/storage/connectors/` | Storage connectors: descriptors, fields, actions, connection tests, upload workflows, and credential requirements | -| `src/storage/drivers/` | Local, S3-compatible, Azure Blob, Tencent COS, OneDrive, and remote drivers | +| `src/storage/drivers/` | Local, S3-compatible, SFTP, Azure Blob, Tencent COS, OneDrive, and remote drivers | | `src/storage/remote_protocol/tunnel/` | Reverse tunnel transport runtime, auth, registry, and streaming responses | | `src/webdav/` | WebDAV filesystem, auth, locks, and DeltaV support | | `frontend-panel/` | React 19 + Vite frontend; build artifacts are served by the backend | diff --git a/developer-docs/en/storage-descriptor-normalization-contract.md b/developer-docs/en/storage-descriptor-normalization-contract.md index 324e39fa1..bdaf1e6d5 100644 --- a/developer-docs/en/storage-descriptor-normalization-contract.md +++ b/developer-docs/en/storage-descriptor-normalization-contract.md @@ -18,6 +18,7 @@ Shared field meanings and pure normalization helpers live in `src/storage/field_ - Admin fields, actions, capabilities, and UI helper metadata must come from backend descriptors first. The frontend must not infer connection tests, authorization, upload strategy, native processing, remote binding, or field visibility from a local `driver_type` capability matrix. - `label_key`, `help_key`, `placeholder`, `required_message_key`, and similar fields are stable localization keys or hint parameters. The frontend owns final localized text, but field presence, required state, sensitivity, and supported actions are backend descriptor concerns. - `secret: true` or secret field kind means the frontend must render a sensitive input and backend logs / `Debug` output must not expose plaintext. Create flows follow descriptor `required` plus backend validation. Edit flows treat omitted secret fields as "preserve the stored value"; explicit values replace the stored secret after normalization. +- `StorageConnectorFieldScope::PolicyOptions` fields belong to `storage_policy.options`. The frontend must render and normalize them from descriptors, not from a local per-driver field matrix. SFTP `sftp_host_key_fingerprint` is such a field: the backend declares the field, label key, trim behavior, and validation rule; the frontend only displays and submits it. - Unsupported drivers must produce stable backend errors. Remote storage targets may expose only known registered drivers that the remote capability payload declares. Unknown wire-level driver ids may be preserved, but they are not locally configurable drivers. - When descriptors, remote capabilities, or capability parsing are missing, the frontend may only fall back conservatively by hiding risky actions or showing an unavailable state. It must not recreate a local capability matrix in that fallback path. - Action descriptors declare whether an entry point requires a saved policy, authorization credential, or remote-state mutation. Routes and services still perform final validation; hidden buttons are not authorization. @@ -30,6 +31,8 @@ Field normalization belongs in backend use cases or connector/driver-specific pu - Local remote storage target `base_path` uses `normalize_relative_local_target_path` through the remote target service wrapper: trim whitespace, collapse `.` segments, reject blanks, absolute paths, `..`, Windows prefixes, and backslash escapes, then resolve within `server.follower.remote_storage_target_local_root`. - Object-storage remote storage target `base_path` is a prefix: trim whitespace and outer `/`; an empty prefix means bucket/container root. - Storage policy object-storage endpoint/bucket normalization uses `normalize_s3_endpoint_and_bucket` plus connector-specific API error mapping. Non-empty endpoints must be `http://` or `https://` and include a hostname; bucket/container is required. +- Storage policy SFTP endpoint normalization is handled by `parse_sftp_endpoint`: it allows `sftp://host:port`, bare `host`, and `host:port`, with default port `22`; only a real `://` scheme separator triggers URL-scheme validation. Paths, query strings, fragments, and URL credentials are invalid; the remote root must use `base_path`. +- SFTP host key fingerprints live in `storage_policy.options.sftp_host_key_fingerprint`. Unknown or mismatched host keys must fail closed and expose actual / expected fingerprints through structured `SftpHostKeyRejected` context; tests must not parse error text. - `max_file_size = 0` means no extra policy limit; negative values are invalid at the service boundary. Upload paths still perform final size checks when applying a policy or target. - Same-driver edits preserve omitted `access_key` / `secret_key` values. Explicit replacements are trimmed and revalidated. - When changing a remote storage target driver, old driver-specific fields must not leak into the new driver. Endpoint, bucket, access key, and secret key reset; base path follows the new driver input/default semantics and is normalized again. @@ -48,6 +51,7 @@ When descriptor or normalization behavior changes, add focused unit tests: - Descriptor tests must cover every built-in driver field, secret marker, action, and key capability. - Normalization tests must cover trimming, blank values, path escapes, prefix slash trimming, negative `max_file_size`, same-driver secret preservation, explicit secret replacement, and driver-change field reset. +- SFTP coverage must include bare host, `host:port`, `sftp://host:port`, wrong schemes, host key fingerprint format, unknown-host-key rejection, and accepted pinned fingerprints. - For storage policy descriptor behavior, run `cargo test --lib storage::connectors` or a narrower filter. - For remote storage target normalization, run `cargo test --lib remote_storage_target_service::tests::`. - OpenAPI schema changes require OpenAPI export and frontend SDK regeneration. This contract slice does not change public API shapes. diff --git a/developer-docs/en/testing.md b/developer-docs/en/testing.md index 0aa2d78ae..f41c35c20 100644 --- a/developer-docs/en/testing.md +++ b/developer-docs/en/testing.md @@ -122,3 +122,23 @@ If you suspect the test did not switch backends as expected, check these three t 1. Does the test case actually use `common::setup()`? 2. Is `ASTER_TEST_DATABASE_BACKEND` exported in the shell? 3. Is Docker available locally, and can the corresponding image start successfully? + +## SFTP Integration Tests + +The SFTP driver has a dedicated integration test: + +```bash +cargo test --test test_sftp +``` + +This test starts an `atmoz/sftp` container through `testcontainers` by default and runs a real upload, download, range read, delete, and host-key fingerprint confirmation flow. It requires a local Docker / container runtime. + +If the current environment cannot run Docker, disable it explicitly: + +```bash +ASTER_SFTP_TEST_DOCKER=0 cargo test --test test_sftp +``` + +With that variable set, the container round trip is skipped. Do not make this the default CI behavior; SFTP is a real storage driver, so PRs touching the driver, connector, descriptor, or upload/download path should keep the default Docker test enabled. + +`src/storage/drivers/sftp.rs` also contains a manual real-server test that requires `ASTER_SFTP_TEST_*` and `ASTER_SFTP_TEST_HOST_KEY_FINGERPRINT`. It does not replace the default Docker coverage in `tests/test_sftp.rs`; it is mainly for debugging compatibility with a specific SFTP server. diff --git a/developer-docs/zh-CN/api/admin.md b/developer-docs/zh-CN/api/admin.md index ee45c9bd3..c142a40a2 100644 --- a/developer-docs/zh-CN/api/admin.md +++ b/developer-docs/zh-CN/api/admin.md @@ -91,7 +91,7 @@ 当前实现注意点: -- `driver_type` 当前支持 `local`、`s3`、`azure_blob`、`tencent_cos`、`remote` 和 `onedrive` +- `driver_type` 当前支持 `local`、`s3`、`sftp`、`azure_blob`、`tencent_cos`、`remote` 和 `one_drive` - `GET /admin/policies/storage-drivers` 返回 `StorageConnectorDescriptor` 列表,前端应以 descriptor 的 `capabilities`、`fields`、`upload_workflows`、`actions` 和 `credential_mode` 决定表单、连接测试、上传/下载策略和操作入口,不要在前端维护一份 driver-type 能力矩阵 - 创建和更新都会采用请求里的 `chunk_size` - `options` 当前承载策略级行为: @@ -103,11 +103,13 @@ - 存储原生缩略图 / 图片预览:`storage_native_processing_enabled`、`thumbnail_processor`、`thumbnail_extensions` - 存储原生媒体元数据:`storage_native_media_metadata_enabled`、`media_metadata_extensions` - OneDrive 位置选项:`onedrive_account_mode`、`onedrive_tenant`、`onedrive_site_id`、`onedrive_drive_id`、`onedrive_group_id`、`onedrive_root_item_id` + - SFTP 主机密钥固定:`sftp_host_key_fingerprint` - `application_config.microsoft_graph` 用于保存 OneDrive / Microsoft Graph 应用配置;client secret 只写入加密存储,API 响应只暴露 `client_secret_configured` - `driver_type = "azure_blob"` 使用 Azure Blob Block Blob 能力,预签名上传使用 SAS URL,前端直传时需要带 `x-ms-blob-type: BlockBlob` -- `driver_type = "onedrive"` 使用 Microsoft Graph OAuth 凭据,授权前需要先保存策略和 `application_config.microsoft_graph` +- `driver_type = "one_drive"` 使用 Microsoft Graph OAuth 凭据,授权前需要先保存策略和 `application_config.microsoft_graph` +- `driver_type = "sftp"` 使用 SSH 用户名 / 密码连接 SFTP 服务器;Endpoint 支持 `sftp://host:port`、裸 `host` 和 `host:port`,远程根目录放在 `base_path`。未知或不匹配的 SSH 主机密钥会以 `StorageErrorKind::Precondition` 拒绝,并通过诊断提示 actual / expected 指纹;确认后的指纹保存在 `options.sftp_host_key_fingerprint`。 - `driver_type = "tencent_cos"` 普通读写复用 S3-compatible 对象存储路径,会校验 Tencent COS endpoint 形态;策略启用后可通过 COS CI 暴露原生缩略图、图片预览和媒体元数据能力 -- 内置 Local、S3-compatible、Azure Blob、OneDrive 和 Remote 驱动不暴露存储原生缩略图、图片预览或媒体元数据能力 +- 内置 Local、S3-compatible、SFTP、Azure Blob、OneDrive 和 Remote 驱动不暴露存储原生缩略图、图片预览或媒体元数据能力 - 旧配置 `{"presigned_upload":true}` 仍兼容,等价于 S3 预签名上传策略 - `POST /admin/policies/{id}/promote-s3-driver` 当前支持把通用 `s3` 策略提升为 `tencent_cos`。请求体必须包含目标驱动和当前 endpoint / bucket,例如 `{ "target_driver_type": "tencent_cos", "endpoint": "https://bucket-1250000000.cos.ap-guangzhou.myqcloud.com", "bucket": "bucket-1250000000" }`。提升时不允许改变 bucket;若该策略还有活动上传 session,或目标驱动不能接受当前 endpoint / bucket 组合,会直接拒绝。 - REST 已经可以通过 `allowed_types` 管理策略允许的 MIME / 类型列表;不传时创建会使用空列表,更新会保持原值 @@ -149,7 +151,7 @@ } ``` -草稿测试请求支持可选 `policy_id`。编辑已保存策略时,如果 `access_key`、`secret_key` 等敏感字段为空,S3-compatible、Azure Blob 和 Tencent COS connector 会从该策略已保存凭据补齐空白字段;新建未保存策略时仍必须传完整凭据。 +草稿测试请求支持可选 `policy_id`。编辑已保存策略时,如果 `access_key`、`secret_key` 等敏感字段为空,S3-compatible、SFTP、Azure Blob 和 Tencent COS connector 会从该策略已保存凭据补齐空白字段;新建未保存策略时仍必须传完整凭据。 ### 存储 OAuth 凭据 diff --git a/developer-docs/zh-CN/architecture.md b/developer-docs/zh-CN/architecture.md index 70d2e97f3..d47d3c3b9 100644 --- a/developer-docs/zh-CN/architecture.md +++ b/developer-docs/zh-CN/architecture.md @@ -148,8 +148,9 @@ WebDAV 不走 `src/api/routes/**`,而是: │ 基础设施层 │ │ - SeaORM + migration │ │ - StorageConnector descriptor / action │ -│ - StorageDriver(Local/S3/Azure/OneDrive) │ -│ - StorageDriver(Tencent COS / Remote) │ +│ - StorageDriver(Local/S3/SFTP/Azure) │ +│ - StorageDriver(Tencent COS/OneDrive) │ +│ - StorageDriver(Remote) │ │ - CacheBackend(Memory / Redis) │ ├─────────────────────────────────────────────┤ │ 数据层 │ @@ -190,7 +191,7 @@ WebDAV 不走 `src/api/routes/**`,而是: | `src/api/routes/remote_tunnel.rs` | primary 侧远端节点 reverse tunnel 内部入口 | | `src/services/` | 业务规则集中层 | | `src/storage/connectors/` | 存储 connector:descriptor、字段、action、连接测试、上传工作流和凭据需求 | -| `src/storage/drivers/` | 本地、S3-compatible、Azure Blob、Tencent COS、OneDrive 和远端驱动 | +| `src/storage/drivers/` | 本地、S3-compatible、SFTP、Azure Blob、Tencent COS、OneDrive 和远端驱动 | | `src/storage/remote_protocol/tunnel/` | reverse tunnel 传输运行时、鉴权、注册表和流式响应 | | `src/webdav/` | WebDAV 文件系统、认证、锁与 DeltaV 支持 | | `frontend-panel/` | React 19 + Vite 前端,构建产物由后端服务 | diff --git a/developer-docs/zh-CN/storage-descriptor-normalization-contract.md b/developer-docs/zh-CN/storage-descriptor-normalization-contract.md index f87d4fea1..fe21797dd 100644 --- a/developer-docs/zh-CN/storage-descriptor-normalization-contract.md +++ b/developer-docs/zh-CN/storage-descriptor-normalization-contract.md @@ -18,6 +18,7 @@ - 管理端字段、动作、能力和 UI 辅助元数据必须优先来自后端 descriptor。前端不能用本地 `driver_type` 白名单或矩阵推断连接测试、授权、上传策略、原生处理、远端绑定、字段可见性等能力。 - `label_key`、`help_key`、`placeholder`、`required_message_key` 和类似字段只表达稳定的本地化 key 或提示参数。具体文案由前端 i18n 决定,但字段是否存在、是否必填、是否敏感由后端 descriptor 决定。 - `secret: true` 或 secret kind 表示前端必须使用敏感输入控件,后端日志和 `Debug` 输出不得打印明文。创建流程按 descriptor 的 `required` 和后端校验执行;编辑流程中,省略 secret 字段表示保留已有值,显式提供新值才替换。 +- `StorageConnectorFieldScope::PolicyOptions` 字段属于 `storage_policy.options`,前端必须按 descriptor 渲染和归一化,不允许为某个 driver 额外写本地字段矩阵。SFTP 的 `sftp_host_key_fingerprint` 就是这种字段:后端声明字段、label key、trim 行为和校验规则,前端只负责展示与提交。 - 不支持的 driver 必须由后端返回稳定错误。remote storage target 只能暴露已注册且远端 capability 声明支持的 known driver;未知 driver id 可以在 wire model 中保留,但不能被前端当成本地可配置 driver。 - descriptor 缺失、远端 capability 缺失或解析失败时,前端只能做保守兜底:隐藏高风险动作或显示不可用状态。不能在兜底路径里恢复本地 capability 矩阵。 - action descriptor 用来声明入口是否需要 saved policy、授权 credential,以及是否会修改远端状态。路由和 service 仍要做最终校验,不能只靠前端隐藏按钮。 @@ -29,6 +30,8 @@ - local remote storage target `base_path` 使用 `normalize_relative_local_path`:trim 空白,支持 `.` 当前目录归一化,拒绝空值、绝对路径、`..`、Windows prefix 和反斜杠逃逸,最终路径必须落在 `server.follower.remote_storage_target_local_root` 内。 - object-storage remote storage target `base_path` 当前按 prefix 处理:trim 空白并去掉首尾 `/`,允许空 prefix 表示 bucket/container 根。 - storage policy object-storage endpoint/bucket 使用 `normalize_s3_endpoint_and_bucket` 及 connector 错误码映射:endpoint 非空时必须是 `http://` 或 `https://` 且包含 hostname,bucket/container 不能为空。 +- storage policy SFTP endpoint 由 `parse_sftp_endpoint` 处理:允许 `sftp://host:port`、裸 `host` 和 `host:port`,默认端口 `22`;只有出现真实 `://` scheme 分隔符时才按 URL scheme 校验。路径、query、fragment 和 URL 内凭据无效,远程根目录必须走 `base_path`。 +- SFTP 主机密钥指纹存放在 `storage_policy.options.sftp_host_key_fingerprint`。未知或不匹配的 host key 必须 fail closed,并通过结构化 `SftpHostKeyRejected` 上下文暴露 actual / expected 指纹,测试不要解析错误文本。 - `max_file_size` 的 `0` 表示不声明额外限制;负数无效。上传链路仍要在使用 policy/target 时执行最终大小校验。 - 同 driver 编辑时,`access_key` / `secret_key` / `base_path` / endpoint / bucket 等可选字段省略表示保留已有值;显式提供字段则重新走对应 driver 的规范化和校验。 - 切换 remote storage target driver 时,旧 driver-specific 字段不能继承到新 driver:endpoint、bucket、access key、secret key 会重置为空;base path 回到新 driver 输入或默认根语义后再规范化。 @@ -47,5 +50,6 @@ - descriptor 必须覆盖每个内置 driver 的字段、secret 标记、action 和关键 capability。 - normalization 必须覆盖 trim、空值、逃逸路径、prefix 首尾斜杠、负数 `max_file_size`、同 driver secret preserve、显式 secret replace、driver 切换字段重置。 +- SFTP 必须覆盖裸 host、`host:port`、`sftp://host:port`、错误协议、host key 指纹格式、未知 host key 拒绝和已确认指纹通过。 - storage policy descriptor 行为改变时,跑 `cargo test --lib storage::connectors` 或更小过滤;remote storage target 归一化改变时,跑 `cargo test --lib remote_storage_target_service::tests::`。 - 改 OpenAPI schema 后必须重新导出 OpenAPI 并生成前端 SDK。本契约切片不改变 API shape。 diff --git a/developer-docs/zh-CN/testing.md b/developer-docs/zh-CN/testing.md index e2261696d..176756d7e 100644 --- a/developer-docs/zh-CN/testing.md +++ b/developer-docs/zh-CN/testing.md @@ -122,3 +122,23 @@ ASTER_TEST_DATABASE_BACKEND=mysql cargo test --test test_admin test_admin_team_c 1. 当前用例是不是走的 `common::setup()` 2. shell 里是否真的导出了 `ASTER_TEST_DATABASE_BACKEND` 3. 本机 Docker 是否可用,以及对应镜像能否拉起 + +## SFTP 集成测试 + +SFTP 驱动有单独的集成测试: + +```bash +cargo test --test test_sftp +``` + +这个测试默认会通过 `testcontainers` 启动 `atmoz/sftp` 容器,完成一次真实上传、下载、range 读取、删除和主机密钥指纹确认流程。它需要本机 Docker / 容器运行时可用。 + +如果当前环境不能跑 Docker,可以显式关闭: + +```bash +ASTER_SFTP_TEST_DOCKER=0 cargo test --test test_sftp +``` + +关闭后测试会跳过容器 round-trip。不要把这个变量作为默认 CI 行为;SFTP 是真实存储驱动,PR 改到驱动、connector、descriptor 或上传下载链路时应优先保留默认 Docker 测试。 + +`src/storage/drivers/sftp.rs` 里还有一个手动真实服务器用例,需要 `ASTER_SFTP_TEST_*` 和 `ASTER_SFTP_TEST_HOST_KEY_FINGERPRINT`。它不替代 `tests/test_sftp.rs` 的默认 Docker 覆盖,主要用于排查特定 SFTP 服务器兼容性。 diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 85ed3b4f5..4f586861b 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -458,6 +458,7 @@ function buildZhSidebar() { { text: 'Azure Blob Storage', link: '/storage/azure-blob' }, { text: '腾讯云 COS', link: '/storage/tencent-cos' }, { text: 'OneDrive', link: '/storage/onedrive' }, + { text: 'SFTP', link: '/storage/sftp' }, { text: '远程节点存储策略', link: '/storage/remote-follower' } ] }, @@ -573,6 +574,7 @@ function buildEnSidebar() { { text: 'Azure Blob Storage', link: '/en/storage/azure-blob' }, { text: 'Tencent COS', link: '/en/storage/tencent-cos' }, { text: 'OneDrive', link: '/en/storage/onedrive' }, + { text: 'SFTP', link: '/en/storage/sftp' }, { text: 'Follower Node Storage Policy', link: '/en/storage/remote-follower' } ] }, diff --git a/docs/config/storage.md b/docs/config/storage.md index a92d4c9cb..457b93337 100644 --- a/docs/config/storage.md +++ b/docs/config/storage.md @@ -26,6 +26,7 @@ | `azure_blob` | 文件存到 Azure Blob Storage container,使用 Azure Blob SDK 和 SAS URL | [Azure Blob Storage](/storage/azure-blob) | | `tencent_cos` | 文件存到腾讯云 COS;基础对象读写复用 S3 兼容能力,并额外暴露 COS 数据万象等腾讯云原生能力 | [腾讯云 COS](/storage/tencent-cos) | | `one_drive` | 文件写到 Microsoft Graph 可访问的 OneDrive、SharePoint 或 Microsoft 365 group drive | [OneDrive](/storage/onedrive) | +| `sftp` | 文件通过 AsterDrive 服务端流式读写到 SSH/SFTP 文件服务器 | [SFTP](/storage/sftp) | | `remote` | 文件通过内部远程存储协议写到另一台 AsterDrive 从节点 | [远程节点存储策略](/storage/remote-follower) | ## 存储策略 vs 策略组 @@ -46,13 +47,13 @@ | 项目 | 作用 | | --- | --- | | 名称 | 后台显示名 | -| 驱动类型 | `local`、`s3`、`azure_blob`、`tencent_cos`、`one_drive` 或 `remote` | -| 连接信息 | 本地目录 / S3 endpoint、bucket、密钥 / Azure Blob endpoint、container、账号密钥 / COS endpoint、bucket、密钥 / OneDrive Microsoft Graph 目标与授权配置 / 绑定的远程节点 | +| 驱动类型 | `local`、`s3`、`azure_blob`、`tencent_cos`、`one_drive`、`sftp` 或 `remote` | +| 连接信息 | 本地目录 / S3 endpoint、bucket、密钥 / Azure Blob endpoint、container、账号密钥 / COS endpoint、bucket、密钥 / OneDrive Microsoft Graph 目标与授权配置 / SFTP endpoint、SSH 凭据、主机密钥指纹 / 绑定的远程节点 | | 基础路径 | 写入该策略时使用的目录、prefix 或远程落点相对路径 | | 单文件大小上限 | 允许上传的最大文件;`0` = 不限 | | 分片大小 | 大文件上传时每一片的大小 | | 默认策略 | 新建默认组或默认分流规则会优先使用 | -| 附加选项 | 本地内容去重、S3 / Azure Blob / COS 上传下载方式、S3 path-style 访问、OneDrive 目标 drive 定位、远程上传下载方式、存储原生处理开关等 | +| 附加选项 | 本地内容去重、S3 / Azure Blob / COS 上传下载方式、S3 path-style 访问、OneDrive 目标 drive 定位、SFTP 主机密钥指纹、远程上传下载方式、存储原生处理开关等 | 后台的存储策略表单不是靠前端硬编码各个厂商字段。AsterDrive 会从后端的 `StorageConnector` descriptor 读取当前 driver 支持的字段、能力、上传工作流和管理动作,所以新增或调整存储后端时,管理界面会尽量跟着后端能力显示。 @@ -119,6 +120,14 @@ AsterDrive 会缓存缩略图和媒体信息等派生结果,避免每次查看 OneDrive 策略需要 Microsoft 应用注册和管理员 delegated OAuth 授权。先保存策略和 Microsoft Graph 应用凭据,再发起授权;授权请求不会携带未保存的 Client ID / Secret 草稿。目标 drive 可以在授权后自动解析,也可以通过 Drive ID、SharePoint site ID 或 group ID 指定。完整流程见 [OneDrive 存储策略教程](/storage/onedrive)。 +### `sftp` + +适合把文件写到 SSH/SFTP 文件服务器、NAS 或传统服务器目录。 + +SFTP 策略使用服务端流式上传和下载,浏览器不会直接连接 SFTP 服务器。Endpoint 可以写成 `sftp://host:port`、`host` 或 `host:port`;没有端口时默认 `22`,远程根目录放在基础路径里。SSH 用户名 / 密码在 API 字段里仍复用 `access_key` / `secret_key`,但后台表单会显示为 SSH 凭据。 + +SFTP 默认拒绝未知主机密钥。第一次连接测试会返回服务器实际 `SHA256:...` 指纹,管理员确认后填入 `storage_policy.options.sftp_host_key_fingerprint`,后续连接必须匹配该指纹。完整流程见 [SFTP 存储策略教程](/storage/sftp)。 + ### `tencent_cos` 适合文件放到腾讯云 COS,并且希望按策略启用腾讯云原生能力。 @@ -141,6 +150,7 @@ OneDrive 策略需要 Microsoft 应用注册和管理员 delegated OAuth 授权 | `s3` / `tencent_cos` | 返回“不支持”;标准 S3 兼容 API 没有统一可靠的 bucket 剩余容量接口 | | `azure_blob` | 返回“不支持”;Blob data API 不提供统一的 storage account 容量观测 | | `one_drive` | 读取 Microsoft Graph drive quota;如果 Graph 未返回 quota,则显示“不可用” | +| `sftp` | 返回“不支持”;SFTP 协议没有统一可靠的远端文件系统容量接口 | | `remote` | 通过内部远程存储协议询问 follower 的实际接收落点;如果 follower 接收落点是 local,通常能看到文件系统容量;如果接收落点是 S3,则同样显示“不支持” | 迁移数据时,预检查会用目标策略的可用容量和“预计需要复制的 blob 字节数”比较,而不是简单使用源策略总大小。目标策略已经有的 content SHA-256 blob 会被视为可复用,不再计入预计复制量。 @@ -174,6 +184,7 @@ OneDrive 策略需要 Microsoft 应用注册和管理员 delegated OAuth 授权 - 本地目录 - Bucket - Endpoint +- SFTP 基础路径 - 绑定的远程节点 旧文件按原位置读取,直接改位置 = 已有文件全部找不到。 diff --git a/docs/en/config/storage.md b/docs/en/config/storage.md index 697a217fa..47ccc9558 100644 --- a/docs/en/config/storage.md +++ b/docs/en/config/storage.md @@ -26,6 +26,7 @@ If you change nothing, new users are automatically bound to the default policy g | `azure_blob` | Files are stored in an Azure Blob Storage container using the Azure Blob SDK and SAS URLs | [Azure Blob Storage](/en/storage/azure-blob) | | `tencent_cos` | Files are stored in Tencent COS; base object operations reuse S3-compatible behavior, with additional Tencent-native capabilities such as COS CI (Cloud Infinite / 数据万象). See the Tencent COS tutorial for what COS CI provides and when it may be billed. | [Tencent COS](/en/storage/tencent-cos) | | `one_drive` | Files are written to Microsoft Graph-accessible OneDrive, SharePoint, or Microsoft 365 group drives | [OneDrive](/en/storage/onedrive) | +| `sftp` | Files are streamed by the AsterDrive server to an SSH/SFTP file server | [SFTP](/en/storage/sftp) | | `remote` | Files are written to another AsterDrive follower node through the internal remote storage protocol | [Follower Node Storage Policy](/en/storage/remote-follower) | ## Storage Policies vs Policy Groups @@ -46,13 +47,13 @@ If you are migrating existing data, do not directly change the old policy path, | Item | Purpose | | --- | --- | | Name | Display name in the admin console | -| Driver type | `local`, `s3`, `azure_blob`, `tencent_cos`, `one_drive`, or `remote` | -| Connection information | Local directory / S3 endpoint, bucket, secrets / Azure Blob endpoint, container, account keys / COS endpoint, bucket, secrets / OneDrive Microsoft Graph target and authorization settings / bound follower node | +| Driver type | `local`, `s3`, `azure_blob`, `tencent_cos`, `one_drive`, `sftp`, or `remote` | +| Connection information | Local directory / S3 endpoint, bucket, secrets / Azure Blob endpoint, container, account keys / COS endpoint, bucket, secrets / OneDrive Microsoft Graph target and authorization settings / SFTP endpoint, SSH credentials, host key fingerprint / bound follower node | | Base path | Directory, prefix, or remote-target relative path used when writing through this policy | | Single-file size limit | Maximum upload size. `0` = unlimited. | | Chunk size | Size of each chunk for large-file uploads | | Default policy | Preferred by newly created default groups or default routing rules | -| Extra options | Local content deduplication, S3 / Azure Blob / COS upload and download modes, S3 path-style access, OneDrive target-drive location, remote upload and download modes, storage-native processing, and so on | +| Extra options | Local content deduplication, S3 / Azure Blob / COS upload and download modes, S3 path-style access, OneDrive target-drive location, SFTP host key fingerprint, remote upload and download modes, storage-native processing, and so on | The storage policy form is not driven only by frontend hardcoded provider fields. AsterDrive reads each driver's `StorageConnector` descriptor from the backend, including fields, capabilities, upload workflows, and management actions. When a storage backend grows or changes, the admin UI can follow the backend descriptor instead of re-creating a parallel capability table. @@ -119,6 +120,14 @@ Suitable when files should be written to Microsoft Graph-accessible OneDrive, Sh OneDrive policies require a Microsoft app registration and administrator delegated OAuth authorization. Save the policy and Microsoft Graph application credentials before starting authorization; the authorization request does not carry unsaved Client ID / Secret drafts. The target drive can be resolved automatically after authorization, or specified with a Drive ID, SharePoint site ID, or group ID. See the [OneDrive storage policy tutorial](/en/storage/onedrive) for the full flow. +### `sftp` + +Suitable when files should be written to an SSH/SFTP file server, NAS, or traditional server directory. + +SFTP policies use server-side streaming for both uploads and downloads; browsers never connect directly to the SFTP server. Endpoint can be `sftp://host:port`, `host`, or `host:port`; the default port is `22`, and the remote root belongs in Base path. SSH username / password still use the API fields `access_key` / `secret_key`, but the admin form labels them as SSH credentials. + +SFTP rejects unknown host keys by default. The first connection test reports the server's actual `SHA256:...` fingerprint; after the administrator confirms it, save it as `storage_policy.options.sftp_host_key_fingerprint`. Later connections must match that fingerprint. See the [SFTP storage policy tutorial](/en/storage/sftp) for the full flow. + ### `tencent_cos` Suitable when files are stored in Tencent COS and you want to enable Tencent-native capabilities per policy. @@ -141,6 +150,7 @@ The storage policy edit dialog shows current capacity observation: | `s3` / `tencent_cos` | Shows unsupported; the standard S3-compatible API does not expose a unified, reliable bucket free-capacity interface | | `azure_blob` | Shows unsupported; the Blob data API does not expose unified storage account capacity observation | | `one_drive` | Reads Microsoft Graph drive quota; if Graph does not return quota data, the result is shown as unavailable | +| `sftp` | Shows unsupported; SFTP has no unified reliable remote filesystem capacity interface | | `remote` | Asks the follower's real ingress target through the internal remote storage protocol. If the follower ingress target is local, filesystem capacity is usually available. If the ingress target is S3, it is also shown as unsupported. | During data migration, preflight compares the target policy's available capacity with the estimated bytes that still need to be copied. It does not simply use the source policy's total size. Content SHA-256 blobs that already exist in the target policy are treated as reusable and are excluded from the estimated copy size. @@ -174,6 +184,7 @@ If an opaque key already exists in the target policy, the migration does not ove - Local directory - Bucket - Endpoint +- SFTP base path - Bound follower node Old files are read from their original locations. Changing the location directly means existing files cannot be found. diff --git a/docs/en/features/upload-storage.md b/docs/en/features/upload-storage.md index d115045d5..2f17a09e8 100644 --- a/docs/en/features/upload-storage.md +++ b/docs/en/features/upload-storage.md @@ -1,5 +1,5 @@ --- -description: AsterDrive uploads and storage feature map covering upload modes, blobs, quota, storage policies, policy groups, local storage, S3, Azure Blob, Tencent COS, OneDrive, follower storage, and routing behavior. +description: AsterDrive uploads and storage feature map covering upload modes, blobs, quota, storage policies, policy groups, local storage, S3, Azure Blob, Tencent COS, OneDrive, SFTP, follower storage, and routing behavior. --- # Uploads and Storage @@ -15,7 +15,8 @@ Uploads and storage turn a file sent by a browser or client into a database file | Object-storage presigned upload | Browser PUTs directly to S3-compatible storage, Azure Blob SAS URLs, or Tencent COS; server verifies and finalizes | [Uploads and Large Files](/en/guide/upload-modes), [Storage Policies](/en/config/storage) | | Object-storage multipart | Browser uploads parts in batches; server completes and validates content | [Uploads and Large Files](/en/guide/upload-modes), [S3 / MinIO / R2](/en/storage/s3-minio-r2), [Azure Blob Storage](/en/storage/azure-blob), [Tencent COS](/en/storage/tencent-cos) | | Microsoft Graph storage | Writes files to OneDrive, SharePoint site drives, or Microsoft 365 group drives after administrator authorization | [OneDrive](/en/storage/onedrive), [Storage Policies](/en/config/storage) | -| Storage policies | Decide whether files land on local, s3, azure_blob, tencent_cos, one_drive, or remote | [Storage Policies](/en/config/storage) | +| SFTP storage | Server-side streaming reads and writes to an SSH/SFTP file server | [SFTP](/en/storage/sftp), [Storage Policies](/en/config/storage) | +| Storage policies | Decide whether files land on local, s3, sftp, azure_blob, tencent_cos, one_drive, or remote | [Storage Policies](/en/config/storage) | | Policy groups | Route by user, team, and file size to storage policies | [Storage Policies](/en/config/storage) | | Follower storage | Primary writes objects to a follower; the follower stores them locally or in S3 | [Follower Node Enrollment](/en/guide/remote-nodes), [Follower Node Storage Policy](/en/storage/remote-follower) | @@ -26,7 +27,7 @@ Uploads and storage turn a file sent by a browser or client into a database file | `upload_service` | Upload sessions, chunks, progress, status transitions | | `workspace_storage_core` | Blob dedupe, file records, quota, policy choice, finalization | | `policy_service` | Storage policies, policy groups, rules | -| `storage::traits`, `storage::drivers`, `storage::connectors` | `StorageDriver` and `StorageConnector` abstractions, local, S3-compatible, Azure Blob, Tencent COS, OneDrive, and remote drivers | +| `storage::traits`, `storage::drivers`, `storage::connectors` | `StorageDriver` and `StorageConnector` abstractions, local, S3-compatible, SFTP, Azure Blob, Tencent COS, OneDrive, and remote drivers | | `storage::remote_protocol` | Primary/follower internal remote storage protocol | | `managed_follower_service`, `managed_ingress_profile_service` | Follower nodes and ingress targets | | `task_service::storage_migration` | Storage migration tasks | @@ -37,6 +38,7 @@ Uploads and storage turn a file sent by a browser or client into a database file - Filename uniqueness, blob ref counts, and upload-session state transitions must use repository-level atomic helpers. - Whether `presigned` works depends on browser reachability to object storage or follower `base_url`, not only primary reachability. - A successful admin connection test only proves that the AsterDrive server can reach the backend. `presigned` upload / download still needs browser network and CORS checks for object storage, Azure Blob, or follower endpoints. +- SFTP is a server-side streaming backend. Its operational checks are Endpoint, SSH credentials, base path, and host key fingerprint; browser CORS and presigned URLs are not involved. - Follower `reverse_tunnel` is suitable for `relay_stream`, not browser-direct `presigned`. ## Troubleshooting Direction diff --git a/docs/en/guide/upload-modes.md b/docs/en/guide/upload-modes.md index 359be7b8c..3bb531ec0 100644 --- a/docs/en/guide/upload-modes.md +++ b/docs/en/guide/upload-modes.md @@ -26,6 +26,7 @@ flowchart TD Policy --> Local["local: write to local storage"] Policy --> S3Relay["s3 / tencent_cos + relay_stream: relay through the server to object storage"] Policy --> S3Presigned["s3 / tencent_cos + presigned: browser uploads directly to object storage"] + Policy --> SftpRelay["sftp: server streams writes to SFTP"] Policy --> RemoteRelay["remote + relay_stream: primary node relays to a follower node"] Policy --> RemotePresigned["remote + presigned: browser connects directly to follower base_url"] ``` @@ -44,6 +45,7 @@ If you are responsible for deployment, confirm these items first: - Whether the service's local temporary directory has enough space - Whether the reverse proxy upload size and timeouts are sufficient - If you use direct-to-object-storage upload, whether the required browser-upload CORS rules are configured +- If you use SFTP, whether the SSH host key fingerprint has been confirmed and saved - If you use follower nodes, whether the follower already has a default ingress target - If you use remote `presigned`, whether the remote node uses direct transport, whether the browser can reach the follower `base_url`, and whether the follower exposes the required CORS response headers @@ -77,6 +79,17 @@ If you use this path, confirm at least: - The upload site's origin is allowed - `ExposeHeaders` includes `ETag` +## If You Use SFTP + +SFTP uploads and downloads are relayed by the AsterDrive server. Browsers do not connect directly to the SFTP server, and there are no presigned URL or CORS settings. + +Before launch, confirm: + +- The AsterDrive server can reach the SFTP endpoint and port +- SSH username / password credentials work +- The base path is writable +- The SSH host key fingerprint has been confirmed through a trusted channel and saved + ## When Uploads Fail, Check in This Order 1. Whether the current workspace is correct @@ -84,8 +97,9 @@ If you use this path, confirm at least: 3. Whether the matched storage policy's single-file size limit is sufficient 4. Whether the reverse proxy request body size and timeout are sufficient 5. If direct-to-object-storage upload is used, whether CORS is correct -6. If a follower node is used, whether the node is enabled, the current transport mode passes connection testing, protocol capabilities are compatible, and the default ingress target has been applied -7. Whether the user or team quota is already full +6. If SFTP is used, whether Endpoint, SSH credentials, base path, and host key fingerprint are correct +7. If a follower node is used, whether the node is enabled, the current transport mode passes connection testing, protocol capabilities are compatible, and the default ingress target has been applied +8. Whether the user or team quota is already full ## When to Change Configuration @@ -96,4 +110,5 @@ If users often upload large files, you usually only need to check these places: - Available space in the server's local temporary directory - Reverse proxy upload size and timeout - Browser direct-upload allow rules on object storage +- SFTP endpoint, base path, and SSH host key fingerprint - Follower node transport mode, protocol capabilities, default ingress target, and network reachability; if remote `presigned` is used, also confirm browsers can reach the follower `base_url` diff --git a/docs/en/index.md b/docs/en/index.md index 8992e3bdf..3cf8b7b3f 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -5,7 +5,7 @@ description: The official AsterDrive documentation home, organized by quick star hero: name: AsterDrive text: Official Documentation Center - tagline: Built with Rust + React. Start with single-node deployment, then add team spaces, S3, Azure Blob, OneDrive, WebDAV, WOPI, and follower nodes as needed. + tagline: Built with Rust + React. Start with single-node deployment, then add team spaces, S3, Azure Blob, OneDrive, SFTP, WebDAV, WOPI, and follower nodes as needed. actions: - theme: brand text: Quick Start @@ -37,7 +37,7 @@ features: ## First, Know What It Is -AsterDrive is a lightweight self-hosted cloud drive built with Rust and React. You can start with the default single-node deployment using SQLite and local storage, then connect PostgreSQL / MySQL, S3-compatible object storage, Azure Blob Storage, Tencent COS, Microsoft Graph-backed OneDrive / SharePoint drives, team spaces, WebDAV, WOPI online preview and editing, and follower node storage as needed. +AsterDrive is a lightweight self-hosted cloud drive built with Rust and React. You can start with the default single-node deployment using SQLite and local storage, then connect PostgreSQL / MySQL, S3-compatible object storage, Azure Blob Storage, Tencent COS, Microsoft Graph-backed OneDrive / SharePoint drives, SFTP file servers, team spaces, WebDAV, WOPI online preview and editing, and follower node storage as needed. It is not a full collaboration suite or a multi-primary cluster system. The current focus is making file management, sharing, uploads, previews, storage policies, and routine operations clear for individuals and small teams. @@ -57,7 +57,7 @@ Start from [User Guides](/en/guide/). Regular users should first read the [User Read [Admin Console](/en/guide/admin-console) first, then [Configuration Overview](/en/config/). AsterDrive configuration is split into startup configuration, admin-console runtime settings, storage policies, policy groups, storage policy backends, and the external network environment. It is much clearer when viewed by layer. -If you are connecting S3 / MinIO / R2 / Azure Blob Storage / Tencent COS / OneDrive, see [Storage Policy Backends](/en/storage/). +If you are connecting S3 / MinIO / R2 / Azure Blob Storage / Tencent COS / OneDrive / SFTP, see [Storage Policy Backends](/en/storage/). ### I Want to Find Documentation by Feature Module diff --git a/docs/en/storage/index.md b/docs/en/storage/index.md index 5651d1721..7339271d1 100644 --- a/docs/en/storage/index.md +++ b/docs/en/storage/index.md @@ -25,6 +25,7 @@ If you have already decided which backend to connect, use the tutorials here. | Azure Blob Storage | Azure Storage accounts, Blob containers, Azure-managed object storage | [Azure Blob Storage](/en/storage/azure-blob) | | Tencent COS | Tencent object storage, COS CI, per-policy native processing | [Tencent COS](/en/storage/tencent-cos) | | OneDrive | Microsoft 365, OneDrive, SharePoint / group drives, Microsoft Graph authorization | [OneDrive](/en/storage/onedrive) | +| SFTP | SSH/SFTP file servers, NAS, traditional server directories, server-side streaming | [SFTP](/en/storage/sftp) | | Follower node | The control plane stays on the primary node, while real objects are written to another AsterDrive node | [Follower Node Storage Policy](/en/storage/remote-follower) | ## General Configuration Flow @@ -51,5 +52,5 @@ Recommended flow: 5. After confirming there are no issues, move real users or teams to the new policy group ::: warning Do not directly change the real destination for policies that already have files -The `local` directory, S3 bucket / endpoint / prefix, Azure Blob endpoint / container / base path, OneDrive drive / root item / site or group identifiers, and follower node binding determine where old files are located. If you change them directly, old files may no longer be found. +The `local` directory, S3 bucket / endpoint / prefix, Azure Blob endpoint / container / base path, OneDrive drive / root item / site or group identifiers, SFTP endpoint / base path, and follower node binding determine where old files are located. If you change them directly, old files may no longer be found. ::: diff --git a/docs/en/storage/sftp.md b/docs/en/storage/sftp.md new file mode 100644 index 000000000..5afe2b39b --- /dev/null +++ b/docs/en/storage/sftp.md @@ -0,0 +1,169 @@ +--- +description: AsterDrive SFTP storage policy tutorial covering endpoints, SSH credentials, remote roots, host key fingerprint confirmation, connection tests, and rollout validation. +--- + +# SFTP Storage Policy Tutorial + +::: tip What this page covers +This page explains how to write AsterDrive files to an SFTP server: prepare an SSH account, create an `sftp` storage policy, confirm the SSH host key fingerprint, configure policy group rules, and verify uploads and downloads. +::: + +## When to Use It + +SFTP is suitable for these scenarios: + +- You already have a file server managed through SSH/SFTP +- The target storage is a NAS, traditional server directory, or storage that is only exposed through SFTP +- You do not want browsers to connect directly to the storage backend; uploads and downloads can be relayed by the AsterDrive server +- You want to route some users, teams, or file sizes to an SFTP backend + +If you need browser direct upload, presigned URLs, or object-storage multipart upload, prefer [S3 / MinIO / R2](/en/storage/s3-minio-r2), [Azure Blob Storage](/en/storage/azure-blob), or [Tencent COS](/en/storage/tencent-cos). SFTP is currently a server-side streaming backend and does not provide object-storage-style direct upload. + +## First, Separate the Layers You Need to Configure + +```mermaid +flowchart TD + Server["SFTP server"] --> Account["SSH account / remote directory / host key fingerprint"] + Account --> Policy["AsterDrive sftp storage policy"] + Policy --> Rule["Policy group rule"] + Rule --> Binding["User or team bound to the policy group"] +``` + +Creating only an SFTP storage policy is not enough. When users or teams upload files, they first match a policy group, then the policy group rule assigns the upload to a storage policy. + +## Entries Used in This Page + +| What you want to do | Entry | +| --- | --- | +| Create an SFTP policy | `Admin -> Storage Policies -> New Policy` | +| Test the SFTP connection | `Admin -> Storage Policies -> Test Connection` | +| Create routing rules | `Admin -> Policy Groups` | +| Bind a policy group to a user | `Admin -> Users -> User Details` | +| Bind a policy group to a team | `Admin -> Teams -> Team Details` | + +## 1. Prepare the SSH Account and Remote Directory + +Prepare a dedicated account and directory on the SFTP server, for example: + +```text +/srv/asterdrive +``` + +Grant this account only the permissions needed for the target directory. AsterDrive needs at least: + +- Create directories +- Write files +- Read files +- Delete files +- Read file metadata + +Do not give AsterDrive an administrator account, a root account, or a directory shared with unrelated applications. + +::: warning Do not manually move objects in the remote directory +The AsterDrive database records object paths. Manually moving, renaming, or deleting objects in the SFTP directory will make database file records inconsistent with the real objects. +::: + +## 2. Create an SFTP Storage Policy + +Open: + +```text +Admin -> Storage Policies -> New Policy +``` + +Choose the driver type: + +```text +sftp +``` + +Fill in the connection fields: + +| Field | Example | Notes | +| --- | --- | --- | +| Endpoint | `sftp://sftp.example.com:22` | SFTP server address | +| SSH Username | `asterdrive` | The internal API field is still `access_key`, but the admin UI labels it as SSH Username | +| SSH Password | `********` | The internal API field is still `secret_key`, but the admin UI labels it as SSH Password | +| Base path | `/srv/asterdrive` | Remote root used when AsterDrive writes objects | +| SSH Host Key Fingerprint | `SHA256:...` | Fill this after first-connection confirmation | + +Endpoint supports these forms: + +```text +sftp://sftp.example.com:22 +sftp.example.com +sftp.example.com:2222 +``` + +When the port is omitted, AsterDrive uses `22`. If a scheme is present, only `sftp://` is supported. Do not put usernames, passwords, paths, query strings, or fragments in the Endpoint; put the remote root in Base path and credentials in their own fields. + +## 3. Confirm the SSH Host Key Fingerprint + +SFTP policies reject unknown host keys by default. During the first connection test, if `SSH Host Key Fingerprint` has not been saved yet, the test fails and reports the server's actual `SHA256:...` fingerprint in the diagnostic message. + +Use this flow: + +1. Fill Endpoint, SSH Username, SSH Password, and Base path +2. Run the connection test +3. Read the returned `SHA256:...` fingerprint from the failure diagnostic +4. Confirm through a trusted channel that this fingerprint belongs to the intended SFTP server +5. Fill `SSH Host Key Fingerprint` with the confirmed value +6. Run the connection test again, then save after it succeeds + +::: warning Do not bypass host key confirmation +Without SSH host key verification, an attacker could impersonate the SFTP server and capture the password. AsterDrive stores the confirmed fingerprint in `storage_policy.options.sftp_host_key_fingerprint`, and later connections must match it. If the server is reinstalled or rotates its SSH host key, confirm and update the fingerprint again. +::: + +## 4. Troubleshoot Failed Connection Tests + +When a connection test fails, the admin console prefers the standard error response's `error.diagnostic.message`. For SFTP, check in this order: + +1. Whether the AsterDrive server can reach the Endpoint and port +2. Whether the Endpoint contains only host and optional port +3. Whether the SSH username and password are correct +4. Whether the account is allowed to use the SFTP subsystem +5. Whether the base path exists, or the account can create it +6. Whether `SSH Host Key Fingerprint` matches the server's current host key +7. Whether the server firewall, fail2ban, or connection limits reject the login + +If the error says the host key does not match, do not overwrite the old fingerprint immediately. First confirm whether the server really rotated its host key. + +## 5. Create a Test Policy Group and Verify + +Do not directly modify the default policy group at the beginning. Create a test policy group, bind one test user or team, and run through: + +1. Upload a small file +2. Upload a file larger than ordinary form-upload size +3. Download a file +4. Preview or publicly share a normal file +5. Delete and restore a file +6. Confirm object files appear under the SFTP server target directory + +After confirming there are no issues, move real users or teams to the policy group that contains the SFTP policy. + +## Migration and Change Boundaries + +::: warning Do not directly change these fields on an SFTP policy that already has files + +- Endpoint +- Base path +- SSH Username + +Old files are read from their original locations. Changing these directly may make existing files unreachable. A safer path is to create a new SFTP policy and use `Admin -> Storage Policies -> Migrate Data`. +::: + +You can rotate the SSH password by editing the policy. When editing a saved policy, leaving the password field blank preserves the existing credential. + +## Current Capability Boundaries + +| Capability | Current behavior | +| --- | --- | +| Upload | AsterDrive server streams writes to SFTP | +| Download | AsterDrive server reads from SFTP and returns data to the browser | +| Browser direct upload | Not supported | +| Object-storage multipart | Not supported | +| Presigned URL | Not supported | +| Capacity observation | No unified capacity query | +| Range reads | Efficient range reads are supported | + +SFTP connection pooling is tracked separately; the current implementation prioritizes host key verification and correct basic read/write behavior. diff --git a/docs/features/upload-storage.md b/docs/features/upload-storage.md index 31688488e..6e76bdadc 100644 --- a/docs/features/upload-storage.md +++ b/docs/features/upload-storage.md @@ -1,5 +1,5 @@ --- -description: AsterDrive 上传与存储功能地图,覆盖上传模式、Blob、配额、存储策略、策略组、本地存储、S3、Azure Blob、腾讯云 COS、OneDrive 和远程节点。 +description: AsterDrive 上传与存储功能地图,覆盖上传模式、Blob、配额、存储策略、策略组、本地存储、S3、Azure Blob、腾讯云 COS、OneDrive、SFTP 和远程节点。 --- # 上传与存储 @@ -15,7 +15,8 @@ description: AsterDrive 上传与存储功能地图,覆盖上传模式、Blob | 对象存储预签名上传 | 浏览器直接 PUT 到 S3-compatible、Azure Blob SAS URL 或腾讯云 COS,服务端最终校验并落账 | [上传与大文件](/guide/upload-modes)、[存储策略](/config/storage) | | 对象存储 multipart | 浏览器分批上传 part,服务端 complete 后校验内容 | [上传与大文件](/guide/upload-modes)、[S3 / MinIO / R2](/storage/s3-minio-r2)、[Azure Blob Storage](/storage/azure-blob)、[腾讯云 COS](/storage/tencent-cos) | | Microsoft Graph 存储 | 通过管理员授权把文件写入 OneDrive、SharePoint site drive 或 Microsoft 365 group drive | [OneDrive](/storage/onedrive)、[存储策略](/config/storage) | -| 存储策略 | 决定文件最终写到 local、s3、azure_blob、tencent_cos、one_drive 或 remote | [存储策略](/config/storage) | +| SFTP 存储 | 通过服务端流式读写把文件写到 SSH/SFTP 文件服务器 | [SFTP](/storage/sftp)、[存储策略](/config/storage) | +| 存储策略 | 决定文件最终写到 local、s3、sftp、azure_blob、tencent_cos、one_drive 或 remote | [存储策略](/config/storage) | | 策略组 | 按用户、团队和文件大小分流到不同存储策略 | [存储策略](/config/storage) | | 远程节点存储 | primary 把对象写到 follower,再由 follower 写本地或 S3 | [远程节点接入](/guide/remote-nodes)、[远程节点存储策略](/storage/remote-follower) | @@ -26,7 +27,7 @@ description: AsterDrive 上传与存储功能地图,覆盖上传模式、Blob | `upload_service` | 上传会话、分片、进度、状态转换 | | `workspace_storage_core` | Blob 去重、文件记录、配额、策略选择和最终落账 | | `policy_service` | 存储策略、策略组和规则 | -| `storage::traits`、`storage::drivers`、`storage::connectors` | `StorageDriver` 和 `StorageConnector` 抽象、本地、S3-compatible、Azure Blob、Tencent COS、OneDrive 和 remote 驱动 | +| `storage::traits`、`storage::drivers`、`storage::connectors` | `StorageDriver` 和 `StorageConnector` 抽象、本地、S3-compatible、SFTP、Azure Blob、Tencent COS、OneDrive 和 remote 驱动 | | `storage::remote_protocol` | primary/follower 内部远程存储协议 | | `managed_follower_service`、`managed_ingress_profile_service` | 远程节点和接收落点 | | `task_service::storage_migration` | 存储迁移任务 | @@ -37,6 +38,7 @@ description: AsterDrive 上传与存储功能地图,覆盖上传模式、Blob - 文件名唯一性、Blob 引用计数和 session 状态转换都必须用 repo 层原子辅助函数。 - `presigned` 能否工作取决于浏览器能否访问对象存储或 follower `base_url`,不是只看 primary 能不能连通。 - 后台连接测试只证明 AsterDrive 服务端能连到目标后端;`presigned` 上传 / 下载还要另外确认浏览器到对象存储、Azure Blob 或 follower 的网络和 CORS。 +- SFTP 是服务端流式后端,重点是 Endpoint、SSH 凭据、基础路径和主机密钥指纹,不涉及浏览器 CORS 或预签名 URL。 - 远程节点 `reverse_tunnel` 适合 `relay_stream`,不适合浏览器直连的 `presigned`。 ## 排障方向 diff --git a/docs/guide/upload-modes.md b/docs/guide/upload-modes.md index 2e3bd0464..312cd9fd5 100644 --- a/docs/guide/upload-modes.md +++ b/docs/guide/upload-modes.md @@ -26,6 +26,7 @@ flowchart TD Policy --> Local["local:写入本地存储"] Policy --> S3Relay["s3 / tencent_cos + relay_stream:服务端转发到对象存储"] Policy --> S3Presigned["s3 / tencent_cos + presigned:浏览器直传对象存储"] + Policy --> SftpRelay["sftp:服务端流式写入 SFTP"] Policy --> RemoteRelay["remote + relay_stream:主控节点转发到从节点"] Policy --> RemotePresigned["remote + presigned:浏览器直连从节点 base_url"] ``` @@ -44,6 +45,7 @@ flowchart TD - 服务本地临时目录有没有足够空间 - 反向代理的上传大小和超时是否足够 - 如果要直传对象存储,浏览器上传所需的 CORS 是否已经配好 +- 如果用 SFTP,SSH 主机密钥指纹是否已经确认并保存 - 如果用远程节点,follower 是否已经有默认接收落点 - 如果用远程节点 `presigned`,远程节点是否使用直连传输、浏览器是否能访问 follower 的 `base_url`,并且 follower 对外暴露了所需的 CORS 响应头 @@ -77,6 +79,17 @@ flowchart TD - 允许上传站点的来源 - `ExposeHeaders` 里包含 `ETag` +## 如果你使用 SFTP + +SFTP 上传和下载都由 AsterDrive 服务端中继。浏览器不会直接连 SFTP 服务器,也没有预签名 URL 或 CORS 配置项。 + +上线前重点确认: + +- AsterDrive 服务器能访问 SFTP endpoint 和端口 +- SSH 用户名 / 密码可用 +- 基础路径可写 +- SSH 主机密钥指纹已经通过可信渠道确认并保存 + ## 上传失败时,先按这个顺序查 1. 当前工作空间是不是切对了 @@ -84,8 +97,9 @@ flowchart TD 3. 命中的存储策略单文件大小上限是否够 4. 反向代理的请求体大小和超时时间是否够 5. 如果走对象存储直传,CORS 配置是否正确 -6. 如果走远程节点,节点是否启用、当前传输方式是否测试通过、协议能力是否兼容、默认接收落点是否已应用 -7. 用户或团队配额是不是已经满了 +6. 如果走 SFTP,Endpoint、SSH 凭据、基础路径和主机密钥指纹是否正确 +7. 如果走远程节点,节点是否启用、当前传输方式是否测试通过、协议能力是否兼容、默认接收落点是否已应用 +8. 用户或团队配额是不是已经满了 ## 什么时候应该改配置 @@ -96,4 +110,5 @@ flowchart TD - 服务器本地临时目录的可用空间 - 反向代理的上传大小和超时 - 对象存储的浏览器直传放行规则 +- SFTP 的 endpoint、基础路径和 SSH 主机密钥指纹 - 远程节点的传输方式、协议能力、默认接收落点和网络可达性;如果使用远程 `presigned`,再确认浏览器能访问 follower 的 `base_url` diff --git a/docs/index.md b/docs/index.md index 727444b7b..64e6335dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,7 +5,7 @@ description: AsterDrive 官方文档首页,按快速开始、使用指南、 hero: name: AsterDrive text: 官方文档中心 - tagline: Rust + React 构建,从单机部署开始,按需要接入团队空间、S3、Azure Blob、OneDrive、WebDAV、WOPI 和远程从节点。 + tagline: Rust + React 构建,从单机部署开始,按需要接入团队空间、S3、Azure Blob、OneDrive、SFTP、WebDAV、WOPI 和远程从节点。 actions: - theme: brand text: 快速开始 @@ -37,7 +37,7 @@ features: ## 先认识它 -AsterDrive 是一个基于 Rust 和 React 的轻量自托管云盘。你可以从默认 SQLite + 本地存储的单机部署开始,之后按需要接入 PostgreSQL / MySQL、S3 兼容对象存储、Azure Blob Storage、腾讯云 COS、Microsoft Graph 支持的 OneDrive / SharePoint drive、团队空间、WebDAV、WOPI 在线预览编辑,以及远程从节点存储。 +AsterDrive 是一个基于 Rust 和 React 的轻量自托管云盘。你可以从默认 SQLite + 本地存储的单机部署开始,之后按需要接入 PostgreSQL / MySQL、S3 兼容对象存储、Azure Blob Storage、腾讯云 COS、Microsoft Graph 支持的 OneDrive / SharePoint drive、SFTP 文件服务器、团队空间、WebDAV、WOPI 在线预览编辑,以及远程从节点存储。 它不是完整协作套件或多主集群系统,当前重点是把个人和小团队的文件管理、分享、上传、预览、存储策略和日常运维路径做清楚。 diff --git a/docs/storage/index.md b/docs/storage/index.md index f9160a984..5afcfd128 100644 --- a/docs/storage/index.md +++ b/docs/storage/index.md @@ -25,6 +25,7 @@ AsterDrive 里有两层概念: | Azure Blob Storage | Azure Storage account、Blob container、Azure 托管对象存储 | [Azure Blob Storage](/storage/azure-blob) | | 腾讯云 COS | 腾讯云对象存储、COS 数据万象、按策略启用原生处理 | [腾讯云 COS](/storage/tencent-cos) | | OneDrive | Microsoft 365、OneDrive、SharePoint / group drive、Microsoft Graph 授权 | [OneDrive](/storage/onedrive) | +| SFTP | SSH/SFTP 文件服务器、NAS、传统服务器目录、服务端流式读写 | [SFTP](/storage/sftp) | | 远程节点 | 控制面在主控,真实对象写到另一台 AsterDrive | [远程节点存储策略](/storage/remote-follower) | ## 通用配置流程 @@ -51,5 +52,5 @@ flowchart TD 5. 确认没有问题后,再把真实用户或团队迁到新策略组 ::: warning 已写入文件的策略,不要直接改真实落点 -`local` 的目录、S3 的 bucket / endpoint / prefix、Azure Blob 的 endpoint / container / 基础路径、OneDrive 的 drive / root item / site 或 group 定位字段、远程节点绑定,这些字段决定旧文件在哪里。直接改掉,旧文件可能会找不到。 +`local` 的目录、S3 的 bucket / endpoint / prefix、Azure Blob 的 endpoint / container / 基础路径、OneDrive 的 drive / root item / site 或 group 定位字段、SFTP 的 endpoint / 基础路径、远程节点绑定,这些字段决定旧文件在哪里。直接改掉,旧文件可能会找不到。 ::: diff --git a/docs/storage/sftp.md b/docs/storage/sftp.md new file mode 100644 index 000000000..f545bf40a --- /dev/null +++ b/docs/storage/sftp.md @@ -0,0 +1,169 @@ +--- +description: AsterDrive SFTP 存储策略教程,覆盖 endpoint、SSH 凭据、远程根目录、主机密钥指纹确认、连接测试和上线验收。 +--- + +# SFTP 存储策略教程 + +::: tip 这一篇覆盖什么 +这一篇讲怎么把 AsterDrive 的文件写到 SFTP 服务器:准备 SSH 账号、创建 `sftp` 存储策略、确认 SSH 主机密钥指纹、配置策略组规则,并验收上传和下载。 +::: + +## 适合什么时候用 + +SFTP 适合这些场景: + +- 已经有一台通过 SSH/SFTP 管理的文件服务器 +- 目标存储是 NAS、传统服务器目录或只能通过 SFTP 暴露的存储 +- 不希望浏览器直接访问存储后端,上传下载都可以由 AsterDrive 服务端中继 +- 需要按用户、团队或文件大小把部分文件写到 SFTP 后端 + +如果你需要浏览器直传、预签名 URL 或对象存储 multipart 上传,优先看 [S3 / MinIO / R2](/storage/s3-minio-r2)、[Azure Blob Storage](/storage/azure-blob) 或 [腾讯云 COS](/storage/tencent-cos)。SFTP 当前是服务端流式读写后端,不提供对象存储式的直传能力。 + +## 先分清你要配哪几层 + +```mermaid +flowchart TD + Server["SFTP 服务器"] --> Account["SSH 账号 / 远程目录 / 主机密钥指纹"] + Account --> Policy["AsterDrive sftp 存储策略"] + Policy --> Rule["策略组规则"] + Rule --> Binding["用户或团队绑定策略组"] +``` + +只创建 SFTP 存储策略还不够。用户或团队真正上传时,会先命中策略组,再由策略组规则分配到某条存储策略。 + +## 这篇用到的入口 + +| 你要做什么 | 入口 | +| --- | --- | +| 创建 SFTP 策略 | `管理 -> 存储策略 -> 新建策略` | +| 测试 SFTP 连接 | `管理 -> 存储策略 -> 测试连接` | +| 创建分流规则 | `管理 -> 策略组` | +| 给用户绑定策略组 | `管理 -> 用户 -> 用户详情` | +| 给团队绑定策略组 | `管理 -> 团队 -> 团队详情` | + +## 1. 准备 SSH 账号和远程目录 + +先在 SFTP 服务器上准备一个专用账号和目录,例如: + +```text +/srv/asterdrive +``` + +建议只授予这个账号对目标目录的读写权限。AsterDrive 至少需要: + +- 创建目录 +- 写入文件 +- 读取文件 +- 删除文件 +- 读取文件元数据 + +不要把管理员账号、root 账号或和其他应用共用的目录直接交给 AsterDrive。 + +::: warning 不建议人工移动远程目录里的对象 +AsterDrive 数据库记录了对象路径。人工移动、重命名或删除 SFTP 目录里的对象,会让数据库里的文件记录和真实对象不一致。 +::: + +## 2. 创建 SFTP 存储策略 + +进入: + +```text +管理 -> 存储策略 -> 新建策略 +``` + +选择驱动类型: + +```text +sftp +``` + +填写连接信息: + +| 字段 | 示例 | 说明 | +| --- | --- | --- | +| Endpoint | `sftp://sftp.example.com:22` | SFTP 服务器地址 | +| SSH 用户名 | `asterdrive` | 内部 API 字段仍是 `access_key`,管理界面会显示为 SSH 用户名 | +| SSH 密码 | `********` | 内部 API 字段仍是 `secret_key`,管理界面会显示为 SSH 密码 | +| 基础路径 | `/srv/asterdrive` | AsterDrive 写入对象时使用的远程根目录 | +| SSH 主机密钥指纹 | `SHA256:...` | 首次连接确认后填写 | + +Endpoint 支持这些写法: + +```text +sftp://sftp.example.com:22 +sftp.example.com +sftp.example.com:2222 +``` + +没有端口时默认使用 `22`。如果写了协议,只支持 `sftp://`。不要把用户名、密码、路径、query 或 fragment 放进 Endpoint;远程根目录放到“基础路径”,账号密码放到对应凭据字段。 + +## 3. 确认 SSH 主机密钥指纹 + +SFTP 策略默认拒绝未知主机密钥。第一次测试连接时,如果还没有保存 `SSH 主机密钥指纹`,连接会失败,并在诊断信息里返回服务器实际的 `SHA256:...` 指纹。 + +正确流程是: + +1. 先填写 Endpoint、SSH 用户名、SSH 密码和基础路径 +2. 点击连接测试 +3. 从失败诊断里读取服务器返回的 `SHA256:...` 指纹 +4. 用可信渠道和服务器管理员确认这个指纹确实属于目标 SFTP 服务器 +5. 把确认后的指纹填入 `SSH 主机密钥指纹` +6. 再次连接测试,确认通过后保存策略 + +::: warning 不要跳过主机密钥确认 +如果不校验 SSH 主机密钥,攻击者可以伪装成 SFTP 服务器并获取密码。AsterDrive 会把确认后的指纹保存在 `storage_policy.options.sftp_host_key_fingerprint`,后续连接必须匹配这个指纹;服务器重装或轮换 SSH host key 后,需要重新确认并更新。 +::: + +## 4. 连接测试失败怎么查 + +连接测试失败时,后台会优先展示标准错误响应里的 `error.diagnostic.message`。SFTP 常见排查顺序: + +1. AsterDrive 服务器能否访问 Endpoint 和端口 +2. Endpoint 是否只包含 host 和可选端口 +3. SSH 用户名和密码是否正确 +4. 账号是否允许 SFTP 子系统 +5. 基础路径是否存在,或者账号是否有权限创建它 +6. `SSH 主机密钥指纹` 是否和服务器当前 host key 一致 +7. 服务器防火墙、fail2ban 或连接数限制是否拒绝了登录 + +如果错误提示主机密钥不匹配,不要直接覆盖旧指纹。先确认服务器是否确实轮换了 host key。 + +## 5. 创建测试策略组并验收 + +不要一上来直接改默认策略组。建议先创建一个测试策略组,绑定一个测试用户或测试团队,然后跑完整流程: + +1. 上传小文件 +2. 上传一个超过普通表单大小的大文件 +3. 下载文件 +4. 预览或公开分享一个普通文件 +5. 删除并恢复文件 +6. 确认 SFTP 服务器目标目录里出现对象文件 + +确认没有问题后,再把真实用户或团队切到包含 SFTP 策略的策略组。 + +## 迁移和修改边界 + +::: warning 已经写入文件的 SFTP 策略,不要直接改这些字段 + +- Endpoint +- 基础路径 +- SSH 用户名 + +旧文件按原位置读取,直接改位置 = 已有文件可能找不到。更稳的做法是新建一条 SFTP 策略,再用 `管理 -> 存储策略 -> 迁移数据` 迁移。 +::: + +SSH 密码可以通过编辑策略轮换。编辑已保存策略时,密码字段留空会保留现有凭据。 + +## 当前能力边界 + +| 能力 | 当前行为 | +| --- | --- | +| 上传 | AsterDrive 服务端流式写入 SFTP | +| 下载 | AsterDrive 服务端从 SFTP 读取后返回给浏览器 | +| 浏览器直传 | 不支持 | +| 对象存储 multipart | 不支持 | +| 预签名 URL | 不支持 | +| 容量观测 | 不支持统一容量查询 | +| Range 读取 | 支持高效 range 读取 | + +SFTP 连接池优化单独跟踪中;当前实现优先保证主机密钥校验和基础读写正确性。